overspend1/over-rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rc: add a full set of stats to core/stats

Browse Source 
This patch adds the missing stats to the output of core/stats

- totalChecks
- totalTransfers
- totalBytes
- eta

This now includes enough information to rebuild the normal stats
output from rclone including percentage completions and ETAs.

Fixes #5116
This commit is contained in:
Nick Craig-Wood
2021-03-15 15:50:04 +00:00
parent b47d6001a9
commit 268a7ff7b8
4 changed files with 96 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
fs/accounting/accounting.go
View File

@@ -527,14 +527,11 @@ func (acc *Account) rcStats() (out rc.Params) {
out["speed"] = spd
out["speedAvg"] = cur
eta, etaok := acc.eta()
out["eta"] = nil
if etaok {
if eta > 0 {
out["eta"] = eta.Seconds()
} else {
out["eta"] = 0
}
eta, etaOK := acc.eta()
if etaOK {
out["eta"] = eta.Seconds()
} else {
out["eta"] = nil
}
out["name"] = acc.name

2
fs/accounting/prometheus.go
View File

@@ -90,7 +90,7 @@ func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) {
s.mu.RLock()
ch <- prometheus.MustNewConstMetric(c.bytesTransferred, prometheus.CounterValue, float64(s.bytes))
ch <- prometheus.MustNewConstMetric(c.transferSpeed, prometheus.GaugeValue, s.Speed())
ch <- prometheus.MustNewConstMetric(c.transferSpeed, prometheus.GaugeValue, s.speed())
ch <- prometheus.MustNewConstMetric(c.numOfErrors, prometheus.CounterValue, float64(s.errors))
ch <- prometheus.MustNewConstMetric(c.numOfCheckFiles, prometheus.CounterValue, float64(s.checks))
ch <- prometheus.MustNewConstMetric(c.transferredFiles, prometheus.CounterValue, float64(s.transfers))

106
fs/accounting/stats.go
View File

@@ -65,9 +65,19 @@ func NewStats(ctx context.Context) *StatsInfo {
// RemoteStats returns stats for rc
func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
// NB if adding values here - make sure you update the docs in
// stats_groups.go
out = make(rc.Params)
ts := s.calculateTransferStats()
out["totalChecks"] = ts.totalChecks
out["totalTransfers"] = ts.totalTransfers
out["totalBytes"] = ts.totalBytes
out["transferTime"] = ts.transferTime
out["speed"] = ts.speed
s.mu.RLock()
out["speed"] = s.Speed()
out["bytes"] = s.bytes
out["errors"] = s.errors
out["fatalError"] = s.fatalError
@@ -77,9 +87,15 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
out["deletes"] = s.deletes
out["deletedDirs"] = s.deletedDirs
out["renames"] = s.renames
out["transferTime"] = s.totalDuration().Seconds()
out["elapsedTime"] = time.Since(startTime).Seconds()
eta, etaOK := eta(s.bytes, ts.totalBytes, ts.speed)
if etaOK {
out["eta"] = eta.Seconds()
} else {
out["eta"] = nil
}
s.mu.RUnlock()
if !s.checking.empty() {
out["checking"] = s.checking.remotes()
}
@@ -89,11 +105,14 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
if s.errors > 0 {
out["lastError"] = s.lastError.Error()
}
return out, nil
}
// Speed returns the average speed of the transfer in bytes/second
func (s *StatsInfo) Speed() float64 {
//
// Call with lock held
func (s *StatsInfo) speed() float64 {
dt := s.totalDuration()
dtSeconds := dt.Seconds()
speed := 0.0
@@ -202,6 +221,9 @@ func eta(size, total int64, rate float64) (eta time.Duration, ok bool) {
return 0, false
}
seconds := float64(remaining) / rate
if seconds < 0 {
seconds = 0
}
return time.Second * time.Duration(seconds), true
}
@@ -227,36 +249,60 @@ func percent(a int64, b int64) string {
return fmt.Sprintf("%d%%", int(float64(a)*100/float64(b)+0.5))
}
// String convert the StatsInfo to a string for printing
func (s *StatsInfo) String() string {
// returned from calculateTransferStats
type transferStats struct {
totalChecks int64
totalTransfers int64
totalBytes int64
transferTime float64
speed float64
}
// calculateTransferStats calculates some addtional transfer stats not
// stored directly in StatsInfo
func (s *StatsInfo) calculateTransferStats() (ts transferStats) {
// checking and transferring have their own locking so read
// here before lock to prevent deadlock on GetBytes
transferring, checking := s.transferring.count(), s.checking.count()
transferringBytesDone, transferringBytesTotal := s.transferring.progress(s)
s.mu.RLock()
defer s.mu.RUnlock()
ts.totalChecks = int64(s.checkQueue) + s.checks + int64(checking)
ts.totalTransfers = int64(s.transferQueue) + s.transfers + int64(transferring)
// note that s.bytes already includes transferringBytesDone so
// we take it off here to avoid double counting
ts.totalBytes = s.transferQueueSize + s.bytes + transferringBytesTotal - transferringBytesDone
dt := s.totalDuration()
ts.transferTime = dt.Seconds()
ts.speed = 0.0
if dt > 0 {
ts.speed = float64(s.bytes) / ts.transferTime
}
return ts
}
// String convert the StatsInfo to a string for printing
func (s *StatsInfo) String() string {
// NB if adding more stats in here, remember to add them into
// RemoteStats() too.
ts := s.calculateTransferStats()
s.mu.RLock()
elapsedTime := time.Since(startTime)
elapsedTimeSecondsOnly := elapsedTime.Truncate(time.Second/10) % time.Minute
dt := s.totalDuration()
dtSeconds := dt.Seconds()
speed := 0.0
if dt > 0 {
speed = float64(s.bytes) / dtSeconds
}
displaySpeed := speed
displaySpeed := ts.speed
if s.ci.DataRateUnit == "bits" {
displaySpeed *= 8
}
var (
totalChecks = int64(s.checkQueue) + s.checks + int64(checking)
totalTransfer = int64(s.transferQueue) + s.transfers + int64(transferring)
// note that s.bytes already includes transferringBytesDone so
// we take it off here to avoid double counting
totalSize = s.transferQueueSize + s.bytes + transferringBytesTotal - transferringBytesDone
currentSize = s.bytes
buf = &bytes.Buffer{}
xfrchkString = ""
dateString = ""
@@ -266,11 +312,11 @@ func (s *StatsInfo) String() string {
_, _ = fmt.Fprintf(buf, "\nTransferred: ")
} else {
xfrchk := []string{}
if totalTransfer > 0 && s.transferQueue > 0 {
xfrchk = append(xfrchk, fmt.Sprintf("xfr#%d/%d", s.transfers, totalTransfer))
if ts.totalTransfers > 0 && s.transferQueue > 0 {
xfrchk = append(xfrchk, fmt.Sprintf("xfr#%d/%d", s.transfers, ts.totalTransfers))
}
if totalChecks > 0 && s.checkQueue > 0 {
xfrchk = append(xfrchk, fmt.Sprintf("chk#%d/%d", s.checks, totalChecks))
if ts.totalChecks > 0 && s.checkQueue > 0 {
xfrchk = append(xfrchk, fmt.Sprintf("chk#%d/%d", s.checks, ts.totalChecks))
}
if len(xfrchk) > 0 {
xfrchkString = fmt.Sprintf(" (%s)", strings.Join(xfrchk, ", "))
@@ -284,16 +330,16 @@ func (s *StatsInfo) String() string {
_, _ = fmt.Fprintf(buf, "%s%10s / %s, %s, %s, ETA %s%s",
dateString,
fs.SizeSuffix(s.bytes),
fs.SizeSuffix(totalSize).Unit("Bytes"),
percent(s.bytes, totalSize),
fs.SizeSuffix(ts.totalBytes).Unit("Bytes"),
percent(s.bytes, ts.totalBytes),
fs.SizeSuffix(displaySpeed).Unit(strings.Title(s.ci.DataRateUnit)+"/s"),
etaString(currentSize, totalSize, speed),
etaString(s.bytes, ts.totalBytes, ts.speed),
xfrchkString,
)
if s.ci.ProgressTerminalTitle {
// Writes ETA to the terminal title
terminal.WriteTerminalTitle("ETA: " + etaString(currentSize, totalSize, speed))
terminal.WriteTerminalTitle("ETA: " + etaString(s.bytes, ts.totalBytes, ts.speed))
}
if !s.ci.StatsOneLine {
@@ -314,9 +360,9 @@ func (s *StatsInfo) String() string {
_, _ = fmt.Fprintf(buf, "Errors: %10d%s\n",
s.errors, errorDetails)
}
if s.checks != 0 || totalChecks != 0 {
if s.checks != 0 || ts.totalChecks != 0 {
_, _ = fmt.Fprintf(buf, "Checks: %10d / %d, %s\n",
s.checks, totalChecks, percent(s.checks, totalChecks))
s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks))
}
if s.deletes != 0 || s.deletedDirs != 0 {
_, _ = fmt.Fprintf(buf, "Deleted: %10d (files), %d (dirs)\n", s.deletes, s.deletedDirs)
@@ -324,9 +370,9 @@ func (s *StatsInfo) String() string {
if s.renames != 0 {
_, _ = fmt.Fprintf(buf, "Renamed: %10d\n", s.renames)
}
if s.transfers != 0 || totalTransfer != 0 {
if s.transfers != 0 || ts.totalTransfers != 0 {
_, _ = fmt.Fprintf(buf, "Transferred: %10d / %d, %s\n",
s.transfers, totalTransfer, percent(s.transfers, totalTransfer))
s.transfers, ts.totalTransfers, percent(s.transfers, ts.totalTransfers))
}
_, _ = fmt.Fprintf(buf, "Elapsed time: %10ss\n", strings.TrimRight(elapsedTime.Truncate(time.Minute).String(), "0s")+fmt.Sprintf("%.1f", elapsedTimeSecondsOnly.Seconds()))
}

24
fs/accounting/stats_groups.go
View File

@@ -86,18 +86,22 @@ Returns the following values:
` + "```" + `
{
"speed": average speed in bytes/sec since start of the process,
"bytes": total transferred bytes since the start of the process,
"bytes": total transferred bytes since the start of the group,
"checks": number of files checked,
"deletes" : number of files deleted,
"elapsedTime": time in floating point seconds since rclone was started,
"errors": number of errors,
"fatalError": whether there has been at least one FatalError,
"retryError": whether there has been at least one non-NoRetryError,
"checks": number of checked files,
"transfers": number of transferred files,
"deletes" : number of deleted files,
"renames" : number of renamed files,
"eta": estimated time in seconds until the group completes,
"fatalError": boolean whether there has been at least one fatal error,
"lastError": last error string,
"renames" : number of files renamed,
"retryError": boolean showing whether there has been at least one non-NoRetryError,
"speed": average speed in bytes/sec since start of the group,
"totalBytes": total number of bytes in the group,
"totalChecks": total number of checks in the group,
"totalTransfers": total number of transfers in the group,
"transferTime" : total time spent on running jobs,
"elapsedTime": time in seconds since the start of the process,
"lastError": last occurred error,
"transfers": number of transferred files,
"transferring": an array of currently active file transfers:
[
{