rms/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2026-05-17 00:26:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

lib/backup/actions: improve progress logging (#9836)

Browse Source 
Currently, it is hard to make sense of progress based on logging as it
requires manual calculation of progress and ETA.
Solve this by:
- making data units humanly readable
- adding an estimation of completion for the operation

---------

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
This commit is contained in:
Zakhar Bessarab
2025-10-10 18:34:01 +04:00
committed by GitHub
parent 4e86ddfdaa
commit b703768e9c
2 changed files with 30 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
lib/backup/actions/backup.go
View File

@@ -14,6 +14,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsnil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot/snapshotutil"
)
@@ -144,6 +145,7 @@ func runBackup(src *fslocal.FS, dst common.RemoteFS, origin common.OriginFS, con
srcCopyParts := common.PartsDifference(partsToCopy, originParts)
uploadSize := getPartsSize(srcCopyParts)
uploadSizeHuman := formatutil.HumanizeBytes(float64(uploadSize))
if len(srcCopyParts) > 0 {
logger.Infof("uploading %d parts from %s to %s", len(srcCopyParts), src, dst)
var bytesUploaded atomic.Uint64
@@ -165,9 +167,21 @@ func runBackup(src *fslocal.FS, dst common.RemoteFS, origin common.OriginFS, con
}
return nil
}, func(elapsed time.Duration) {
if elapsed.Seconds() <= 0 {
// The only way for this to happen is when the operation is immediately canceled.
// There is no need to log progress in this case, and this prevents division by zero below.
return
}
n := bytesUploaded.Load()
uploadedHuman := formatutil.HumanizeBytes(float64(n))
prc := 100 * float64(n) / float64(uploadSize)
logger.Infof("uploaded %d out of %d bytes (%.2f%%) from %s to %s in %s", n, uploadSize, prc, src, dst, elapsed)
speed := float64(n) / elapsed.Seconds()
estimatedTotal := time.Duration(float64(uploadSize)/speed) * time.Second
eta := estimatedTotal - elapsed
if eta < 0 {
eta = 0
}
logger.Infof("uploaded %s out of %s bytes (%.2f%%) from %s to %s in %s; estimated time to completion: %s", uploadedHuman, uploadSizeHuman, prc, src, dst, elapsed, eta)
})
if err != nil {
return err

16
lib/backup/actions/restore.go
View File

@@ -14,6 +14,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/backupnames"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
@@ -148,6 +149,7 @@ func (r *Restore) Run(ctx context.Context) error {
partsToCopy := common.PartsDifference(srcParts, dstParts)
downloadSize := getPartsSize(partsToCopy)
downloadSizeHuman := formatutil.HumanizeBytes(float64(downloadSize))
if len(partsToCopy) > 0 {
perPath := make(map[string][]common.Part)
for _, p := range partsToCopy {
@@ -180,9 +182,21 @@ func (r *Restore) Run(ctx context.Context) error {
}
return nil
}, func(elapsed time.Duration) {
if elapsed.Seconds() <= 0 {
// The only way for this to happen is when the operation is immediately canceled.
// There is no need to log progress in this case, and this prevents division by zero below.
return
}
n := bytesDownloaded.Load()
downloadedHuman := formatutil.HumanizeBytes(float64(n))
prc := 100 * float64(n) / float64(downloadSize)
logger.Infof("downloaded %d out of %d bytes (%.2f%%) from %s to %s in %s", n, downloadSize, prc, src, dst, elapsed)
speed := float64(n) / elapsed.Seconds()
estimatedTotal := time.Duration(float64(downloadSize)/speed) * time.Second
eta := estimatedTotal - elapsed
if eta < 0 {
eta = 0
}
logger.Infof("downloaded %s out of %s bytes (%.2f%%) from %s to %s in %s; estimated time to completion: %s", downloadedHuman, downloadSizeHuman, prc, src, dst, elapsed, eta)
})
if err != nil {
return err