rms/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2026-05-17 00:26:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

apply review suggestions

Browse Source
This commit is contained in:
Andrii Chubatiuk
2025-11-17 14:30:54 +02:00
parent cbf896b0da
commit d0fd4e0da4
3 changed files with 52 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
docs/victoriametrics/changelog/CHANGELOG.md
View File

@@ -26,6 +26,8 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
## tip
* FEATURE: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/), [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): add support for SSE KMS Key ID and ACL for use with S3-compatible storages.
## [v1.130.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.130.0)
Released at 2025-11-14
@@ -124,7 +126,6 @@ Released at 2025-10-03
* BUGFIX: all components: restore sorting order of summary and quantile metrics exposed by VictoriaMetrics components on `/metrics` page. See [metrics#105](https://github.com/VictoriaMetrics/metrics/pull/105) for details.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): avoid applying offset modifier twice to the request time when an instant query uses rollup functions `rate()` or `avg_over_time()` with cache enabled. See [#9762](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): restore support for `query` templates in alert rule labels after the regression introduced in [#9543](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9543). See [#9783](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9783) for details.
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/), [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): add support for SSE KMS Key ID and ACLfor use with S3-compatible storages.
## [v1.126.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.126.0)

22
lib/backup/actions/util.go
View File

@@ -5,7 +5,6 @@ import (
"flag"
"fmt"
"path/filepath"
"slices"
"strings"
"sync"
"time"
@@ -16,7 +15,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/gcsremote"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/s3remote"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
)
var (
@@ -264,22 +262,6 @@ func NewRemoteFS(ctx context.Context, path string) (common.RemoteFS, error) {
return nil, fmt.Errorf("cannot parse s3 tags %q: %w", *s3Tags, err)
}
sseAlgorithm := s3types.ServerSideEncryptionAwsKms
if s3SSEAlgorithm != nil && *s3SSEAlgorithm != "" {
if !slices.Contains(sseAlgorithm.Values(), s3types.ServerSideEncryption(*s3SSEAlgorithm)) {
return nil, fmt.Errorf("unsupported SSE algorithm: %s. Supported values: %v", *s3SSEAlgorithm, sseAlgorithm.Values())
}
sseAlgorithm = s3types.ServerSideEncryption(*s3SSEAlgorithm)
}
acl := s3types.ObjectCannedACL("")
if s3ACL != nil && *s3ACL != "" {
if !slices.Contains(acl.Values(), s3types.ObjectCannedACL(*s3ACL)) {
return nil, fmt.Errorf("unsupported ACL: %s. Supported values: %v", *s3ACL, acl.Values())
}
acl = s3types.ObjectCannedACL(*s3ACL)
}
fs := &s3remote.FS{
CredsFilePath: *credsFilePath,
ConfigFilePath: *configFilePath,
@@ -288,9 +270,9 @@ func NewRemoteFS(ctx context.Context, path string) (common.RemoteFS, error) {
StorageClass: s3remote.StringToStorageClass(*s3StorageClass),
ChecksumAlgorithm: s3remote.StringToChecksumAlgorithm(*s3ChecksumAlgorithm),
S3ForcePathStyle: *s3ForcePathStyle,
ACL: acl,
ACL: s3remote.StringToObjectACL(*s3ACL),
SSEKMSKeyId: *s3SSEKMSKeyId,
SSEAlgorithm: sseAlgorithm,
SSEAlgorithm: s3remote.StringToEncryptionAlgorithm(*s3SSEAlgorithm),
ProfileName: *configProfile,
Bucket: bucket,
Dir: dir,

72
lib/backup/s3remote/s3.go
View File

@@ -8,6 +8,7 @@ import (
"io"
"net/http"
"path"
"slices"
"sort"
"strings"
"time"
@@ -26,34 +27,54 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
)
var (
supportedStorageClasses = []s3types.StorageClass{s3types.StorageClassGlacier, s3types.StorageClassDeepArchive, s3types.StorageClassGlacierIr, s3types.StorageClassIntelligentTiering, s3types.StorageClassOnezoneIa, s3types.StorageClassOutposts, s3types.StorageClassReducedRedundancy, s3types.StorageClassStandard, s3types.StorageClassStandardIa}
)
func validateStorageClass(storageClass s3types.StorageClass) error {
func validateStorageClass(v s3types.StorageClass) error {
// if no storageClass set, no need to validate against supported values
// backwards compatibility
if len(storageClass) == 0 {
if len(v) == 0 || slices.Contains(v.Values(), v) {
return nil
}
return fmt.Errorf("unsupported S3 storage class %q. Supported values: %v", v, v.Values())
}
for _, supported := range supportedStorageClasses {
if supported == storageClass {
return nil
}
func validateObjectACL(v s3types.ObjectCannedACL) error {
if len(v) == 0 || slices.Contains(v.Values(), v) {
return nil
}
return fmt.Errorf("unsupported S3 object ACL %q. Supported values: %v", v, v.Values())
}
return fmt.Errorf("unsupported S3 storage class: %s. Supported values: %v", storageClass, supportedStorageClasses)
func validateChecksumAlgorithm(v s3types.ChecksumAlgorithm) error {
if len(v) == 0 || slices.Contains(v.Values(), v) {
return nil
}
return fmt.Errorf("unsupported S3 checksum algorithm %q. Supported values: %v", v, v.Values())
}
func validateSSEAlgorithm(v s3types.ServerSideEncryption) error {
if len(v) == 0 || slices.Contains(v.Values(), v) {
return nil
}
return fmt.Errorf("unsupported S3 server-side algorithm %q. Supported values: %v", v, v.Values())
}
// StringToStorageClass converts string types to AWS S3 StorageClass type for value comparison
func StringToStorageClass(sc string) s3types.StorageClass {
return s3types.StorageClass(sc)
func StringToStorageClass(s string) s3types.StorageClass {
return s3types.StorageClass(s)
}
// StringToChecksumAlgorithm converts string types to AWS S3 ChecksumAlgorithm type for value comparison
func StringToChecksumAlgorithm(alg string) s3types.ChecksumAlgorithm {
return s3types.ChecksumAlgorithm(alg)
func StringToChecksumAlgorithm(s string) s3types.ChecksumAlgorithm {
return s3types.ChecksumAlgorithm(s)
}
// StringToObjectACL converts string types to AWS S3 ACL type for value comparison
func StringToObjectACL(s string) s3types.ObjectCannedACL {
return s3types.ObjectCannedACL(s)
}
// StringToEncryptionAlgorithm converts string types to AWS S3 server-side encryption type for value comparison
func StringToEncryptionAlgorithm(s string) s3types.ServerSideEncryption {
return s3types.ServerSideEncryption(s)
}
// FS represents filesystem for backups in S3.
@@ -174,6 +195,15 @@ func (fs *FS) Init(ctx context.Context) error {
if err = validateStorageClass(fs.StorageClass); err != nil {
return err
}
if err = validateChecksumAlgorithm(fs.ChecksumAlgorithm); err != nil {
return err
}
if err = validateObjectACL(fs.ACL); err != nil {
return err
}
if err = validateSSEAlgorithm(fs.SSEAlgorithm); err != nil {
return err
}
// Use AWS client in order to allow SDK to override transport configuration
// based on additional configuration from environment variables.
@@ -327,14 +357,12 @@ func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error {
Metadata: fs.Metadata,
MetadataDirective: s3types.MetadataDirectiveReplace,
Tagging: fs.tags,
ACL: fs.ACL,
}
if len(fs.SSEKMSKeyId) > 0 {
input.SSEKMSKeyId = aws.String(fs.SSEKMSKeyId)
input.ServerSideEncryption = fs.SSEAlgorithm
}
if len(fs.ACL) > 0 {
input.ACL = fs.ACL
}
_, err := fs.s3.CopyObject(fs.ctx, input)
if err != nil {
@@ -382,14 +410,12 @@ func (fs *FS) UploadPart(p common.Part, r io.Reader) error {
Metadata: fs.Metadata,
ChecksumAlgorithm: fs.ChecksumAlgorithm,
Tagging: fs.tags,
ACL: fs.ACL,
}
if len(fs.SSEKMSKeyId) > 0 {
input.SSEKMSKeyId = aws.String(fs.SSEKMSKeyId)
input.ServerSideEncryption = fs.SSEAlgorithm
}
if len(fs.ACL) > 0 {
input.ACL = fs.ACL
}
_, err := fs.uploader.Upload(fs.ctx, input)
if err != nil {
@@ -482,14 +508,12 @@ func (fs *FS) CreateFile(filePath string, data []byte) error {
Metadata: fs.Metadata,
ChecksumAlgorithm: fs.ChecksumAlgorithm,
Tagging: fs.tags,
ACL: fs.ACL,
}
if len(fs.SSEKMSKeyId) > 0 {
input.SSEKMSKeyId = aws.String(fs.SSEKMSKeyId)
input.ServerSideEncryption = fs.SSEAlgorithm
}
if len(fs.ACL) > 0 {
input.ACL = fs.ACL
}
_, err := fs.uploader.Upload(fs.ctx, input)
if err != nil {