diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index deb031122d..89b62de7fe 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -295,10 +295,7 @@ func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpress } func parseConfig(data []byte) ([]Group, error) { - data, err := envtemplate.ReplaceBytes(data) - if err != nil { - return nil, fmt.Errorf("cannot expand environment vars: %w", err) - } + data = envtemplate.ReplaceBytes(data) var result []Group type cfgFile struct { @@ -310,13 +307,13 @@ func parseConfig(data []byte) ([]Group, error) { decoder := yaml.NewDecoder(bytes.NewReader(data)) for { var cf cfgFile - if err = decoder.Decode(&cf); err != nil { + if err := decoder.Decode(&cf); err != nil { if err == io.EOF { // EOF indicates no more documents to read break } return nil, err } - if err = checkOverflow(cf.XXX, "config"); err != nil { + if err := checkOverflow(cf.XXX, "config"); err != nil { return nil, err } result = append(result, cf.Groups...) diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index 0d15705a5f..0e7cb35755 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -723,14 +723,11 @@ func reloadAuthConfigData(data []byte) (bool, error) { } func parseAuthConfig(data []byte) (*AuthConfig, error) { - data, err := envtemplate.ReplaceBytes(data) - if err != nil { - return nil, fmt.Errorf("cannot expand environment vars: %w", err) - } + data = envtemplate.ReplaceBytes(data) ac := &AuthConfig{ ms: metrics.NewSet(), } - if err = yaml.UnmarshalStrict(data, ac); err != nil { + if err := yaml.UnmarshalStrict(data, ac); err != nil { return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %w", err) } diff --git a/docs/victoriametrics/Cluster-VictoriaMetrics.md b/docs/victoriametrics/Cluster-VictoriaMetrics.md index a992c91fac..80483f1d92 100644 --- a/docs/victoriametrics/Cluster-VictoriaMetrics.md +++ b/docs/victoriametrics/Cluster-VictoriaMetrics.md @@ -465,20 +465,7 @@ The currently discovered `vmstorage` nodes can be [monitored](#monitoring) with ### Environment variables -All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax. -For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret` -if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup. -This expansion is performed by VictoriaMetrics itself. - -VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup. -For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`. - -Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules: - -- The `-envflag.enable` flag must be set -- Each `.` in flag names must be substituted by `_` (for example `-insert.maxQueueDuration ` will translate to `insert_maxQueueDuration=`) -- For repeating flags, an alternative syntax can be used by joining the different values into one using `,` as separator (for example `-storageNode -storageNode ` will translate to `storageNode=,`) -- It is possible setting prefix for environment vars with `-envflag.prefix`. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_` +See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#environment-variables). ## Security diff --git a/docs/victoriametrics/README.md b/docs/victoriametrics/README.md index 44eb701d29..199f299c3e 100644 --- a/docs/victoriametrics/README.md +++ b/docs/victoriametrics/README.md @@ -155,13 +155,14 @@ if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics This expansion is performed by VictoriaMetrics itself. VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup. -For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`. +For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc` environment variables exist. Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules: * The `-envflag.enable` flag must be set. * Each `.` char in flag name must be substituted with `_` (for example `-insert.maxQueueDuration ` will translate to `insert_maxQueueDuration=`). -* For repeating flags an alternative syntax can be used by joining the different values into one using `,` char as separator (for example `-storageNode -storageNode ` will translate to `storageNode=,`). +* Repated flags can be replaced by an environment variable with `,`-separted values for the repeated flags. + For example `-storageNode -storageNode ` command-line flags can be set as `storageNode=,` environment variable. * Environment var prefix can be set via `-envflag.prefix` flag. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_`. ### Setting up service diff --git a/docs/victoriametrics/changelog/CHANGELOG.md b/docs/victoriametrics/changelog/CHANGELOG.md index d637625bb6..b868a3543c 100644 --- a/docs/victoriametrics/changelog/CHANGELOG.md +++ b/docs/victoriametrics/changelog/CHANGELOG.md @@ -22,6 +22,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): protect graphite `/render` API endpoint with new flag `-search.maxGraphitePathExpressionLen`. See this PR [#9534](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9534) for details. * FEATURE: expose `vm_total_disk_space_bytes` metric at the [`/metrics` page](https://docs.victoriametrics.com/#monitoring), which shows the total disk space for the data directory specified via [`-storageDataPath`](https://docs.victoriametrics.com/#storage). This metric can be useful for building alerts and graphs for the percentatge of free disk space via `vm_free_disk_space_bytes / vm_total_disk_space_bytes`. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9523#issuecomment-3149459926). +* FEATURE: all: leave non-existing environment variables as is in config files instead of failure. For example, if the file referred by `-promscrape.config` contains `%{NON_EXISTING_ENV_VAR}` placeholder, then it is left as is instead of failing to load the file. This simplifies the usage of environment variables in config files and in command-line flags according to [these docs](https://docs.victoriametrics.com/victoriametrics/#environment-variables). Users can easily notice non-existing env vars in config files and in command-line flags by looking at their values - they will literally contain `%{NON_EXISTING_ENV_VAR}` strings. * BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): print a proper error message when templating function fails during execution. Previously, vmalert-tool could throw a misleading panic message instead. * BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly read proxy-protocol header. See this PR [#9546](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9546) for details. diff --git a/lib/envflag/envflag.go b/lib/envflag/envflag.go index d670b2a8f2..11c71010e0 100644 --- a/lib/envflag/envflag.go +++ b/lib/envflag/envflag.go @@ -68,11 +68,7 @@ func ParseFlagSet(fs *flag.FlagSet, args []string) { func expandArgs(args []string) []string { dstArgs := make([]string, 0, len(args)) for _, arg := range args { - s, err := envtemplate.ReplaceString(arg) - if err != nil { - // Do not use lib/logger here, since it is uninitialized yet. - log.Fatalf("cannot process arg %q: %s", arg, err) - } + s := envtemplate.ReplaceString(arg) if len(s) > 0 { dstArgs = append(dstArgs, s) } diff --git a/lib/envtemplate/envtemplate.go b/lib/envtemplate/envtemplate.go index 393cc44263..ac632bd0a0 100644 --- a/lib/envtemplate/envtemplate.go +++ b/lib/envtemplate/envtemplate.go @@ -3,34 +3,21 @@ package envtemplate import ( "fmt" "io" - "log" "os" - "regexp" "strings" "github.com/valyala/fasttemplate" ) // ReplaceBytes replaces `%{ENV_VAR}` placeholders in b with the corresponding ENV_VAR values. -// -// Error is returned if ENV_VAR isn't set for some `%{ENV_VAR}` placeholder. -func ReplaceBytes(b []byte) ([]byte, error) { - result, err := expand(envVars, string(b)) - if err != nil { - return nil, err - } - return []byte(result), nil +func ReplaceBytes(b []byte) []byte { + result := expand(envVars, string(b)) + return []byte(result) } // ReplaceString replaces `%{ENV_VAR}` placeholders in b with the corresponding ENV_VAR values. -// -// Error is returned if ENV_VAR isn't set for some `%{ENV_VAR}` placeholder. -func ReplaceString(s string) (string, error) { - result, err := expand(envVars, s) - if err != nil { - return "", err - } - return result, nil +func ReplaceString(s string) string { + return expand(envVars, s) } // LookupEnv returns the expanded environment variable value for the given name. @@ -70,11 +57,7 @@ func expandTemplates(m map[string]string) map[string]string { mExpanded := make(map[string]string, len(m)) expands := 0 for name, value := range m { - valueExpanded, err := expand(m, value) - if err != nil { - // Do not use lib/logger here, since it is uninitialized yet. - log.Fatalf("cannot expand %q env var value %q: %s", name, value, err) - } + valueExpanded := expand(m, value) mExpanded[name] = valueExpanded if valueExpanded != value { expands++ @@ -88,32 +71,13 @@ func expandTemplates(m map[string]string) map[string]string { return m } -func expand(m map[string]string, s string) (string, error) { - if !strings.Contains(s, "%{") { - // Fast path - nothing to expand - return s, nil - } - result, err := fasttemplate.ExecuteFuncStringWithErr(s, "%{", "}", func(w io.Writer, tag string) (int, error) { - if !isValidEnvVarName(tag) { - return fmt.Fprintf(w, "%%{%s}", tag) - } +func expand(m map[string]string, s string) string { + return fasttemplate.ExecuteFuncString(s, "%{", "}", func(w io.Writer, tag string) (int, error) { v, ok := m[tag] if !ok { - return 0, fmt.Errorf("missing %q env var", tag) + // Cannot find the tag in m. Leave it as is. + return fmt.Fprintf(w, "%%{%s}", tag) } return fmt.Fprintf(w, "%s", v) }) - if err != nil { - return "", err - } - return result, nil } - -func isValidEnvVarName(s string) bool { - return envVarNameRegex.MatchString(s) -} - -// envVarNameRegex is used for validating environment variable names. -// -// Allow dashes and dots in env var names - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3999 -var envVarNameRegex = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_\-.]*$`) diff --git a/lib/envtemplate/envtemplate_test.go b/lib/envtemplate/envtemplate_test.go index 420d679609..57d27dbdf7 100644 --- a/lib/envtemplate/envtemplate_test.go +++ b/lib/envtemplate/envtemplate_test.go @@ -57,40 +57,25 @@ func TestReplaceSuccess(t *testing.T) { } f := func(s, resultExpected string) { t.Helper() - result, err := ReplaceBytes([]byte(s)) - if err != nil { - t.Fatalf("unexpected error: %s", err) - } + result := ReplaceBytes([]byte(s)) if string(result) != resultExpected { t.Fatalf("unexpected result for ReplaceBytes(%q);\ngot\n%q\nwant\n%q", s, result, resultExpected) } - resultS, err := ReplaceString(s) - if err != nil { - t.Fatalf("unexpected error: %s", err) - } + resultS := ReplaceString(s) if resultS != resultExpected { t.Fatalf("unexpected result for ReplaceString(%q);\ngot\n%q\nwant\n%q", s, result, resultExpected) } } + + // Zero placeholders f("", "") f("foo", "foo") + + // Matching placeholders f("a %{foo}-x", "a bar-x") f("%{foo.bar_1}", "baz") f("qq.%{foo-bar_2}.ww", "qq.test.ww") -} -func TestReplaceFailure(t *testing.T) { - f := func(s string) { - t.Helper() - if _, err := ReplaceBytes([]byte(s)); err == nil { - t.Fatalf("expecting non-nil error for ReplaceBytes(%q)", s) - } - if _, err := ReplaceString(s); err == nil { - t.Fatalf("expecting non-nil error for ReplaceString(%q)", s) - } - } - f("foo %{bar} %{baz}") - f("%{Foo_Foo_1}") - f("%{Foo-Bar-2}") - f("%{Foo.Baz.3}") + // Missing placeholders + f("foo %{bar} %{baz} %{foo}", "foo %{bar} %{baz} bar") } diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go index 0c7da81c87..a687e421f4 100644 --- a/lib/promrelabel/config.go +++ b/lib/promrelabel/config.go @@ -160,10 +160,7 @@ func LoadRelabelConfigs(path string) (*ParsedConfigs, error) { if err != nil { return nil, fmt.Errorf("cannot read `relabel_configs` from %q: %w", path, err) } - data, err = envtemplate.ReplaceBytes(data) - if err != nil { - return nil, fmt.Errorf("cannot expand environment vars at %q: %w", path, err) - } + data = envtemplate.ReplaceBytes(data) pcs, err := ParseRelabelConfigsData(data) if err != nil { return nil, fmt.Errorf("cannot unmarshal `relabel_configs` from %q: %w", path, err) diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 74d0598eab..b9acbb79b8 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -121,19 +121,15 @@ type Config struct { } func (cfg *Config) unmarshal(data []byte, isStrict bool) error { - var err error - data, err = envtemplate.ReplaceBytes(data) - if err != nil { - return fmt.Errorf("cannot expand environment variables: %w", err) + data = envtemplate.ReplaceBytes(data) + if !isStrict { + return yaml.Unmarshal(data, cfg) } - if isStrict { - if err = yaml.UnmarshalStrict(data, cfg); err != nil { - err = fmt.Errorf("%w; pass -promscrape.config.strictParse=false command-line flag for ignoring unknown fields in yaml config", err) - } - } else { - err = yaml.Unmarshal(data, cfg) + + if err := yaml.UnmarshalStrict(data, cfg); err != nil { + return fmt.Errorf("%w; pass -promscrape.config.strictParse=false command-line flag for ignoring unknown fields in yaml config", err) } - return err + return nil } func (cfg *Config) marshal() []byte { @@ -441,10 +437,7 @@ func loadStaticConfigs(path string) ([]StaticConfig, error) { if err != nil { return nil, fmt.Errorf("cannot read `static_configs` from %q: %w", path, err) } - data, err = envtemplate.ReplaceBytes(data) - if err != nil { - return nil, fmt.Errorf("cannot expand environment vars in %q: %w", path, err) - } + data = envtemplate.ReplaceBytes(data) var stcs []StaticConfig if err := yaml.UnmarshalStrict(data, &stcs); err != nil { return nil, fmt.Errorf("cannot unmarshal `static_configs` from %q: %w", path, err) @@ -485,11 +478,7 @@ func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string, isStrict logger.Errorf("skipping %q at `scrape_config_files` because of error: %s", path, err) continue } - data, err = envtemplate.ReplaceBytes(data) - if err != nil { - logger.Errorf("skipping %q at `scrape_config_files` because of failure to expand environment vars: %s", path, err) - continue - } + data = envtemplate.ReplaceBytes(data) var scs []*ScrapeConfig if isStrict { if err = yaml.UnmarshalStrict(data, &scs); err != nil { diff --git a/lib/streamaggr/streamaggr.go b/lib/streamaggr/streamaggr.go index 320544b7bc..e518190d6b 100644 --- a/lib/streamaggr/streamaggr.go +++ b/lib/streamaggr/streamaggr.go @@ -77,10 +77,7 @@ func LoadFromFile(path string, pushFunc PushFunc, opts *Options, alias string) ( if err != nil { return nil, fmt.Errorf("cannot load aggregators: %w", err) } - data, err = envtemplate.ReplaceBytes(data) - if err != nil { - return nil, fmt.Errorf("cannot expand environment variables in %q: %w", path, err) - } + data = envtemplate.ReplaceBytes(data) as, err := loadFromData(data, path, pushFunc, opts, alias) if err != nil {