lib/envtemplate: allow referring non-existing environment variables in config files and in command-line flags

A few users reported unexpected errors when environment variables referred other environment variables
at VictoriaMetrics startup. This resulted in the following fatal error on startup:

    cannot expand "..." env var value "...%{SOME_NON_EXISTING_ENV_VAR}..."

Fix this by leaving placeholders with non-existing env vars as is.
This improves the general usability of environment variables by VictoriaMetrics components
inside command-line flags and inside config files. User can easily notice placeholders with non-existing
environment variables by looking at the corresponding command-line flag or at the corresponding config option value.

While at it, replace duplicate docs about environment variables at the https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#environment-variables
with the link to the same docs at https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#environment-variables .

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3999

app/vmalert/config/config.go

@@ -295,10 +295,7 @@ func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpress
 }
 
 func parseConfig(data []byte) ([]Group, error) {
-	data, err := envtemplate.ReplaceBytes(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
-	}
+	data = envtemplate.ReplaceBytes(data)
 
 	var result []Group
 	type cfgFile struct {
@@ -310,13 +307,13 @@ func parseConfig(data []byte) ([]Group, error) {
 	decoder := yaml.NewDecoder(bytes.NewReader(data))
 	for {
 		var cf cfgFile
-		if err = decoder.Decode(&cf); err != nil {
+		if err := decoder.Decode(&cf); err != nil {
 			if err == io.EOF { // EOF indicates no more documents to read
 				break
 			}
 			return nil, err
 		}
-		if err = checkOverflow(cf.XXX, "config"); err != nil {
+		if err := checkOverflow(cf.XXX, "config"); err != nil {
 			return nil, err
 		}
 		result = append(result, cf.Groups...)

app/vmauth/auth_config.go

@@ -723,14 +723,11 @@ func reloadAuthConfigData(data []byte) (bool, error) {
 }
 
 func parseAuthConfig(data []byte) (*AuthConfig, error) {
-	data, err := envtemplate.ReplaceBytes(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
-	}
+	data = envtemplate.ReplaceBytes(data)
 	ac := &AuthConfig{
 		ms: metrics.NewSet(),
 	}
-	if err = yaml.UnmarshalStrict(data, ac); err != nil {
+	if err := yaml.UnmarshalStrict(data, ac); err != nil {
 		return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %w", err)
 	}
 

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -465,20 +465,7 @@ The currently discovered `vmstorage` nodes can be [monitored](#monitoring) with
 
 ### Environment variables
 
-All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax.
-For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret`
-if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup.
-This expansion is performed by VictoriaMetrics itself.
-
-VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup.
-For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`.
-
-Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules:
-
-- The `-envflag.enable` flag must be set
-- Each `.` in flag names must be substituted by `_` (for example `-insert.maxQueueDuration <duration>` will translate to `insert_maxQueueDuration=<duration>`)
-- For repeating flags, an alternative syntax can be used by joining the different values into one using `,` as separator (for example `-storageNode <nodeA> -storageNode <nodeB>` will translate to `storageNode=<nodeA>,<nodeB>`)
-- It is possible setting prefix for environment vars with `-envflag.prefix`. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_`
+See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#environment-variables).
 
 ## Security
 

docs/victoriametrics/README.md

@@ -155,13 +155,14 @@ if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics
 This expansion is performed by VictoriaMetrics itself.
 
 VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup.
-For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`.
+For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc` environment variables exist.
 
 Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules:
 
 * The `-envflag.enable` flag must be set.
 * Each `.` char in flag name must be substituted with `_` (for example `-insert.maxQueueDuration <duration>` will translate to `insert_maxQueueDuration=<duration>`).
-* For repeating flags an alternative syntax can be used by joining the different values into one using `,` char as separator (for example `-storageNode <nodeA> -storageNode <nodeB>` will translate to `storageNode=<nodeA>,<nodeB>`).
+* Repated flags can be replaced by an environment variable with `,`-separted values for the repeated flags.
+  For example `-storageNode <nodeA> -storageNode <nodeB>` command-line flags can be set as `storageNode=<nodeA>,<nodeB>` environment variable.
 * Environment var prefix can be set via `-envflag.prefix` flag. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_`.
 
 ### Setting up service

docs/victoriametrics/changelog/CHANGELOG.md

@@ -22,6 +22,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): protect graphite `/render` API endpoint with new flag `-search.maxGraphitePathExpressionLen`. See this PR [#9534](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9534) for details.
 * FEATURE: expose `vm_total_disk_space_bytes` metric at the [`/metrics` page](https://docs.victoriametrics.com/#monitoring), which shows the total disk space for the data directory specified via [`-storageDataPath`](https://docs.victoriametrics.com/#storage). This metric can be useful for building alerts and graphs for the percentatge of free disk space via `vm_free_disk_space_bytes / vm_total_disk_space_bytes`. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9523#issuecomment-3149459926).
+* FEATURE: all: leave non-existing environment variables as is in config files instead of failure. For example, if the file referred by `-promscrape.config` contains `%{NON_EXISTING_ENV_VAR}` placeholder, then it is left as is instead of failing to load the file. This simplifies the usage of environment variables in config files and in command-line flags according to [these docs](https://docs.victoriametrics.com/victoriametrics/#environment-variables). Users can easily notice non-existing env vars in config files and in command-line flags by looking at their values - they will literally contain `%{NON_EXISTING_ENV_VAR}` strings.
 
 * BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): print a proper error message when templating function fails during execution. Previously, vmalert-tool could throw a misleading panic message instead.
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly read proxy-protocol header. See this PR [#9546](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9546) for details.

lib/envflag/envflag.go

@@ -68,11 +68,7 @@ func ParseFlagSet(fs *flag.FlagSet, args []string) {
 func expandArgs(args []string) []string {
 	dstArgs := make([]string, 0, len(args))
 	for _, arg := range args {
-		s, err := envtemplate.ReplaceString(arg)
-		if err != nil {
-			// Do not use lib/logger here, since it is uninitialized yet.
-			log.Fatalf("cannot process arg %q: %s", arg, err)
-		}
+		s := envtemplate.ReplaceString(arg)
 		if len(s) > 0 {
 			dstArgs = append(dstArgs, s)
 		}

lib/envtemplate/envtemplate.go

@@ -3,34 +3,21 @@ package envtemplate
 import (
 	"fmt"
 	"io"
-	"log"
 	"os"
-	"regexp"
 	"strings"
 
 	"github.com/valyala/fasttemplate"
 )
 
 // ReplaceBytes replaces `%{ENV_VAR}` placeholders in b with the corresponding ENV_VAR values.
-//
-// Error is returned if ENV_VAR isn't set for some `%{ENV_VAR}` placeholder.
-func ReplaceBytes(b []byte) ([]byte, error) {
-	result, err := expand(envVars, string(b))
-	if err != nil {
-		return nil, err
-	}
-	return []byte(result), nil
+func ReplaceBytes(b []byte) []byte {
+	result := expand(envVars, string(b))
+	return []byte(result)
 }
 
 // ReplaceString replaces `%{ENV_VAR}` placeholders in b with the corresponding ENV_VAR values.
-//
-// Error is returned if ENV_VAR isn't set for some `%{ENV_VAR}` placeholder.
-func ReplaceString(s string) (string, error) {
-	result, err := expand(envVars, s)
-	if err != nil {
-		return "", err
-	}
-	return result, nil
+func ReplaceString(s string) string {
+	return expand(envVars, s)
 }
 
 // LookupEnv returns the expanded environment variable value for the given name.
@@ -70,11 +57,7 @@ func expandTemplates(m map[string]string) map[string]string {
 		mExpanded := make(map[string]string, len(m))
 		expands := 0
 		for name, value := range m {
-			valueExpanded, err := expand(m, value)
-			if err != nil {
-				// Do not use lib/logger here, since it is uninitialized yet.
-				log.Fatalf("cannot expand %q env var value %q: %s", name, value, err)
-			}
+			valueExpanded := expand(m, value)
 			mExpanded[name] = valueExpanded
 			if valueExpanded != value {
 				expands++
@@ -88,32 +71,13 @@ func expandTemplates(m map[string]string) map[string]string {
 	return m
 }
 
-func expand(m map[string]string, s string) (string, error) {
-	if !strings.Contains(s, "%{") {
-		// Fast path - nothing to expand
-		return s, nil
-	}
-	result, err := fasttemplate.ExecuteFuncStringWithErr(s, "%{", "}", func(w io.Writer, tag string) (int, error) {
-		if !isValidEnvVarName(tag) {
-			return fmt.Fprintf(w, "%%{%s}", tag)
-		}
+func expand(m map[string]string, s string) string {
+	return fasttemplate.ExecuteFuncString(s, "%{", "}", func(w io.Writer, tag string) (int, error) {
 		v, ok := m[tag]
 		if !ok {
-			return 0, fmt.Errorf("missing %q env var", tag)
+			// Cannot find the tag in m. Leave it as is.
+			return fmt.Fprintf(w, "%%{%s}", tag)
 		}
 		return fmt.Fprintf(w, "%s", v)
 	})
-	if err != nil {
-		return "", err
-	}
-	return result, nil
 }
-
-func isValidEnvVarName(s string) bool {
-	return envVarNameRegex.MatchString(s)
-}
-
-// envVarNameRegex is used for validating environment variable names.
-//
-// Allow dashes and dots in env var names - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3999
-var envVarNameRegex = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_\-.]*$`)

lib/envtemplate/envtemplate_test.go

@@ -57,40 +57,25 @@ func TestReplaceSuccess(t *testing.T) {
 	}
 	f := func(s, resultExpected string) {
 		t.Helper()
-		result, err := ReplaceBytes([]byte(s))
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
+		result := ReplaceBytes([]byte(s))
 		if string(result) != resultExpected {
 			t.Fatalf("unexpected result for ReplaceBytes(%q);\ngot\n%q\nwant\n%q", s, result, resultExpected)
 		}
-		resultS, err := ReplaceString(s)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
+		resultS := ReplaceString(s)
 		if resultS != resultExpected {
 			t.Fatalf("unexpected result for ReplaceString(%q);\ngot\n%q\nwant\n%q", s, result, resultExpected)
 		}
 	}
+
+	// Zero placeholders
 	f("", "")
 	f("foo", "foo")
+
+	// Matching placeholders
 	f("a %{foo}-x", "a bar-x")
 	f("%{foo.bar_1}", "baz")
 	f("qq.%{foo-bar_2}.ww", "qq.test.ww")
-}
 
-func TestReplaceFailure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		if _, err := ReplaceBytes([]byte(s)); err == nil {
-			t.Fatalf("expecting non-nil error for ReplaceBytes(%q)", s)
-		}
-		if _, err := ReplaceString(s); err == nil {
-			t.Fatalf("expecting non-nil error for ReplaceString(%q)", s)
-		}
-	}
-	f("foo %{bar} %{baz}")
-	f("%{Foo_Foo_1}")
-	f("%{Foo-Bar-2}")
-	f("%{Foo.Baz.3}")
+	// Missing placeholders
+	f("foo %{bar} %{baz} %{foo}", "foo %{bar} %{baz} bar")
 }

lib/promrelabel/config.go

@@ -160,10 +160,7 @@ func LoadRelabelConfigs(path string) (*ParsedConfigs, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot read `relabel_configs` from %q: %w", path, err)
 	}
-	data, err = envtemplate.ReplaceBytes(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot expand environment vars at %q: %w", path, err)
-	}
+	data = envtemplate.ReplaceBytes(data)
 	pcs, err := ParseRelabelConfigsData(data)
 	if err != nil {
 		return nil, fmt.Errorf("cannot unmarshal `relabel_configs` from %q: %w", path, err)

lib/promscrape/config.go

@@ -121,19 +121,15 @@ type Config struct {
 }
 
 func (cfg *Config) unmarshal(data []byte, isStrict bool) error {
-	var err error
-	data, err = envtemplate.ReplaceBytes(data)
-	if err != nil {
-		return fmt.Errorf("cannot expand environment variables: %w", err)
+	data = envtemplate.ReplaceBytes(data)
+	if !isStrict {
+		return yaml.Unmarshal(data, cfg)
 	}
-	if isStrict {
-		if err = yaml.UnmarshalStrict(data, cfg); err != nil {
-			err = fmt.Errorf("%w; pass -promscrape.config.strictParse=false command-line flag for ignoring unknown fields in yaml config", err)
-		}
-	} else {
-		err = yaml.Unmarshal(data, cfg)
+
+	if err := yaml.UnmarshalStrict(data, cfg); err != nil {
+		return fmt.Errorf("%w; pass -promscrape.config.strictParse=false command-line flag for ignoring unknown fields in yaml config", err)
 	}
-	return err
+	return nil
 }
 
 func (cfg *Config) marshal() []byte {
@@ -441,10 +437,7 @@ func loadStaticConfigs(path string) ([]StaticConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot read `static_configs` from %q: %w", path, err)
 	}
-	data, err = envtemplate.ReplaceBytes(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot expand environment vars in %q: %w", path, err)
-	}
+	data = envtemplate.ReplaceBytes(data)
 	var stcs []StaticConfig
 	if err := yaml.UnmarshalStrict(data, &stcs); err != nil {
 		return nil, fmt.Errorf("cannot unmarshal `static_configs` from %q: %w", path, err)
@@ -485,11 +478,7 @@ func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string, isStrict
 				logger.Errorf("skipping %q at `scrape_config_files` because of error: %s", path, err)
 				continue
 			}
-			data, err = envtemplate.ReplaceBytes(data)
-			if err != nil {
-				logger.Errorf("skipping %q at `scrape_config_files` because of failure to expand environment vars: %s", path, err)
-				continue
-			}
+			data = envtemplate.ReplaceBytes(data)
 			var scs []*ScrapeConfig
 			if isStrict {
 				if err = yaml.UnmarshalStrict(data, &scs); err != nil {

lib/streamaggr/streamaggr.go

@@ -77,10 +77,7 @@ func LoadFromFile(path string, pushFunc PushFunc, opts *Options, alias string) (
 	if err != nil {
 		return nil, fmt.Errorf("cannot load aggregators: %w", err)
 	}
-	data, err = envtemplate.ReplaceBytes(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot expand environment variables in %q: %w", path, err)
-	}
+	data = envtemplate.ReplaceBytes(data)
 
 	as, err := loadFromData(data, path, pushFunc, opts, alias)
 	if err != nil {