Merge branch 'master' into improve-template

app/vmalert-tool/unittest/testdata/disable-group-label.yaml

@@ -8,6 +8,8 @@ tests:
     input_series:
       - series: 'up{job="vmagent2", instance="localhost:9090"}'
         values: "0+0x1440"
+      - series: "test_query"
+        values: "1x200"
 
     metricsql_expr_test:
       - expr: suquery_interval_test
@@ -32,7 +34,8 @@ tests:
       - eval_time: 0
         alertname: AlwaysFiring
         exp_alerts:
-          - {}
+          - exp_annotations:
+              queryValue: "1"
 
       - eval_time: 0
         alertname: InstanceDown

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -12,6 +12,8 @@ groups:
           dashboard: '{{ $externalURL }}/d/dashboard?orgId=1'
       - alert: AlwaysFiring
         expr: 1
+        annotations:
+          queryValue: "{{ query \"test_query\" | first | value }}"
       - alert: SameAlertNameWithDifferentGroup
         expr: absent(test)
         for: 1m

app/vmalert-tool/unittest/testdata/test1.yaml

@@ -10,6 +10,8 @@ tests:
     input_series:
       - series: "test"
         values: "_x5 1x5 _ stale"
+      - series: "test_query"
+        values: "1x100"
 
     alert_rule_test:
       - eval_time: 1m
@@ -50,6 +52,8 @@ tests:
         values: "0+0x1440"
       - series: "test"
         values: "0+1x1440"
+      - series: "test_query"
+        values: "0+1x100"
 
     metricsql_expr_test:
       - expr: count(ALERTS) by (alertgroup, alertname, alertstate)
@@ -97,6 +101,17 @@ tests:
         exp_alerts:
           - exp_labels:
               cluster: prod
+            exp_annotations:
+              queryValue: "0"
+
+      - eval_time: 5m
+        groupname: group1
+        alertname: AlwaysFiring
+        exp_alerts:
+          - exp_labels:
+              cluster: prod
+            exp_annotations:
+              queryValue: "5"
 
       - eval_time: 0
         groupname: alerts

app/vmalert-tool/unittest/testdata/test2.yaml

@@ -8,6 +8,8 @@ tests:
     input_series:
       - series: 'up{job="vmagent2", instance="localhost:9090"}'
         values: "0+0x1440"
+      - series: "test_query"
+        values: "0+1x200"
 
     metricsql_expr_test:
       - expr: suquery_interval_test
@@ -37,6 +39,8 @@ tests:
         exp_alerts:
           - exp_labels:
               cluster: prod
+            exp_annotations:
+              queryValue: "0"
 
       - eval_time: 0
         groupname: group1

app/vmalert-tool/unittest/unittest.go

@@ -64,7 +64,18 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	if err != nil {
 		logger.Fatalf("failed to parse external URL: %w", err)
 	}
-	if err := templates.Load([]string{}, *eu); err != nil {
+	labels := make(map[string]string)
+	for _, s := range externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
+	}
+	if err := templates.Init([]string{}, labels, *eu); err != nil {
 		logger.Fatalf("failed to load template: %v", err)
 	}
 	storagePath = filepath.Join(os.TempDir(), testStoragePath)
@@ -84,19 +95,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	if len(testfiles) == 0 {
 		logger.Fatalf("no test file found")
 	}
-
-	labels := make(map[string]string)
-	for _, s := range externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
-		}
-		labels[s[:n]] = s[n+1:]
-	}
-	_, err = notifier.Init(nil, labels, externalURL)
+	_, err = notifier.Init(nil)
 	if err != nil {
 		logger.Fatalf("failed to init notifier: %v", err)
 	}

app/vmalert/config/config_test.go

@@ -9,14 +9,13 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"gopkg.in/yaml.v2"
 )
 
 func TestMain(m *testing.M) {
-	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, url.URL{}); err != nil {
+	if err := templates.Init([]string{"testdata/templates/*good.tmpl"}, nil, url.URL{}); err != nil {
 		os.Exit(1)
 	}
 	os.Exit(m.Run())
@@ -79,7 +78,7 @@ groups:
 		for i, u := range urls {
 			urls[i] = srv.URL + u
 		}
-		_, err := Parse(urls, notifier.ValidateTemplates, true)
+		_, err := Parse(urls, templates.ValidateTemplates, true)
 		if err != nil && !expErr {
 			t.Fatalf("error parsing URLs %s", err)
 		}
@@ -95,7 +94,7 @@ groups:
 }
 
 func TestParse_Success(t *testing.T) {
-	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true)
+	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, templates.ValidateTemplates, true)
 	if err != nil {
 		t.Fatalf("error parsing files %s", err)
 	}
@@ -105,7 +104,7 @@ func TestParse_Failure(t *testing.T) {
 	f := func(paths []string, errStrExpected string) {
 		t.Helper()
 
-		_, err := Parse(paths, notifier.ValidateTemplates, true)
+		_, err := Parse(paths, templates.ValidateTemplates, true)
 		if err == nil {
 			t.Fatalf("expected to get error")
 		}
@@ -116,7 +115,7 @@ func TestParse_Failure(t *testing.T) {
 
 	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
 	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
-	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
+	f([]string{"testdata/dir/rules0-bad.rules"}, "failed to parse text")
 	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
 	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
 	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
@@ -330,7 +329,7 @@ func TestGroupValidate_Success(t *testing.T) {
 
 		var validateTplFn ValidateTplFn
 		if validateAnnotations {
-			validateTplFn = notifier.ValidateTemplates
+			validateTplFn = templates.ValidateTemplates
 		}
 		err := group.Validate(validateTplFn, validateExpressions)
 		if err != nil {

app/vmalert/main.go

@@ -83,7 +83,6 @@ absolute path to all .tpl files in root.
 
 var (
 	alertURLGeneratorFn notifier.AlertURLGenerator
-	extURL              *url.URL
 )
 
 func main() {
@@ -99,18 +98,29 @@ func main() {
 	logger.Init()
 
 	var err error
-	extURL, err = getExternalURL(*externalURL)
+	extURL, err := getExternalURL(*externalURL)
 	if err != nil {
 		logger.Fatalf("failed to init external.url %q: %s", *externalURL, err)
 	}
+	externalls := make(map[string]string)
+	for _, s := range *externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			logger.Fatalf("wrong format in `-external.label`, it must contain label as `Name=value`; got %q", s)
+		}
+		externalls[s[:n]] = s[n+1:]
+	}
 
-	err = templates.Load(*ruleTemplatesPath, *extURL)
+	err = templates.Init(*ruleTemplatesPath, externalls, *extURL)
 	if err != nil {
 		logger.Fatalf("failed to load template %q: %s", *ruleTemplatesPath, err)
 	}
 
 	if *dryRun {
-		groups, err := config.Parse(*rulePath, notifier.ValidateTemplates, true)
+		groups, err := config.Parse(*rulePath, templates.ValidateTemplates, true)
 		if err != nil {
 			logger.Fatalf("failed to parse %q: %s", *rulePath, err)
 		}
@@ -127,7 +137,7 @@ func main() {
 
 	var validateTplFn config.ValidateTplFn
 	if *validateTemplates {
-		validateTplFn = notifier.ValidateTemplates
+		validateTplFn = templates.ValidateTemplates
 	}
 
 	if *replayFrom != "" {
@@ -156,7 +166,7 @@ func main() {
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
-	manager, err := newManager(ctx)
+	manager, err := newManager(ctx, externalls)
 	if err != nil {
 		logger.Fatalf("failed to init: %s", err)
 	}
@@ -203,25 +213,13 @@ var (
 	configTimestamp    = metrics.NewCounter(`vmalert_config_last_reload_success_timestamp_seconds`)
 )
 
-func newManager(ctx context.Context) (*manager, error) {
+func newManager(ctx context.Context, externalls map[string]string) (*manager, error) {
 	q, err := datasource.Init(nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
 
-	labels := make(map[string]string)
-	for _, s := range *externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s)
-		}
-		labels[s[:n]] = s[n+1:]
-	}
-
-	nts, err := notifier.Init(alertURLGeneratorFn, labels, *externalURL)
+	nts, err := notifier.Init(alertURLGeneratorFn)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
@@ -229,7 +227,7 @@ func newManager(ctx context.Context) (*manager, error) {
 		groups:         make(map[uint64]*rule.Group),
 		querierBuilder: q,
 		notifiers:      nts,
-		labels:         labels,
+		labels:         externalls,
 	}
 	rw, err := remotewrite.Init(ctx)
 	if err != nil {
@@ -293,24 +291,36 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		}, nil
 	}
 	if validateTemplate {
-		if err := notifier.ValidateTemplates(map[string]string{
+		if err := templates.ValidateTemplates(map[string]string{
 			"tpl": externalAlertSource,
 		}); err != nil {
-			return nil, fmt.Errorf("error validating source template %s: %w", externalAlertSource, err)
+			return nil, fmt.Errorf("cannot parse `external.alert.source` %q: %w", externalAlertSource, err)
 		}
 	}
-	m := map[string]string{
-		"tpl": externalAlertSource,
+	var err error
+	tmpl := templates.GetCurrentTmpl()
+	tmpl, err = templates.ParseWithFixedHeader(externalAlertSource, tmpl)
+	if err != nil {
+		return nil, err
 	}
 	return func(alert notifier.Alert) string {
-		qFn := func(_ string) ([]datasource.Metric, error) {
-			return nil, fmt.Errorf("`query` template isn't supported for alert source template")
+		// recreate template if it was changed during config reload
+		cm := templates.GetCurrentTmpl()
+		if tmpl.Name() != cm.Name() {
+			tmpl = cm
+			tmpl, err = templates.ParseWithFixedHeader(externalAlertSource, tmpl)
+			if err != nil {
+				logger.Errorf("cannot parse `external.alert.source` %q: %w", externalAlertSource, err)
+				return fmt.Sprintf("%s/%s", externalURL, externalAlertSource)
+			}
 		}
-		templated, err := alert.ExecTemplate(qFn, alert.Labels, m)
+		tplData := alert.ToTplData()
+		rr, err := templates.ExecuteWithTemplate(tplData, tmpl)
 		if err != nil {
-			logger.Errorf("cannot template alert source: %s", err)
+			logger.Errorf("can not template alert source: %v", err)
+			return fmt.Sprintf("%s/%s", externalURL, externalAlertSource)
 		}
-		return fmt.Sprintf("%s/%s", externalURL, templated["tpl"])
+		return fmt.Sprintf("%s/%s", externalURL, rr)
 	}, nil
 }
 
@@ -334,7 +344,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 
 	var validateTplFn config.ValidateTplFn
 	if *validateTemplates {
-		validateTplFn = notifier.ValidateTemplates
+		validateTplFn = templates.ValidateTemplates
 	}
 
 	// init metrics for config state with positive values to improve alerting conditions
@@ -363,7 +373,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("failed to reload notifier config: %s", err)
 			continue
 		}
-		err := templates.Load(*ruleTemplatesPath, *extURL)
+		err := templates.LoadTemplateFile(*ruleTemplatesPath)
 		if err != nil {
 			setConfigError(err)
 			logger.Errorf("failed to load new templates: %s", err)
@@ -376,7 +386,6 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			continue
 		}
 		if configsEqual(newGroupsCfg, groupsCfg) {
-			templates.Reload()
 			// set success to 1 since previous reload could have been unsuccessful
 			// do not update configTimestamp as config version remains old.
 			configSuccess.Set(1)
@@ -390,7 +399,6 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("error while reloading rules: %s", err)
 			continue
 		}
-		templates.Reload()
 		groupsCfg = newGroupsCfg
 		setConfigSuccessAt(fasttime.UnixTimestamp())
 		logger.Infof("Rules reloaded successfully from %q", *rulePath)

app/vmalert/main_test.go

@@ -74,10 +74,7 @@ func TestGetAlertURLGenerator(t *testing.T) {
 
 func TestConfigReload(t *testing.T) {
 	originalRulePath := *rulePath
-	originalExternalURL := extURL
-	extURL = &url.URL{}
 	defer func() {
-		extURL = originalExternalURL
 		*rulePath = originalRulePath
 	}()
 

app/vmalert/manager_test.go

@@ -19,7 +19,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, url.URL{}); err != nil {
+	if err := templates.Init([]string{"testdata/templates/*good.tmpl"}, nil, url.URL{}); err != nil {
 		os.Exit(1)
 	}
 	os.Exit(m.Run())
@@ -72,7 +72,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
-				cfg, err := config.Parse([]string{paths[rnd]}, notifier.ValidateTemplates, true)
+				cfg, err := config.Parse([]string{paths[rnd]}, templates.ValidateTemplates, true)
 				if err != nil { // update can fail and this is expected
 					continue
 				}
@@ -135,7 +135,7 @@ func TestManagerUpdate_Success(t *testing.T) {
 			t.Fatalf("failed to complete initial rules update: %s", err)
 		}
 
-		cfgUpdate, err := config.Parse([]string{updatePath}, notifier.ValidateTemplates, true)
+		cfgUpdate, err := config.Parse([]string{updatePath}, templates.ValidateTemplates, true)
 		if err == nil { // update can fail and that's expected
 			_ = m.update(ctx, cfgUpdate, false)
 		}
@@ -326,7 +326,7 @@ func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressio
 	t.Helper()
 	var validateTplFn config.ValidateTplFn
 	if validateAnnotations {
-		validateTplFn = notifier.ValidateTemplates
+		validateTplFn = templates.ValidateTemplates
 	}
 	cfg, err := config.Parse(path, validateTplFn, validateExpressions)
 	if err != nil {

app/vmalert/notifier/alert.go

@@ -1,15 +1,9 @@
 package notifier
 
 import (
-	"bytes"
-	"fmt"
-	"io"
-	"strings"
-	textTpl "text/template"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -76,118 +70,18 @@ func (as AlertState) String() string {
 	return "inactive"
 }
 
-// AlertTplData is used to execute templating
-type AlertTplData struct {
-	Labels   map[string]string
-	Value    float64
-	Expr     string
-	AlertID  uint64
-	GroupID  uint64
-	ActiveAt time.Time
-	For      time.Duration
-}
-
-var tplHeaders = []string{
-	"{{ $value := .Value }}",
-	"{{ $labels := .Labels }}",
-	"{{ $expr := .Expr }}",
-	"{{ $externalLabels := .ExternalLabels }}",
-	"{{ $externalURL := .ExternalURL }}",
-	"{{ $alertID := .AlertID }}",
-	"{{ $groupID := .GroupID }}",
-	"{{ $activeAt := .ActiveAt }}",
-	"{{ $for := .For }}",
-}
-
-// ExecTemplate executes the Alert template for given
-// map of annotations.
-// Every alert could have a different datasource, so function
-// requires a queryFunction as an argument.
-func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string]string) (map[string]string, error) {
-	tplData := AlertTplData{
+// ToTplData converts Alert to AlertTplData,
+// which only exposes necessary fields for template.
+func (a Alert) ToTplData() templates.AlertTplData {
+	return templates.AlertTplData{
 		Value:    a.Value,
-		Labels:   labels,
+		Labels:   a.Labels,
 		Expr:     a.Expr,
 		AlertID:  a.ID,
 		GroupID:  a.GroupID,
 		ActiveAt: a.ActiveAt,
 		For:      a.For,
 	}
-	return ExecTemplate(q, annotations, tplData)
-}
-
-// ExecTemplate executes the given template for given annotations map.
-func ExecTemplate(q templates.QueryFn, annotations map[string]string, tplData AlertTplData) (map[string]string, error) {
-	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
-	if err != nil {
-		return nil, fmt.Errorf("error cloning template: %w", err)
-	}
-	return templateAnnotations(annotations, tplData, tmpl, true)
-}
-
-// ValidateTemplates validate annotations for possible template error, uses empty data for template population
-func ValidateTemplates(annotations map[string]string) error {
-	tmpl, err := templates.GetWithFuncs(nil)
-	if err != nil {
-		return err
-	}
-	_, err = templateAnnotations(annotations, AlertTplData{
-		Labels: map[string]string{},
-		Value:  0,
-	}, tmpl, false)
-	return err
-}
-
-func templateAnnotations(annotations map[string]string, data AlertTplData, tmpl *textTpl.Template, execute bool) (map[string]string, error) {
-	var builder strings.Builder
-	var buf bytes.Buffer
-	eg := new(utils.ErrGroup)
-	r := make(map[string]string, len(annotations))
-	tData := tplData{data, externalLabels, externalURL}
-	header := strings.Join(tplHeaders, "")
-	for key, text := range annotations {
-		// simple check to skip text without template
-		if !strings.Contains(text, "{{") || !strings.Contains(text, "}}") {
-			r[key] = text
-			continue
-		}
-
-		buf.Reset()
-		builder.Reset()
-		builder.Grow(len(header) + len(text))
-		builder.WriteString(header)
-		builder.WriteString(text)
-		// clone a new template for each parse to avoid collision
-		ctmpl, _ := tmpl.Clone()
-		ctmpl = ctmpl.Option("missingkey=zero")
-		if err := templateAnnotation(&buf, builder.String(), tData, ctmpl, execute); err != nil {
-			r[key] = text
-			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
-			continue
-		}
-		r[key] = buf.String()
-	}
-	return r, eg.Err()
-}
-
-type tplData struct {
-	AlertTplData
-	ExternalLabels map[string]string
-	ExternalURL    string
-}
-
-func templateAnnotation(dst io.Writer, text string, data tplData, tpl *textTpl.Template, execute bool) error {
-	tpl, err := tpl.Parse(text)
-	if err != nil {
-		return fmt.Errorf("error parsing annotation template: %w", err)
-	}
-	if !execute {
-		return nil
-	}
-	if err = tpl.Execute(dst, data); err != nil {
-		return fmt.Errorf("error evaluating annotation template: %w", err)
-	}
-	return nil
 }
 
 func (a Alert) applyRelabelingIfNeeded(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {

app/vmalert/notifier/alert_test.go

@@ -1,207 +1,13 @@
 package notifier
 
 import (
-	"fmt"
 	"reflect"
 	"testing"
-	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-func TestAlertExecTemplate(t *testing.T) {
-	extLabels := make(map[string]string)
-	const (
-		extCluster = "prod"
-		extDC      = "east"
-		extURL     = "https://foo.bar"
-	)
-	extLabels["cluster"] = extCluster
-	extLabels["dc"] = extDC
-	_, err := Init(nil, extLabels, extURL)
-	checkErr(t, err)
-
-	f := func(alert *Alert, annotations map[string]string, tplExpected map[string]string) {
-		t.Helper()
-
-		if err := ValidateTemplates(annotations); err != nil {
-			t.Fatalf("cannot validate annotations: %s", err)
-		}
-
-		qFn := func(_ string) ([]datasource.Metric, error) {
-			return []datasource.Metric{
-				{
-					Labels: []prompbmarshal.Label{
-						{Name: "foo", Value: "bar"},
-						{Name: "baz", Value: "qux"},
-					},
-					Values:     []float64{1},
-					Timestamps: []int64{1},
-				},
-				{
-					Labels: []prompbmarshal.Label{
-						{Name: "foo", Value: "garply"},
-						{Name: "baz", Value: "fred"},
-					},
-					Values:     []float64{2},
-					Timestamps: []int64{1},
-				},
-			}, nil
-		}
-
-		tpl, err := alert.ExecTemplate(qFn, alert.Labels, annotations)
-		if err != nil {
-			t.Fatalf("cannot execute template: %s", err)
-		}
-		if len(tpl) != len(tplExpected) {
-			t.Fatalf("unexpected number of elements; got %d; want %d", len(tpl), len(tplExpected))
-		}
-		for k := range tplExpected {
-			got, exp := tpl[k], tplExpected[k]
-			if got != exp {
-				t.Fatalf("unexpected template for key=%q; got %q; want %q", k, got, exp)
-			}
-		}
-	}
-
-	// empty-alert
-	f(&Alert{}, map[string]string{}, map[string]string{})
-
-	// no-template
-	f(&Alert{
-		Value: 1e4,
-		Labels: map[string]string{
-			"instance": "localhost",
-		},
-	}, map[string]string{
-		"summary":     "it's a test summary",
-		"description": "it's a test description",
-	}, map[string]string{
-		"summary":     "it's a test summary",
-		"description": "it's a test description",
-	})
-
-	// label-template
-	f(&Alert{
-		Value: 1e4,
-		Labels: map[string]string{
-			"job":      "staging",
-			"instance": "localhost",
-		},
-		For: 5 * time.Minute,
-	}, map[string]string{
-		"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
-		"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
-	}, map[string]string{
-		"summary":     "Too high connection number for localhost for job staging",
-		"description": "It is 10000 connections for localhost for more than 5m0s",
-	})
-
-	// label template override
-	f(&Alert{
-		Value: 1e4,
-	}, map[string]string{
-		"summary":     `{{- define "default.template" -}} {{ printf "summary" }} {{- end -}} {{ template "default.template" . }}`,
-		"description": `{{ template "default.template" . }}`,
-		"value":       `{{$value }}`,
-	}, map[string]string{
-		"summary":     "summary",
-		"description": "",
-		"value":       "10000",
-	})
-
-	// expression-template
-	f(&Alert{
-		Expr: `vm_rows{"label"="bar"}<0`,
-	}, map[string]string{
-		"exprEscapedQuery":  "{{ $expr|queryEscape }}",
-		"exprEscapedPath":   "{{ $expr|pathEscape }}",
-		"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
-		"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
-		"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
-	}, map[string]string{
-		"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
-		"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
-		"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
-		"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
-		"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
-	})
-
-	// query
-	f(&Alert{
-		Expr: `vm_rows{"label"="bar"}>0`,
-	}, map[string]string{
-		"summary": `{{ query "foo" | first | value }}`,
-		"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
-	}, map[string]string{
-		"summary": "1",
-		"desc":    "bar 1;garply 2;",
-	})
-
-	// external
-	f(&Alert{
-		Value: 1e4,
-		Labels: map[string]string{
-			"job":      "staging",
-			"instance": "localhost",
-		},
-	}, map[string]string{
-		"url":         "{{ $externalURL }}",
-		"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
-		"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
-	}, map[string]string{
-		"url":         extURL,
-		"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
-		"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
-	})
-
-	// alert and group IDs
-	f(&Alert{
-		ID:      42,
-		GroupID: 24,
-	}, map[string]string{
-		"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
-	}, map[string]string{
-		"url": "/api/v1/alert?alertID=42&groupID=24",
-	})
-
-	// ActiveAt time
-	f(&Alert{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
-	}, map[string]string{
-		"diagram": "![](http://example.com?render=1660941298",
-	})
-
-	// ActiveAt time is nil
-	f(&Alert{}, map[string]string{
-		"default_time": "{{$activeAt}}",
-	}, map[string]string{
-		"default_time": "0001-01-01 00:00:00 +0000 UTC",
-	})
-
-	// ActiveAt custom format
-	f(&Alert{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
-	}, map[string]string{
-		"fire_time": "2022/08/19 20:34:58",
-	})
-
-	// ActiveAt query range
-	f(&Alert{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
-	}, map[string]string{
-		"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
-	})
-}
-
 func TestAlert_toPromLabels(t *testing.T) {
 	fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
 		t.Helper()

app/vmalert/notifier/init.go

@@ -3,7 +3,6 @@ package notifier
 import (
 	"flag"
 	"fmt"
-	"net/url"
 	"strings"
 	"time"
 
@@ -73,15 +72,6 @@ func Reload() error {
 
 var staticNotifiersFn func() []Notifier
 
-var (
-	// externalLabels is a global variable for holding external labels configured via flags
-	// It is supposed to be inited via Init function only.
-	externalLabels map[string]string
-	// externalURL is a global variable for holding external URL value configured via flag
-	// It is supposed to be inited via Init function only.
-	externalURL string
-)
-
 // Init returns a function for retrieving actual list of Notifier objects.
 // Init works in two mods:
 //   - configuration via flags (for backward compatibility). Is always static
@@ -89,14 +79,7 @@ var (
 //   - configuration via file. Supports live reloads and service discovery.
 //
 // Init returns an error if both mods are used.
-func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
-	externalURL = extURL
-	externalLabels = extLabels
-	_, err := url.Parse(externalURL)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse external URL: %w", err)
-	}
-
+func Init(gen AlertURLGenerator) (func() []Notifier, error) {
 	if *blackHole {
 		if len(*addrs) > 0 || *configPath != "" {
 			return nil, fmt.Errorf("only one of -notifier.blackhole, -notifier.url and -notifier.config flags must be specified")
@@ -126,6 +109,7 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
 		return staticNotifiersFn, nil
 	}
 
+	var err error
 	cw, err = newWatcher(*configPath, gen)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init config watcher: %w", err)

app/vmalert/notifier/init_test.go

@@ -12,7 +12,7 @@ func TestInit(t *testing.T) {
 
 	*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
 
-	fn, err := Init(nil, nil, "")
+	fn, err := Init(nil)
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
@@ -52,7 +52,7 @@ func TestInitNegative(t *testing.T) {
 		*configPath = path
 		*addrs = flagutil.ArrayString{addr}
 		*blackHole = bh
-		if _, err := Init(nil, nil, ""); err == nil {
+		if _, err := Init(nil); err == nil {
 			t.Fatalf("expected to get error; got nil instead")
 		}
 	}
@@ -69,7 +69,7 @@ func TestBlackHole(t *testing.T) {
 
 	*blackHole = true
 
-	fn, err := Init(nil, nil, "")
+	fn, err := Init(nil)
 	if err != nil {
 		t.Fatalf("%s", err)
 	}

app/vmalert/notifier/package_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, url.URL{}); err != nil {
+	if err := templates.Init([]string{"testdata/templates/*good.tmpl"}, nil, url.URL{}); err != nil {
 		os.Exit(1)
 	}
 	os.Exit(m.Run())

app/vmalert/rule/alerting.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	textTpl "text/template"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -47,6 +48,12 @@ type AlertingRule struct {
 	state *ruleState
 
 	metrics *alertingRuleMetrics
+
+	// set rootTemplateName with loaded global template name,
+	// so we can check if the template has changed when evaluating.
+	rootTemplateName     string
+	LabelTemplates       map[string]*textTpl.Template
+	AnnotationsTemplates map[string]*textTpl.Template
 }
 
 type alertingRuleMetrics struct {
@@ -142,9 +149,47 @@ func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return seriesFetched
 		})
+
+	ar.initTemplate()
+
 	return ar
 }
 
+// initTemplate pre-creates templates that can be reused in execution.
+func (ar *AlertingRule) initTemplate() {
+	currentTmpl := templates.GetCurrentTmpl()
+	ar.rootTemplateName = currentTmpl.Name()
+	ar.LabelTemplates = make(map[string]*textTpl.Template, len(ar.Labels))
+	ar.AnnotationsTemplates = make(map[string]*textTpl.Template, len(ar.Annotations))
+	for k, v := range ar.Labels {
+		var err error
+		tmpl, _ := currentTmpl.Clone()
+		tmpl, err = templates.ParseWithFixedHeader(v, tmpl)
+		if err != nil {
+			// parse can fail in two cases:
+			// 1. the text contains `query` function, which is not supported during rule initialization.
+			// 2. the text itself is invalid.
+			// In both case, we skip the error here, and try it again during rule execution.
+			continue
+		}
+		ar.LabelTemplates[k] = tmpl
+
+	}
+	for k, v := range ar.Annotations {
+		var err error
+		tmpl, _ := currentTmpl.Clone()
+		tmpl, err = templates.ParseWithFixedHeader(v, tmpl)
+		if err != nil {
+			// parse can fail in two cases:
+			// 1. the text contains `query` function, which is not supported during rule initialization.
+			// 2. the text itself is invalid.
+			// In both case, we skip the error here, and try it again during rule execution.
+			continue
+		}
+		ar.AnnotationsTemplates[k] = tmpl
+	}
+}
+
 // close unregisters rule metrics
 func (ar *AlertingRule) close() {
 	ar.metrics.active.Unregister()
@@ -225,6 +270,8 @@ func (ar *AlertingRule) updateWith(r Rule) error {
 	ar.KeepFiringFor = nr.KeepFiringFor
 	ar.Labels = nr.Labels
 	ar.Annotations = nr.Annotations
+	ar.initTemplate()
+
 	ar.EvalInterval = nr.EvalInterval
 	ar.Debug = nr.Debug
 	ar.q = nr.q
@@ -279,15 +326,32 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		}
 		ls.processed[l.Name] = l.Value
 	}
-
-	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
+	extraLabels := make(map[string]string, len(ar.Labels))
+	// compare to annotation, label value can only use limited variables for now
+	labelTplData := templates.AlertTplData{
 		Labels: ls.origin,
 		Value:  m.Values[0],
 		Expr:   ar.Expr,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to expand labels: %w", err)
 	}
+	for k := range ar.Labels {
+		if ar.LabelTemplates[k] == nil {
+			// this label may contain `query` function, which requires creating new template with query function in each evaluation.
+			v, err := templates.ExecuteWithoutTemplate(qFn, ar.Labels[k], labelTplData)
+			if err != nil {
+				logger.Errorf("error templating label %q for rule %q: %w", ar.Labels[k], ar.Name, err)
+				v = ar.Labels[k]
+			}
+			extraLabels[k] = v
+			continue
+		}
+		v, err := templates.ExecuteWithTemplate(labelTplData, ar.LabelTemplates[k])
+		if err != nil {
+			logger.Errorf("error templating label %q for rule %q: %w", ar.Labels[k], ar.Name, err)
+			v = ar.Labels[k]
+		}
+		extraLabels[k] = v
+	}
+
 	for k, v := range extraLabels {
 		ls.add(k, v)
 	}
@@ -513,12 +577,18 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 }
 
 func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.QueryFn, ts time.Time) (*labelSet, map[string]string, error) {
+	// check if the rule template has changed during reload,
+	// if so, label&annotation templates must be re-created.
+	if ar.rootTemplateName != templates.GetCurrentTmpl().Name() {
+		ar.initTemplate()
+	}
+
 	ls, err := ar.toLabels(m, qFn)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to expand labels: %w", err)
 	}
-
-	tplData := notifier.AlertTplData{
+	extraAnnotation := make(map[string]string, len(ar.Annotations))
+	annotationTplData := templates.AlertTplData{
 		Value:    m.Values[0],
 		Labels:   ls.origin,
 		Expr:     ar.Expr,
@@ -527,11 +597,26 @@ func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.Query
 		ActiveAt: ts,
 		For:      ar.For,
 	}
-	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to template annotations: %w", err)
+	for k := range ar.Annotations {
+		if ar.AnnotationsTemplates[k] == nil {
+			// this label may contain `query` function, which requires creating new template with query function in each evaluation.
+			v, err := templates.ExecuteWithoutTemplate(qFn, ar.Annotations[k], annotationTplData)
+			if err != nil {
+				logger.Errorf("error templating annotation %q for rule %q: %w", ar.Annotations[k], ar.Name, err)
+				v = ar.Annotations[k]
+			}
+			extraAnnotation[k] = v
+			continue
+		}
+		v, err := templates.ExecuteWithTemplate(annotationTplData, ar.AnnotationsTemplates[k])
+		if err != nil {
+			logger.Errorf("error templating annotation %q for rule %q: %w", ar.Annotations[k], ar.Name, err)
+			v = ar.Annotations[k]
+		}
+		extraAnnotation[k] = v
 	}
-	return ls, as, nil
+
+	return ls, extraAnnotation, nil
 }
 
 // toTimeSeries creates `ALERTS` and `ALERTS_FOR_STATE` for active alerts

app/vmalert/rule/alerting_test.go

@@ -1046,6 +1046,7 @@ func TestAlertingRule_Template(t *testing.T) {
 		}
 		fq.Add(metrics...)
 
+		rule.initTemplate()
 		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -1262,6 +1263,7 @@ func newTestAlertingRuleWithCustomFields(name string, waitFor, evalInterval, kee
 	}
 	rule.KeepFiringFor = keepFiringFor
 	rule.Annotations = annotation
+	rule.initTemplate()
 	return rule
 }
 

app/vmalert/rule/group_test.go

@@ -27,7 +27,7 @@ func init() {
 }
 
 func TestMain(m *testing.M) {
-	if err := templates.Load([]string{}, url.URL{}); err != nil {
+	if err := templates.Init([]string{}, nil, url.URL{}); err != nil {
 		fmt.Println("failed to load template for test")
 		os.Exit(1)
 	}

app/vmalert/templates/init.go

@@ -0,0 +1,553 @@
+package templates
+
+import (
+	"errors"
+	"fmt"
+	htmlTpl "html/template"
+	"io"
+	"math"
+	"net"
+	"net/url"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	textTpl "text/template"
+	"time"
+
+	"github.com/bmatcuk/doublestar/v4"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+var (
+	tplMu      sync.RWMutex
+	masterTmpl *textTpl.Template
+
+	// externalLabels is a global variable for holding external labels configured via flags
+	// It is supposed to be initiated via Init function only.
+	externalLabels map[string]string
+	// externalURL is a global variable for holding external URL value configured via flag
+	// It is supposed to be initiated via Init function only.
+	externalURL url.URL
+)
+
+// Init initializes global externalLabels and externalURL variables, and load templates from pathPatterns.
+func Init(pathPatterns []string, extLabels map[string]string, extURL url.URL) error {
+	externalURL = extURL
+	externalLabels = extLabels
+	return LoadTemplateFile(pathPatterns)
+}
+
+// LoadTemplateFile loads templates from multiple globs specified in pathPatterns:
+// 1. if it's the first load, sets them directly to current template;
+// 2. if it's not the first load, only update masterTmpl when the contents change.
+func LoadTemplateFile(pathPatterns []string) error {
+	templateName := fmt.Sprintf("rule-template-%d", time.Now().UnixMilli())
+	// using Load timestamp as template root name,
+	// so we can check if this global template has been reloaded when use it elsewhere, like during alerting rules execution.
+	tmpl := newTemplate(templateName)
+	tmpl = tmpl.Funcs(funcsWithExternalURL(externalURL))
+
+	for _, tp := range pathPatterns {
+		p, err := doublestar.FilepathGlob(tp)
+		if err != nil {
+			return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err)
+		}
+		if len(p) > 0 {
+			tmpl, err = tmpl.ParseFiles(p...)
+			if err != nil {
+				return fmt.Errorf("failed to parse template glob %q: %w", tp, err)
+			}
+		}
+	}
+	err := tmpl.Execute(io.Discard, nil)
+	if err != nil {
+		return fmt.Errorf("failed to test rule template: %w", err)
+	}
+
+	tplMu.Lock()
+	defer tplMu.Unlock()
+
+	if masterTmpl == nil {
+		masterTmpl = tmpl
+	} else {
+		// only update the masterTmpl when content has changed
+		if !isTemplatesTheSame(masterTmpl, tmpl) {
+			masterTmpl = tmpl
+		}
+	}
+	return nil
+}
+
+func newTemplate(name string) *textTpl.Template {
+	tmpl := textTpl.New(name).Funcs(templateFuncs())
+	return textTpl.Must(tmpl.Parse(""))
+}
+
+// isTemplatesTheSame returns true if the content of two templates are the same,
+// the root template name difference is ignored.
+func isTemplatesTheSame(t1, t2 *textTpl.Template) bool {
+	getRootString := func(t *textTpl.Template) string {
+		if t.Tree == nil || t.Tree.Root == nil {
+			return ""
+		}
+		return t.Tree.Root.String()
+	}
+	if getRootString(t1) != getRootString(t2) {
+		return false
+	}
+	t1Templates := make(map[string]string)
+	t2Templates := make(map[string]string)
+	for _, tmpl := range t1.Templates() {
+		// skip root template, since it's null and changes every time
+		if tmpl.Name() == t1.Name() {
+			continue
+		}
+		t1Templates[tmpl.Name()] = getRootString(tmpl)
+	}
+	for _, tmpl := range t2.Templates() {
+		if tmpl.Name() == t2.Name() {
+			continue
+		}
+		t2Templates[tmpl.Name()] = getRootString(tmpl)
+	}
+	if len(t1Templates) != len(t2Templates) {
+		return false
+	}
+	for k, v := range t1Templates {
+		if t2Templates[k] != v {
+			return false
+		}
+		delete(t2Templates, k)
+	}
+	return len(t2Templates) == 0
+}
+
+// funcsWithExternalURL returns a function map that depends on externalURL value
+func funcsWithExternalURL(externalURL url.URL) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"externalURL": func() string {
+			return externalURL.String()
+		},
+
+		"pathPrefix": func() string {
+			return externalURL.Path
+		},
+	}
+}
+
+// templateFuncs initiates template helper functions
+func templateFuncs() textTpl.FuncMap {
+	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
+	// and https://github.com/prometheus/prometheus/blob/fa6e05903fd3ce52e374a6e1bf4eb98c9f1f45a7/template/template.go#L150
+	return textTpl.FuncMap{
+		/* Strings */
+
+		// title returns a copy of the string s with all Unicode letters
+		// that begin words mapped to their Unicode title case.
+		// alias for https://golang.org/pkg/strings/#Title
+		"title": strings.Title,
+
+		// toUpper returns s with all Unicode letters mapped to their upper case.
+		// alias for https://golang.org/pkg/strings/#ToUpper
+		"toUpper": strings.ToUpper,
+
+		// toLower returns s with all Unicode letters mapped to their lower case.
+		// alias for https://golang.org/pkg/strings/#ToLower
+		"toLower": strings.ToLower,
+
+		// crlfEscape replaces '\n' and '\r' chars with `\\n` and `\\r`.
+		// This function is deprecated.
+		//
+		// It is better to use quotesEscape, jsonEscape, queryEscape or pathEscape instead -
+		// these functions properly escape `\n` and `\r` chars according to their purpose.
+		"crlfEscape": func(q string) string {
+			q = strings.Replace(q, "\n", `\n`, -1)
+			return strings.Replace(q, "\r", `\r`, -1)
+		},
+
+		// quotesEscape escapes the string, so it can be safely put inside JSON string.
+		//
+		// See also jsonEscape.
+		"quotesEscape": quotesEscape,
+
+		// jsonEscape converts the string to properly encoded JSON string.
+		//
+		// See also quotesEscape.
+		"jsonEscape": jsonEscape,
+
+		// htmlEscape applies html-escaping to q, so it can be safely embedded as plaintext into html.
+		//
+		// See also safeHtml.
+		"htmlEscape": htmlEscape,
+
+		// stripPort splits string into host and port, then returns only host.
+		"stripPort": func(hostPort string) string {
+			host, _, err := net.SplitHostPort(hostPort)
+			if err != nil {
+				return hostPort
+			}
+			return host
+		},
+
+		// stripDomain removes the domain part of a FQDN. Leaves port untouched.
+		"stripDomain": func(hostPort string) string {
+			host, port, err := net.SplitHostPort(hostPort)
+			if err != nil {
+				host = hostPort
+			}
+			ip := net.ParseIP(host)
+			if ip != nil {
+				return hostPort
+			}
+			host = strings.Split(host, ".")[0]
+			if port != "" {
+				return net.JoinHostPort(host, port)
+			}
+			return host
+		},
+
+		// match reports whether the string s
+		// contains any match of the regular expression pattern.
+		// alias for https://golang.org/pkg/regexp/#MatchString
+		"match": regexp.MatchString,
+
+		// reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with
+		// the replacement string repl. Inside repl, $ signs are interpreted as in Expand,
+		// so for instance $1 represents the text of the first submatch.
+		// alias for https://golang.org/pkg/regexp/#Regexp.ReplaceAllString
+		"reReplaceAll": func(pattern, repl, text string) string {
+			re := regexp.MustCompile(pattern)
+			return re.ReplaceAllString(text, repl)
+		},
+
+		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
+		"parseDuration": func(s string) (float64, error) {
+			d, err := promutils.ParseDuration(s)
+			if err != nil {
+				return 0, err
+			}
+			return d.Seconds(), nil
+		},
+
+		// same with parseDuration but returns a time.Duration
+		"parseDurationTime": func(s string) (time.Duration, error) {
+			d, err := promutils.ParseDuration(s)
+			if err != nil {
+				return 0, err
+			}
+			return d, nil
+		},
+
+		/* Numbers */
+
+		// humanize converts given number to a human readable format
+		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
+		"humanize": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			if math.Abs(v) >= 1 {
+				prefix := ""
+				for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
+					if math.Abs(v) < 1000 {
+						break
+					}
+					prefix = p
+					v /= 1000
+				}
+				return fmt.Sprintf("%.4g%s", v, prefix), nil
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%s", v, prefix), nil
+		},
+
+		// humanize1024 converts given number to a human readable format with 1024 as base
+		"humanize1024": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			return formatutil.HumanizeBytes(v), nil
+		},
+
+		// humanizeDuration converts given seconds to a human-readable duration
+		"humanizeDuration": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			if v == 0 {
+				return fmt.Sprintf("%.4gs", v), nil
+			}
+			if math.Abs(v) >= 1 {
+				sign := ""
+				if v < 0 {
+					sign = "-"
+					v = -v
+				}
+				seconds := int64(v) % 60
+				minutes := (int64(v) / 60) % 60
+				hours := (int64(v) / 60 / 60) % 24
+				days := int64(v) / 60 / 60 / 24
+				// For days to minutes, we display seconds as an integer.
+				if days != 0 {
+					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil
+				}
+				if hours != 0 {
+					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil
+				}
+				if minutes != 0 {
+					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil
+				}
+				// For seconds, we display 4 significant digits.
+				return fmt.Sprintf("%s%.4gs", sign, v), nil
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%ss", v, prefix), nil
+		},
+
+		// humanizePercentage converts given ratio value to a fraction of 100
+		"humanizePercentage": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			return fmt.Sprintf("%.4g%%", v*100), nil
+		},
+
+		// humanizeTimestamp converts given timestamp to a human readable time equivalent
+		"humanizeTimestamp": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			t := timeFromUnixTimestamp(v).Time().UTC()
+			return fmt.Sprint(t), nil
+		},
+
+		// toTime converts given timestamp to a time.Time.
+		"toTime": func(i any) (time.Time, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return time.Time{}, err
+			}
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return time.Time{}, fmt.Errorf("cannot convert %v to time.Time", v)
+			}
+			t := timeFromUnixTimestamp(v).Time().UTC()
+			return t, nil
+		},
+
+		/* URLs */
+
+		// externalURL returns value of `external.url` flag
+		"externalURL": func() string {
+			// externalURL function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
+		},
+
+		// pathPrefix returns a Path segment from the URL value in `external.url` flag
+		"pathPrefix": func() string {
+			// pathPrefix function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
+		},
+
+		// pathEscape escapes the string so it can be safely placed inside a URL path segment.
+		//
+		// See also queryEscape.
+		"pathEscape": url.PathEscape,
+
+		// queryEscape escapes the string so it can be safely placed inside a query arg in URL.
+		//
+		// See also queryEscape.
+		"queryEscape": url.QueryEscape,
+
+		// first returns the first by order element from the given metrics list.
+		// usually used alongside with `query` template function.
+		"first": func(metrics []metric) (metric, error) {
+			if len(metrics) > 0 {
+				return metrics[0], nil
+			}
+			return metric{}, errors.New("first() called on vector with no elements")
+		},
+
+		// label returns the value of the given label name for the given metric.
+		// usually used alongside with `query` template function.
+		"label": func(label string, m metric) string {
+			return m.Labels[label]
+		},
+
+		// value returns the value of the given metric.
+		// usually used alongside with `query` template function.
+		"value": func(m metric) float64 {
+			return m.Value
+		},
+
+		// strvalue returns metric name.
+		"strvalue": func(m metric) string {
+			return m.Labels["__name__"]
+		},
+
+		// sortByLabel sorts the given metrics by provided label key
+		"sortByLabel": func(label string, metrics []metric) []metric {
+			sort.SliceStable(metrics, func(i, j int) bool {
+				return metrics[i].Labels[label] < metrics[j].Labels[label]
+			})
+			return metrics
+		},
+
+		/* Helpers */
+
+		// Converts a list of objects to a map with keys arg0, arg1 etc.
+		// This is intended to allow multiple arguments to be passed to templates.
+		"args": func(args ...any) map[string]any {
+			result := make(map[string]any)
+			for i, a := range args {
+				result[fmt.Sprintf("arg%d", i)] = a
+			}
+			return result
+		},
+
+		// safeHtml marks string as HTML not requiring auto-escaping.
+		//
+		// See also htmlEscape.
+		"safeHtml": func(text string) htmlTpl.HTML {
+			return htmlTpl.HTML(text)
+		},
+	}
+}
+
+// metric is private copy of datasource.Metric,
+// it is used for templating annotations,
+// Labels as map simplifies templates evaluation.
+type metric struct {
+	Labels    map[string]string
+	Timestamp int64
+	Value     float64
+}
+
+// datasourceMetricsToTemplateMetrics converts Metrics from datasource package to private copy for templating.
+func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric {
+	mss := make([]metric, 0, len(ms))
+	for _, m := range ms {
+		labelsMap := make(map[string]string, len(m.Labels))
+		for _, labelValue := range m.Labels {
+			labelsMap[labelValue.Name] = labelValue.Value
+		}
+		mss = append(mss, metric{
+			Labels:    labelsMap,
+			Timestamp: m.Timestamps[0],
+			Value:     m.Values[0]})
+	}
+	return mss
+}
+
+// QueryFn is used to wrap a call to datasource into simple-to-use function
+// for templating functions.
+type QueryFn func(query string) ([]datasource.Metric, error)
+
+// FuncsWithQuery returns a function map that depends on metric data
+func FuncsWithQuery(query QueryFn) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"query": func(q string) ([]metric, error) {
+			if query == nil {
+				return nil, fmt.Errorf("cannot execute query %q: query is not available in this context", q)
+			}
+
+			result, err := query(q)
+			if err != nil {
+				return nil, err
+			}
+			return datasourceMetricsToTemplateMetrics(result), nil
+		},
+	}
+}
+
+// Time is the number of milliseconds since the epoch
+// (1970-01-01 00:00 UTC) excluding leap seconds.
+type Time int64
+
+// timeFromUnixTimestamp returns the Time equivalent to t in unix timestamp.
+func timeFromUnixTimestamp(t float64) Time {
+	return Time(t * 1e3)
+}
+
+// The number of nanoseconds per minimum tick.
+const nanosPerTick = int64(minimumTick / time.Nanosecond)
+
+// MinimumTick is the minimum supported time resolution. This has to be
+// at least time.Second in order for the code below to work.
+const minimumTick = time.Millisecond
+
+// second is the Time duration equivalent to one second.
+const second = int64(time.Second / minimumTick)
+
+// Time returns the time.Time representation of t.
+func (t Time) Time() time.Time {
+	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
+}
+
+func toFloat64(v any) (float64, error) {
+	switch i := v.(type) {
+	case float64:
+		return i, nil
+	case float32:
+		return float64(i), nil
+	case int64:
+		return float64(i), nil
+	case int32:
+		return float64(i), nil
+	case int:
+		return float64(i), nil
+	case uint64:
+		return float64(i), nil
+	case uint32:
+		return float64(i), nil
+	case uint:
+		return float64(i), nil
+	case string:
+		return strconv.ParseFloat(i, 64)
+	default:
+		return 0, fmt.Errorf("unexpected value type %v", i)
+	}
+}

app/vmalert/templates/init_test.go

@@ -0,0 +1,261 @@
+package templates
+
+import (
+	"fmt"
+	"math"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	textTpl "text/template"
+)
+
+func TestMain(m *testing.M) {
+	if err := Init([]string{}, nil, url.URL{}); err != nil {
+		fmt.Println("failed to load template for test")
+		os.Exit(1)
+	}
+	os.Exit(m.Run())
+}
+
+func TestTemplateFuncs_Match(t *testing.T) {
+	funcs := templateFuncs()
+	// check "match" func
+	matchFunc := funcs["match"].(func(pattern, s string) (bool, error))
+	if _, err := matchFunc("invalid[regexp", "abc"); err == nil {
+		t.Fatalf("expecting non-nil error on invalid regexp")
+	}
+	ok, err := matchFunc("abc", "def")
+	if err != nil {
+		t.Fatalf("unexpected error")
+	}
+	if ok {
+		t.Fatalf("unexpected match")
+	}
+	ok, err = matchFunc("a.+b", "acsdb")
+	if err != nil {
+		t.Fatalf("unexpected error")
+	}
+	if !ok {
+		t.Fatalf("unexpected mismatch")
+	}
+}
+
+func TestTemplateFuncs_Formatting(t *testing.T) {
+	f := func(funcName string, p any, resultExpected string) {
+		t.Helper()
+
+		funcs := templateFuncs()
+		v := funcs[funcName]
+		fLocal := v.(func(s any) (string, error))
+		result, err := fLocal(p)
+		if err != nil {
+			t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
+		}
+	}
+
+	f("humanize1024", float64(0), "0")
+	f("humanize1024", math.Inf(0), "+Inf")
+	f("humanize1024", math.NaN(), "NaN")
+	f("humanize1024", float64(127087), "124.1ki")
+	f("humanize1024", float64(130137088), "124.1Mi")
+	f("humanize1024", float64(133260378112), "124.1Gi")
+	f("humanize1024", float64(136458627186688), "124.1Ti")
+	f("humanize1024", float64(139733634239168512), "124.1Pi")
+	f("humanize1024", float64(143087241460908556288), "124.1Ei")
+	f("humanize1024", float64(146521335255970361638912), "124.1Zi")
+	f("humanize1024", float64(150037847302113650318245888), "124.1Yi")
+	f("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
+
+	f("humanize", float64(127087), "127.1k")
+	f("humanize", float64(136458627186688), "136.5T")
+
+	f("humanizeDuration", 1, "1s")
+	f("humanizeDuration", 0.2, "200ms")
+	f("humanizeDuration", 42000, "11h 40m 0s")
+	f("humanizeDuration", 16790555, "194d 8h 2m 35s")
+
+	f("humanizePercentage", 1, "100%")
+	f("humanizePercentage", 0.8, "80%")
+	f("humanizePercentage", 0.015, "1.5%")
+
+	f("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
+}
+
+func TestTemplateFuncs_StringConversion(t *testing.T) {
+	f := func(funcName, s, resultExpected string) {
+		t.Helper()
+
+		funcs := templateFuncs()
+		v := funcs[funcName]
+		fLocal := v.(func(s string) string)
+		result := fLocal(s)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %s(%q); got\n%s\nwant\n%s", funcName, s, result, resultExpected)
+		}
+	}
+
+	f("title", "foo bar", "Foo Bar")
+	f("toUpper", "foo", "FOO")
+	f("toLower", "FOO", "foo")
+	f("pathEscape", "foo/bar\n+baz", "foo%2Fbar%0A+baz")
+	f("queryEscape", "foo+bar\n+baz", "foo%2Bbar%0A%2Bbaz")
+	f("jsonEscape", `foo{bar="baz"}`+"\n + 1", `"foo{bar=\"baz\"}\n + 1"`)
+	f("quotesEscape", `foo{bar="baz"}`+"\n + 1", `foo{bar=\"baz\"}\n + 1`)
+	f("htmlEscape", "foo < 10\nabc", "foo &lt; 10\nabc")
+	f("crlfEscape", "foo\nbar\rx", `foo\nbar\rx`)
+	f("stripPort", "foo", "foo")
+	f("stripPort", "foo:1234", "foo")
+	f("stripDomain", "foo.bar.baz", "foo")
+	f("stripDomain", "foo.bar:123", "foo:123")
+}
+
+func TestTemplatesLoad_Success(t *testing.T) {
+	f := func(pathPatterns []string, expectedTmpl *textTpl.Template) {
+		t.Helper()
+
+		masterTmplOrig := masterTmpl
+		defer func() {
+			masterTmpl = masterTmplOrig
+		}()
+		masterTmpl = nil
+
+		if err := LoadTemplateFile(pathPatterns); err != nil {
+			t.Fatalf("cannot load templates: %s", err)
+		}
+		if !isTemplatesTheSame(masterTmpl, expectedTmpl) {
+			t.Fatalf("unexpected template\ngot\n%+v\nwant\n%+v", masterTmpl, expectedTmpl)
+		}
+	}
+
+	// non existing path
+	pathPatterns := []string{
+		"templates/non-existing/good-*.tpl",
+		"templates/absent/good-*.tpl",
+	}
+	expectedTmpl := textTpl.Must(newTemplate("").Parse(""))
+	f(pathPatterns, expectedTmpl)
+
+	// existing path
+	pathPatterns = []string{
+		"templates/test/good0-*.tpl",
+	}
+	expectedTmpl = textTpl.Must(newTemplate("").Parse(`
+		{{- define "good0-test.tpl" -}}{{- end -}}
+		{{- define "test.0" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.2" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.3" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+	`))
+	f(pathPatterns, expectedTmpl)
+
+	// template update
+	pathPatterns = []string{
+		"templates/other/nested/good0-*.tpl",
+	}
+	expectedTmpl = textTpl.Must(newTemplate("").Parse(`
+		{{- define "good0-test.tpl" -}}{{- end -}}
+		{{- define "test.0" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.1" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.3" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+	`))
+	f(pathPatterns, expectedTmpl)
+}
+
+func TestTemplatesLoad_Failure(t *testing.T) {
+	f := func(pathPatterns []string, expectedErrStr string) {
+		t.Helper()
+
+		err := LoadTemplateFile(pathPatterns)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+
+		errStr := err.Error()
+		if !strings.Contains(errStr, expectedErrStr) {
+			t.Fatalf("the returned error %q doesn't contain %q", errStr, expectedErrStr)
+		}
+	}
+
+	// load template with syntax error
+	f([]string{
+		"templates/other/nested/bad0-*.tpl",
+		"templates/test/good0-*.tpl",
+	}, "failed to parse template glob")
+}
+
+func TestTemplatesReload(t *testing.T) {
+	masterTmplOrig := masterTmpl
+	defer func() {
+		masterTmpl = masterTmplOrig
+	}()
+	masterTmpl = nil
+
+	// load with non existing path
+	pathPatterns := []string{
+		"templates/non-existing/good-*.tpl",
+		"templates/absent/good-*.tpl",
+	}
+	if err := LoadTemplateFile(pathPatterns); err != nil {
+		t.Fatalf("cannot load templates: %s", err)
+	}
+	tpl1 := GetCurrentTmpl()
+
+	// reload with existing path
+	pathPatterns = []string{
+		"templates/test/good0-*.tpl",
+	}
+	if err := LoadTemplateFile(pathPatterns); err != nil {
+		t.Fatalf("cannot load templates: %s", err)
+	}
+	tpl2 := GetCurrentTmpl()
+	if isTemplatesTheSame(tpl1, tpl2) {
+		t.Fatalf("tpl1 should be different from tpl2")
+	}
+
+	// reload the same path
+	pathPatterns = []string{
+		"templates/test/good0-*.tpl",
+	}
+	if err := LoadTemplateFile(pathPatterns); err != nil {
+		t.Fatalf("cannot load templates: %s", err)
+	}
+	tpl3 := GetCurrentTmpl()
+	if !isTemplatesTheSame(tpl2, tpl3) || tpl2.Name() != tpl3.Name() {
+		t.Fatalf("tpl3 should be the same as tpl2")
+	}
+}
+
+func TestIsTemplateTheSame(t *testing.T) {
+	f := func(tmpl1, tmpl2 *textTpl.Template, isTheSame bool) {
+		t.Helper()
+
+		if isTemplatesTheSame(tmpl1, tmpl2) != isTheSame {
+			t.Fatalf("unexpected result for isTemplatesTheSame")
+		}
+	}
+
+	tmpl1 := textTpl.Must(newTemplate("t1").Parse("{{- define \"test\" -}}{{- end -}}"))
+	tmpl2 := textTpl.Must(newTemplate("t2").Parse("{{- define \"test\" -}}{{- end -}}"))
+	f(tmpl1, tmpl2, true)
+
+	tmpl1, _ = tmpl1.Parse("{{- define \"test2\" -}}{{- end -}}")
+	f(tmpl1, tmpl2, false)
+
+	tmpl2, _ = tmpl2.Parse("{{- define \"test3\" -}}{{- end -}}")
+	f(tmpl1, tmpl2, false)
+}

app/vmalert/templates/template.go

@@ -1,554 +1,95 @@
-// Copyright 2013 The Prometheus Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 package templates
 
 import (
-	"errors"
+	"bytes"
 	"fmt"
-	htmlTpl "html/template"
-	"io"
-	"math"
-	"net"
-	"net/url"
-	"regexp"
-	"sort"
-	"strconv"
 	"strings"
-	"sync"
 	textTpl "text/template"
 	"time"
 
-	"github.com/bmatcuk/doublestar/v4"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
-// go template execution fails when it's tree is empty
-const defaultTemplate = `{{- define "default.template" -}}{{- end -}}`
+// supported variables are list in https://docs.victoriametrics.com/vmalert/#templating.
+const tplHeaders = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}{{ $externalLabels := .ExternalLabels }}{{ $externalURL := .ExternalURL }}{{ $alertID := .AlertID }}{{ $groupID := .GroupID }}{{ $activeAt := .ActiveAt }}{{ $for := .For }}`
 
-var tplMu sync.RWMutex
-
-type textTemplate struct {
-	current     *textTpl.Template
-	replacement *textTpl.Template
+// AlertTplData is used to execute templating
+type AlertTplData struct {
+	Labels   map[string]string
+	Value    float64
+	Expr     string
+	AlertID  uint64
+	GroupID  uint64
+	ActiveAt time.Time
+	For      time.Duration
 }
 
-var masterTmpl textTemplate
-
-func newTemplate() *textTpl.Template {
-	tmpl := textTpl.New("").Option("missingkey=zero").Funcs(templateFuncs())
-	return textTpl.Must(tmpl.Parse(defaultTemplate))
-}
-
-// Load func loads templates from multiple globs specified in pathPatterns and either
-// sets them directly to current template if it's the first init;
-// or sets replacement templates and wait for Reload() to replace current template with replacement.
-func Load(pathPatterns []string, externalURL url.URL) error {
-	tmpl := newTemplate()
-	for _, tp := range pathPatterns {
-		p, err := doublestar.FilepathGlob(tp)
+// ValidateTemplates validates the given annotations,
+// mock the `query` function during validation.
+func ValidateTemplates(annotations map[string]string) error {
+	// it's ok to reuse one template for multiple text validations.
+	tmpl := GetCurrentTmpl()
+	tmpl = tmpl.Funcs(FuncsWithQuery(nil))
+	for _, v := range annotations {
+		_, err := tmpl.Parse(tplHeaders + v)
 		if err != nil {
-			return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err)
+			return fmt.Errorf("failed to parse text %q into template: %w", v, err)
 		}
-		if len(p) > 0 {
-			tmpl, err = tmpl.ParseFiles(p...)
-			if err != nil {
-				return fmt.Errorf("failed to parse template glob %q: %w", tp, err)
-			}
-		}
-	}
-	if len(tmpl.Templates()) > 0 {
-		err := tmpl.Execute(io.Discard, nil)
-		if err != nil {
-			return fmt.Errorf("failed to execute template: %w", err)
-		}
-	}
-	tplMu.Lock()
-	defer tplMu.Unlock()
-	tmpl = tmpl.Funcs(funcsWithExternalURL(externalURL))
-
-	if masterTmpl.current == nil {
-		masterTmpl.current = tmpl
-	} else {
-		masterTmpl.replacement = tmpl
 	}
 	return nil
 }
 
-// Reload func replaces current template with a replacement template
-// which was set by Load with override=false
-func Reload() {
-	tplMu.Lock()
-	defer tplMu.Unlock()
-	if masterTmpl.replacement != nil {
-		masterTmpl.current = masterTmpl.replacement
-		masterTmpl.replacement = nil
-	}
-}
-
-// metric is private copy of datasource.Metric,
-// it is used for templating annotations,
-// Labels as map simplifies templates evaluation.
-type metric struct {
-	Labels    map[string]string
-	Timestamp int64
-	Value     float64
-}
-
-// datasourceMetricsToTemplateMetrics converts Metrics from datasource package to private copy for templating.
-func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric {
-	mss := make([]metric, 0, len(ms))
-	for _, m := range ms {
-		labelsMap := make(map[string]string, len(m.Labels))
-		for _, labelValue := range m.Labels {
-			labelsMap[labelValue.Name] = labelValue.Value
-		}
-		mss = append(mss, metric{
-			Labels:    labelsMap,
-			Timestamp: m.Timestamps[0],
-			Value:     m.Values[0]})
-	}
-	return mss
-}
-
-// QueryFn is used to wrap a call to datasource into simple-to-use function
-// for templating functions.
-type QueryFn func(query string) ([]datasource.Metric, error)
-
-// GetWithFuncs returns a copy of current template with additional FuncMap
-// provided with funcs argument
-func GetWithFuncs(funcs textTpl.FuncMap) (*textTpl.Template, error) {
+// GetCurrentTmpl returns a copy of the current global template
+func GetCurrentTmpl() *textTpl.Template {
 	tplMu.RLock()
 	defer tplMu.RUnlock()
-	tmpl, err := masterTmpl.current.Clone()
+	tmpl, err := masterTmpl.Clone()
 	if err != nil {
-		return nil, err
+		logger.Panicf("failed to clone current rule template: %w", err)
 	}
-	// Clone() doesn't copy tpl Options, so we set them manually
-	tmpl = tmpl.Option("missingkey=zero")
-	return tmpl.Funcs(funcs), nil
+	return tmpl
 }
 
-// FuncsWithQuery returns a function map that depends on metric data
-func FuncsWithQuery(query QueryFn) textTpl.FuncMap {
-	return textTpl.FuncMap{
-		"query": func(q string) ([]metric, error) {
-			if query == nil {
-				return nil, fmt.Errorf("cannot execute query %q: query is not available in this context", q)
-			}
+type tplData struct {
+	AlertTplData
+	ExternalLabels map[string]string
+	ExternalURL    string
+}
 
-			result, err := query(q)
-			if err != nil {
-				return nil, err
-			}
-			return datasourceMetricsToTemplateMetrics(result), nil
-		},
+// ParseWithFixedHeader parses the text with the fixed tplHeaders into the given template
+func ParseWithFixedHeader(text string, tpl *textTpl.Template) (*textTpl.Template, error) {
+	return tpl.Parse(tplHeaders + text)
+}
+
+// ExecuteWithoutTemplate retrieves the current global templates, parses the text and executes with the given data
+func ExecuteWithoutTemplate(q QueryFn, text string, data AlertTplData) (string, error) {
+	if !strings.Contains(text, "{{") || !strings.Contains(text, "}}") {
+		return text, nil
 	}
-}
 
-// funcsWithExternalURL returns a function map that depends on externalURL value
-func funcsWithExternalURL(externalURL url.URL) textTpl.FuncMap {
-	return textTpl.FuncMap{
-		"externalURL": func() string {
-			return externalURL.String()
-		},
+	var err error
+	tmpl := GetCurrentTmpl()
+	tmpl = tmpl.Funcs(FuncsWithQuery(q))
+	tmpl, err = tmpl.Parse(tplHeaders + text)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse text %q into template: %w", text, err)
 
-		"pathPrefix": func() string {
-			return externalURL.Path
-		},
 	}
+	return ExecuteWithTemplate(data, tmpl)
 }
 
-// templateFuncs initiates template helper functions
-func templateFuncs() textTpl.FuncMap {
-	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
-	// and https://github.com/prometheus/prometheus/blob/fa6e05903fd3ce52e374a6e1bf4eb98c9f1f45a7/template/template.go#L150
-	return textTpl.FuncMap{
-		/* Strings */
-
-		// title returns a copy of the string s with all Unicode letters
-		// that begin words mapped to their Unicode title case.
-		// alias for https://golang.org/pkg/strings/#Title
-		"title": strings.Title,
-
-		// toUpper returns s with all Unicode letters mapped to their upper case.
-		// alias for https://golang.org/pkg/strings/#ToUpper
-		"toUpper": strings.ToUpper,
-
-		// toLower returns s with all Unicode letters mapped to their lower case.
-		// alias for https://golang.org/pkg/strings/#ToLower
-		"toLower": strings.ToLower,
-
-		// crlfEscape replaces '\n' and '\r' chars with `\\n` and `\\r`.
-		// This function is deprecated.
-		//
-		// It is better to use quotesEscape, jsonEscape, queryEscape or pathEscape instead -
-		// these functions properly escape `\n` and `\r` chars according to their purpose.
-		"crlfEscape": func(q string) string {
-			q = strings.Replace(q, "\n", `\n`, -1)
-			return strings.Replace(q, "\r", `\r`, -1)
-		},
-
-		// quotesEscape escapes the string, so it can be safely put inside JSON string.
-		//
-		// See also jsonEscape.
-		"quotesEscape": quotesEscape,
-
-		// jsonEscape converts the string to properly encoded JSON string.
-		//
-		// See also quotesEscape.
-		"jsonEscape": jsonEscape,
-
-		// htmlEscape applies html-escaping to q, so it can be safely embedded as plaintext into html.
-		//
-		// See also safeHtml.
-		"htmlEscape": htmlEscape,
-
-		// stripPort splits string into host and port, then returns only host.
-		"stripPort": func(hostPort string) string {
-			host, _, err := net.SplitHostPort(hostPort)
-			if err != nil {
-				return hostPort
-			}
-			return host
-		},
-
-		// stripDomain removes the domain part of a FQDN. Leaves port untouched.
-		"stripDomain": func(hostPort string) string {
-			host, port, err := net.SplitHostPort(hostPort)
-			if err != nil {
-				host = hostPort
-			}
-			ip := net.ParseIP(host)
-			if ip != nil {
-				return hostPort
-			}
-			host = strings.Split(host, ".")[0]
-			if port != "" {
-				return net.JoinHostPort(host, port)
-			}
-			return host
-		},
-
-		// match reports whether the string s
-		// contains any match of the regular expression pattern.
-		// alias for https://golang.org/pkg/regexp/#MatchString
-		"match": regexp.MatchString,
-
-		// reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with
-		// the replacement string repl. Inside repl, $ signs are interpreted as in Expand,
-		// so for instance $1 represents the text of the first submatch.
-		// alias for https://golang.org/pkg/regexp/#Regexp.ReplaceAllString
-		"reReplaceAll": func(pattern, repl, text string) string {
-			re := regexp.MustCompile(pattern)
-			return re.ReplaceAllString(text, repl)
-		},
-
-		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
-		"parseDuration": func(s string) (float64, error) {
-			d, err := promutils.ParseDuration(s)
-			if err != nil {
-				return 0, err
-			}
-			return d.Seconds(), nil
-		},
-
-		// same with parseDuration but returns a time.Duration
-		"parseDurationTime": func(s string) (time.Duration, error) {
-			d, err := promutils.ParseDuration(s)
-			if err != nil {
-				return 0, err
-			}
-			return d, nil
-		},
-
-		/* Numbers */
-
-		// humanize converts given number to a human readable format
-		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
-		"humanize": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			if math.Abs(v) >= 1 {
-				prefix := ""
-				for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
-					if math.Abs(v) < 1000 {
-						break
-					}
-					prefix = p
-					v /= 1000
-				}
-				return fmt.Sprintf("%.4g%s", v, prefix), nil
-			}
-			prefix := ""
-			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
-				if math.Abs(v) >= 1 {
-					break
-				}
-				prefix = p
-				v *= 1000
-			}
-			return fmt.Sprintf("%.4g%s", v, prefix), nil
-		},
-
-		// humanize1024 converts given number to a human readable format with 1024 as base
-		"humanize1024": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			return formatutil.HumanizeBytes(v), nil
-		},
-
-		// humanizeDuration converts given seconds to a human-readable duration
-		"humanizeDuration": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			if v == 0 {
-				return fmt.Sprintf("%.4gs", v), nil
-			}
-			if math.Abs(v) >= 1 {
-				sign := ""
-				if v < 0 {
-					sign = "-"
-					v = -v
-				}
-				seconds := int64(v) % 60
-				minutes := (int64(v) / 60) % 60
-				hours := (int64(v) / 60 / 60) % 24
-				days := int64(v) / 60 / 60 / 24
-				// For days to minutes, we display seconds as an integer.
-				if days != 0 {
-					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil
-				}
-				if hours != 0 {
-					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil
-				}
-				if minutes != 0 {
-					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil
-				}
-				// For seconds, we display 4 significant digits.
-				return fmt.Sprintf("%s%.4gs", sign, v), nil
-			}
-			prefix := ""
-			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
-				if math.Abs(v) >= 1 {
-					break
-				}
-				prefix = p
-				v *= 1000
-			}
-			return fmt.Sprintf("%.4g%ss", v, prefix), nil
-		},
-
-		// humanizePercentage converts given ratio value to a fraction of 100
-		"humanizePercentage": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			return fmt.Sprintf("%.4g%%", v*100), nil
-		},
-
-		// humanizeTimestamp converts given timestamp to a human readable time equivalent
-		"humanizeTimestamp": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			t := timeFromUnixTimestamp(v).Time().UTC()
-			return fmt.Sprint(t), nil
-		},
-
-		// toTime converts given timestamp to a time.Time.
-		"toTime": func(i any) (time.Time, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return time.Time{}, err
-			}
-			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return time.Time{}, fmt.Errorf("cannot convert %v to time.Time", v)
-			}
-			t := timeFromUnixTimestamp(v).Time().UTC()
-			return t, nil
-		},
-
-		/* URLs */
-
-		// externalURL returns value of `external.url` flag
-		"externalURL": func() string {
-			// externalURL function supposed to be substituted at FuncsWithExteralURL().
-			// it is present here only for validation purposes, when there is no
-			// provided datasource.
-			//
-			// return non-empty slice to pass validation with chained functions in template
-			return ""
-		},
-
-		// pathPrefix returns a Path segment from the URL value in `external.url` flag
-		"pathPrefix": func() string {
-			// pathPrefix function supposed to be substituted at FuncsWithExteralURL().
-			// it is present here only for validation purposes, when there is no
-			// provided datasource.
-			//
-			// return non-empty slice to pass validation with chained functions in template
-			return ""
-		},
-
-		// pathEscape escapes the string so it can be safely placed inside a URL path segment.
-		//
-		// See also queryEscape.
-		"pathEscape": url.PathEscape,
-
-		// queryEscape escapes the string so it can be safely placed inside a query arg in URL.
-		//
-		// See also queryEscape.
-		"queryEscape": url.QueryEscape,
-
-		// query executes the MetricsQL/PromQL query against
-		// configured `datasource.url` address.
-		// For example, {{ query "foo" | first | value }} will
-		// execute "/api/v1/query?query=foo" request and will return
-		// the first value in response.
-		"query": func(_ string) ([]metric, error) {
-			// query function supposed to be substituted at FuncsWithQuery().
-			// it is present here only for validation purposes, when there is no
-			// provided datasource.
-			//
-			// return non-empty slice to pass validation with chained functions in template
-			// see issue #989 for details
-			return []metric{{}}, nil
-		},
-
-		// first returns the first by order element from the given metrics list.
-		// usually used alongside with `query` template function.
-		"first": func(metrics []metric) (metric, error) {
-			if len(metrics) > 0 {
-				return metrics[0], nil
-			}
-			return metric{}, errors.New("first() called on vector with no elements")
-		},
-
-		// label returns the value of the given label name for the given metric.
-		// usually used alongside with `query` template function.
-		"label": func(label string, m metric) string {
-			return m.Labels[label]
-		},
-
-		// value returns the value of the given metric.
-		// usually used alongside with `query` template function.
-		"value": func(m metric) float64 {
-			return m.Value
-		},
-
-		// strvalue returns metric name.
-		"strvalue": func(m metric) string {
-			return m.Labels["__name__"]
-		},
-
-		// sortByLabel sorts the given metrics by provided label key
-		"sortByLabel": func(label string, metrics []metric) []metric {
-			sort.SliceStable(metrics, func(i, j int) bool {
-				return metrics[i].Labels[label] < metrics[j].Labels[label]
-			})
-			return metrics
-		},
-
-		/* Helpers */
-
-		// Converts a list of objects to a map with keys arg0, arg1 etc.
-		// This is intended to allow multiple arguments to be passed to templates.
-		"args": func(args ...any) map[string]any {
-			result := make(map[string]any)
-			for i, a := range args {
-				result[fmt.Sprintf("arg%d", i)] = a
-			}
-			return result
-		},
-
-		// safeHtml marks string as HTML not requiring auto-escaping.
-		//
-		// See also htmlEscape.
-		"safeHtml": func(text string) htmlTpl.HTML {
-			return htmlTpl.HTML(text)
-		},
+// ExecuteWithTemplate executes with the given template and data
+func ExecuteWithTemplate(data AlertTplData, tpl *textTpl.Template) (string, error) {
+	fullData := tplData{
+		data,
+		externalLabels,
+		externalURL.String(),
 	}
-}
-
-// Time is the number of milliseconds since the epoch
-// (1970-01-01 00:00 UTC) excluding leap seconds.
-type Time int64
-
-// timeFromUnixTimestamp returns the Time equivalent to t in unix timestamp.
-func timeFromUnixTimestamp(t float64) Time {
-	return Time(t * 1e3)
-}
-
-// The number of nanoseconds per minimum tick.
-const nanosPerTick = int64(minimumTick / time.Nanosecond)
-
-// MinimumTick is the minimum supported time resolution. This has to be
-// at least time.Second in order for the code below to work.
-const minimumTick = time.Millisecond
-
-// second is the Time duration equivalent to one second.
-const second = int64(time.Second / minimumTick)
-
-// Time returns the time.Time representation of t.
-func (t Time) Time() time.Time {
-	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
-}
-
-func toFloat64(v any) (float64, error) {
-	switch i := v.(type) {
-	case float64:
-		return i, nil
-	case float32:
-		return float64(i), nil
-	case int64:
-		return float64(i), nil
-	case int32:
-		return float64(i), nil
-	case int:
-		return float64(i), nil
-	case uint64:
-		return float64(i), nil
-	case uint32:
-		return float64(i), nil
-	case uint:
-		return float64(i), nil
-	case string:
-		return strconv.ParseFloat(i, 64)
-	default:
-		return 0, fmt.Errorf("unexpected value type %v", i)
+	var buf bytes.Buffer
+	// returns the zero value for the map type's element
+	tpl.Option("missingkey=zero")
+	if err := tpl.Execute(&buf, fullData); err != nil {
+		return "", fmt.Errorf("failed to execute template: %w", err)
 	}
+	return buf.String(), nil
 }

app/vmalert/templates/template_test.go

@@ -1,239 +1,226 @@
 package templates
 
 import (
-	"math"
+	"fmt"
 	"net/url"
-	"strings"
 	"testing"
-	textTpl "text/template"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
-func TestTemplateFuncs_StringConversion(t *testing.T) {
-	f := func(funcName, s, resultExpected string) {
+func TestValidateTemplates(t *testing.T) {
+	f := func(annotations map[string]string, isValid bool) {
 		t.Helper()
 
-		funcs := templateFuncs()
-		v := funcs[funcName]
-		fLocal := v.(func(s string) string)
-		result := fLocal(s)
-		if result != resultExpected {
-			t.Fatalf("unexpected result for %s(%q); got\n%s\nwant\n%s", funcName, s, result, resultExpected)
+		err := ValidateTemplates(annotations)
+		if (err == nil) != isValid {
+			t.Fatalf("failed to validate template, got %t; want %t", (err == nil), isValid)
 		}
 	}
 
-	f("title", "foo bar", "Foo Bar")
-	f("toUpper", "foo", "FOO")
-	f("toLower", "FOO", "foo")
-	f("pathEscape", "foo/bar\n+baz", "foo%2Fbar%0A+baz")
-	f("queryEscape", "foo+bar\n+baz", "foo%2Bbar%0A%2Bbaz")
-	f("jsonEscape", `foo{bar="baz"}`+"\n + 1", `"foo{bar=\"baz\"}\n + 1"`)
-	f("quotesEscape", `foo{bar="baz"}`+"\n + 1", `foo{bar=\"baz\"}\n + 1`)
-	f("htmlEscape", "foo < 10\nabc", "foo &lt; 10\nabc")
-	f("crlfEscape", "foo\nbar\rx", `foo\nbar\rx`)
-	f("stripPort", "foo", "foo")
-	f("stripPort", "foo:1234", "foo")
-	f("stripDomain", "foo.bar.baz", "foo")
-	f("stripDomain", "foo.bar:123", "foo:123")
+	// empty
+	f(map[string]string{}, true)
+
+	// wrong text
+	f(map[string]string{
+		"summary": "{{",
+	}, false)
+
+	// valid
+	f(map[string]string{
+		"value":   "{{$value}}",
+		"summary": "it's a test summary",
+	}, true)
+
+	// invalid variable
+	f(map[string]string{
+		"value":   "{{$invalidValue}}",
+		"summary": "it's a test summary",
+	}, false)
 }
 
-func TestTemplateFuncs_Match(t *testing.T) {
-	funcs := templateFuncs()
-	// check "match" func
-	matchFunc := funcs["match"].(func(pattern, s string) (bool, error))
-	if _, err := matchFunc("invalid[regexp", "abc"); err == nil {
-		t.Fatalf("expecting non-nil error on invalid regexp")
-	}
-	ok, err := matchFunc("abc", "def")
+func TestExecuteWithoutTemplate(t *testing.T) {
+	extLabels := make(map[string]string)
+	const (
+		extCluster = "prod"
+		extDC      = "east"
+		extURL     = "https://foo.bar"
+	)
+	url, _ := url.Parse(extURL)
+	extLabels["cluster"] = extCluster
+	extLabels["dc"] = extDC
+
+	err := Init(nil, extLabels, *url)
 	if err != nil {
-		t.Fatalf("unexpected error")
+		t.Fatalf("cannot init templates: %s", err)
 	}
-	if ok {
-		t.Fatalf("unexpected match")
-	}
-	ok, err = matchFunc("a.+b", "acsdb")
-	if err != nil {
-		t.Fatalf("unexpected error")
-	}
-	if !ok {
-		t.Fatalf("unexpected mismatch")
-	}
-}
 
-func TestTemplateFuncs_Formatting(t *testing.T) {
-	f := func(funcName string, p any, resultExpected string) {
+	f := func(data AlertTplData, annotations, expResults map[string]string) {
 		t.Helper()
 
-		funcs := templateFuncs()
-		v := funcs[funcName]
-		fLocal := v.(func(s any) (string, error))
-		result, err := fLocal(p)
-		if err != nil {
-			t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
+		qFn := func(_ string) ([]datasource.Metric, error) {
+			return []datasource.Metric{
+				{
+					Labels: []prompbmarshal.Label{
+						{Name: "foo", Value: "bar"},
+						{Name: "baz", Value: "qux"},
+					},
+					Values:     []float64{1},
+					Timestamps: []int64{1},
+				},
+				{
+					Labels: []prompbmarshal.Label{
+						{Name: "foo", Value: "garply"},
+						{Name: "baz", Value: "fred"},
+					},
+					Values:     []float64{2},
+					Timestamps: []int64{1},
+				},
+			}, nil
 		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
-		}
-	}
 
-	f("humanize1024", float64(0), "0")
-	f("humanize1024", math.Inf(0), "+Inf")
-	f("humanize1024", math.NaN(), "NaN")
-	f("humanize1024", float64(127087), "124.1ki")
-	f("humanize1024", float64(130137088), "124.1Mi")
-	f("humanize1024", float64(133260378112), "124.1Gi")
-	f("humanize1024", float64(136458627186688), "124.1Ti")
-	f("humanize1024", float64(139733634239168512), "124.1Pi")
-	f("humanize1024", float64(143087241460908556288), "124.1Ei")
-	f("humanize1024", float64(146521335255970361638912), "124.1Zi")
-	f("humanize1024", float64(150037847302113650318245888), "124.1Yi")
-	f("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
-
-	f("humanize", float64(127087), "127.1k")
-	f("humanize", float64(136458627186688), "136.5T")
-
-	f("humanizeDuration", 1, "1s")
-	f("humanizeDuration", 0.2, "200ms")
-	f("humanizeDuration", 42000, "11h 40m 0s")
-	f("humanizeDuration", 16790555, "194d 8h 2m 35s")
-
-	f("humanizePercentage", 1, "100%")
-	f("humanizePercentage", 0.8, "80%")
-	f("humanizePercentage", 0.015, "1.5%")
-
-	f("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
-}
-
-func mkTemplate(current, replacement any) textTemplate {
-	tmpl := textTemplate{}
-	if current != nil {
-		switch val := current.(type) {
-		case string:
-			tmpl.current = textTpl.Must(newTemplate().Parse(val))
-		}
-	}
-	if replacement != nil {
-		switch val := replacement.(type) {
-		case string:
-			tmpl.replacement = textTpl.Must(newTemplate().Parse(val))
-		}
-	}
-	return tmpl
-}
-
-func equalTemplates(tmpls ...*textTpl.Template) bool {
-	var cmp *textTpl.Template
-	for i, tmpl := range tmpls {
-		if i == 0 {
-			cmp = tmpl
-		} else {
-			if cmp == nil || tmpl == nil {
-				if cmp != tmpl {
-					return false
-				}
-				continue
+		for k := range annotations {
+			v, err := ExecuteWithoutTemplate(qFn, annotations[k], data)
+			if err != nil {
+				t.Fatalf("cannot execute template: %s", err)
 			}
-			if len(tmpl.Templates()) != len(cmp.Templates()) {
-				return false
-			}
-			for _, t := range tmpl.Templates() {
-				tp := cmp.Lookup(t.Name())
-				if tp == nil {
-					return false
-				}
-				if tp.Root.String() != t.Root.String() {
-					return false
-				}
+			if v != expResults[k] {
+				t.Fatalf("unexpected result; got %s; want %s", v, expResults[k])
 			}
 		}
+
 	}
-	return true
-}
-
-func TestTemplatesLoad_Failure(t *testing.T) {
-	f := func(pathPatterns []string, expectedErrStr string) {
-		t.Helper()
-
-		err := Load(pathPatterns, url.URL{})
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-
-		errStr := err.Error()
-		if !strings.Contains(errStr, expectedErrStr) {
-			t.Fatalf("the returned error %q doesn't contain %q", errStr, expectedErrStr)
-		}
-	}
-
-	// load template with syntax error
-	f([]string{
-		"templates/other/nested/bad0-*.tpl",
-		"templates/test/good0-*.tpl",
-	}, "failed to parse template glob")
-}
-
-func TestTemplatesLoad_Success(t *testing.T) {
-	f := func(pathPatterns []string, expectedTmpl textTemplate) {
-		t.Helper()
-
-		masterTmplOrig := masterTmpl
-		defer func() {
-			masterTmpl = masterTmplOrig
-		}()
-
-		if err := Load(pathPatterns, url.URL{}); err != nil {
-			t.Fatalf("cannot load templates: %s", err)
-		}
-		Reload()
-
-		if !equalTemplates(masterTmpl.replacement, expectedTmpl.replacement) {
-			t.Fatalf("unexpected replacement template\ngot\n%+v\nwant\n%+v", masterTmpl.replacement, expectedTmpl.replacement)
-		}
-		if !equalTemplates(masterTmpl.current, expectedTmpl.current) {
-			t.Fatalf("unexpected current template\ngot\n%+v\nwant\n%+v", masterTmpl.current, expectedTmpl.current)
-		}
-	}
-
-	// non existing path
-	pathPatterns := []string{
-		"templates/non-existing/good-*.tpl",
-		"templates/absent/good-*.tpl",
-	}
-	expectedTmpl := mkTemplate(``, nil)
-	f(pathPatterns, expectedTmpl)
-
-	// existing path
-	pathPatterns = []string{
-		"templates/test/good0-*.tpl",
-	}
-	expectedTmpl = mkTemplate(`
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.2" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`, nil)
-	f(pathPatterns, expectedTmpl)
-
-	// existing path defined template override
-	pathPatterns = []string{
-		"templates/other/nested/good0-*.tpl",
-	}
-	expectedTmpl = mkTemplate(`
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.1" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`, nil)
-	f(pathPatterns, expectedTmpl)
+
+	// empty-alert
+	f(AlertTplData{}, map[string]string{}, map[string]string{})
+
+	// no-template
+	f(AlertTplData{
+		Value: 1e4,
+		Labels: map[string]string{
+			"instance": "localhost",
+		},
+	}, map[string]string{
+		"summary":     "it's a test summary",
+		"description": "it's a test description",
+	}, map[string]string{
+		"summary":     "it's a test summary",
+		"description": "it's a test description",
+	})
+
+	// label-template
+	f(AlertTplData{
+		Value: 1e4,
+		Labels: map[string]string{
+			"job":      "staging",
+			"instance": "localhost",
+		},
+		For: 5 * time.Minute,
+	}, map[string]string{
+		"summary":            "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
+		"description":        "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
+		"non-existing-label": "{{$labels.nonexisting}}",
+	}, map[string]string{
+		"summary":            "Too high connection number for localhost for job staging",
+		"description":        "It is 10000 connections for localhost for more than 5m0s",
+		"non-existing-label": "",
+	})
+
+	// label template override
+	f(AlertTplData{
+		Value: 1e4,
+	}, map[string]string{
+		"summary":     `{{- define "default.template" -}} {{ printf "summary" }} {{- end -}} {{ template "default.template" . }}`,
+		"description": `{{- define "default.template" -}} {{ printf "description" }} {{- end -}} {{ template "default.template" . }}`,
+		"value":       `{{$value }}`,
+	}, map[string]string{
+		"summary":     "summary",
+		"description": "description",
+		"value":       "10000",
+	})
+
+	// expression-template
+	f(AlertTplData{
+		Expr: `vm_rows{"label"="bar"}<0`,
+	}, map[string]string{
+		"exprEscapedQuery":  "{{ $expr|queryEscape }}",
+		"exprEscapedPath":   "{{ $expr|pathEscape }}",
+		"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
+		"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
+		"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
+	}, map[string]string{
+		"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
+		"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
+		"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
+		"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
+		"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
+	})
+
+	// query function
+	f(AlertTplData{
+		Expr: `vm_rows{"label"="bar"}>0`,
+	}, map[string]string{
+		"summary": `{{ query "foo" | first | value }}`,
+		"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
+	}, map[string]string{
+		"summary": "1",
+		"desc":    "bar 1;garply 2;",
+	})
+
+	// external
+	f(AlertTplData{
+		Value: 1e4,
+		Labels: map[string]string{
+			"job":      "staging",
+			"instance": "localhost",
+		},
+	}, map[string]string{
+		"url":         "{{ $externalURL }}",
+		"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
+		"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
+	}, map[string]string{
+		"url":         extURL,
+		"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
+		"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
+	})
+
+	// alert, group IDs & ActiveAt time
+	f(AlertTplData{
+		AlertID:  42,
+		GroupID:  24,
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"url":     "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
+		"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
+	}, map[string]string{
+		"url":     "/api/v1/alert?alertID=42&groupID=24",
+		"diagram": "![](http://example.com?render=1660941298",
+	})
+
+	// ActiveAt time is nil
+	f(AlertTplData{}, map[string]string{
+		"default_time": "{{$activeAt}}",
+	}, map[string]string{
+		"default_time": "0001-01-01 00:00:00 +0000 UTC",
+	})
+
+	// ActiveAt custom format
+	f(AlertTplData{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
+	}, map[string]string{
+		"fire_time": "2022/08/19 20:34:58",
+	})
+
+	// ActiveAt query range
+	f(AlertTplData{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
+	}, map[string]string{
+		"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
+	})
 }

docs/changelog/CHANGELOG.md

@@ -31,6 +31,7 @@ Released at 2025-01-10
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): improve query performance on systems with high number of CPU cores. See [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7416) for details.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): add command-line flag `-search.maxBinaryOpPushdownLabelValues` to allow using labels with more candidate values as push down filter in binary operation. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7243). Thanks to @tydhot for implementation.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): add command-line flag `storage.finalDedupScheduleCheckInterval` to control the final deduplication process interval. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7880) for details.
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): improve alerting rules [templating](https://docs.victoriametrics.com/vmalert/#templating) performance.
 
 * BUGFIX: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards): consistently use `vmagent_remotewrite_pending_data_bytes` on vmagent dashboard to represent persistent queue size.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): fix the auto-generated metrics `ALERTS` and `ALERTS_FOR_STATE` for alerting rules. Previously, metrics might have incorrect labels and affect the restore process. See this [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7796).

docs/vmalert.md

@@ -273,6 +273,7 @@ expr: <string>
 
 # Labels to add or overwrite for each alert.
 # In case of conflicts, original labels are kept with prefix `exported_`.
+# label value supports [templating functions](#template-functions), [reusable templates](#reusable-templates) and limited variables, such as {{$value}}, {{$expr}}.
 labels:
   [ <labelname>: <tmpl_string> ]