lib/httpserver: support multitnenacy via headers

This commit adds possibility to omit tenantID in the URL path. In this case,
tenantID will be fetched from HTTP headers `AccountID` and `ProjectID`.
If headers are missing too, then default `0:0` tenantID is used.

This functionality can be enabled only if -enableMultitenantHandlers
cmd-line flag was set to vminsert, vmselect or vmagent.

Motivation: this change makes VM configuration for multienancy
consistent with VL configuration - see
https://docs.victoriametrics.com/victorialogs/#multitenancy. And keeps
backward compatibility in the same time.

fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4241

app/vmagent/main.go

@@ -83,6 +83,9 @@ var (
 	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 0, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
 	maxLabelNameLen        = flag.Int("maxLabelNameLen", 0, "The maximum length of label names in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
 	maxLabelValueLen       = flag.Int("maxLabelValueLen", 0, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
+
+	enableMultitenancyViaHeaders = flag.Bool("enableMultitenancyViaHeaders", false, "Enables multitenancy via HTTP headers. "+
+		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy")
 )
 
 var (
@@ -216,7 +219,7 @@ func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
 	}
 	return func(req *http.Request) error {
 		path := strings.ReplaceAll(req.URL.Path, "//", "/")
-		at, err := getAuthTokenFromPath(path)
+		at, err := getAuthTokenFromPath(path, req.Header)
 		if err != nil {
 			return fmt.Errorf("cannot obtain auth token from path %q: %w", path, err)
 		}
@@ -224,8 +227,15 @@ func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
 	}
 }
 
-func getAuthTokenFromPath(path string) (*auth.Token, error) {
-	p, err := httpserver.ParsePath(path)
+func parsePath(path string, header http.Header) (*httpserver.Path, error) {
+	if *enableMultitenancyViaHeaders {
+		return httpserver.ParsePathAndHeaders(path, header)
+	}
+	return httpserver.ParsePath(path)
+}
+
+func getAuthTokenFromPath(path string, header http.Header) (*auth.Token, error) {
+	p, err := parsePath(path, header)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse multitenant path: %w", err)
 	}
@@ -559,14 +569,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path string) bool {
-	p, err := httpserver.ParsePath(path)
+	p, err := parsePath(path, r.Header)
 	if err != nil {
 		// Cannot parse multitenant path. Skip it - probably it will be parsed later.
 		return false
 	}
 	if p.Prefix != "insert" {
-		httpserver.Errorf(w, r, `unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
-		return true
+		// processMultitenantRequest is called for all unmatched path variants,
+		// but we should try parsing only /insert prefixed to avoid catching all possible paths.
+		return false
 	}
 	at, err := auth.NewTokenPossibleMultitenant(p.AuthToken)
 	if err != nil {

apptest/client.go

@@ -2,6 +2,7 @@ package apptest
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -105,6 +106,35 @@ func (c *Client) Write(t *testing.T, address string, data []string) {
 	}
 }
 
+// getClusterPath returns path in cluster's URL format.
+// Based on QueryOpts, it will either put tenant ID into URL
+// or will skip it if tenant is set via HTTP headers.
+func getClusterPath(addr, prefix, suffix string, o QueryOpts) string {
+	if o.Tenant != "" {
+		// QueryOpts.Tenant has priority over headers
+		return tenantViaURL(addr, prefix, o.Tenant, suffix)
+	}
+
+	h := o.getHeaders()
+	if h.Get("AccountID") != "" || h.Get("ProjectID") != "" {
+		return tenantViaHeaders(addr, prefix, suffix)
+	}
+
+	// tenant is missing in QueryOpts and in HTTP headers. Falling back to default 0:0 tenant in URL
+	return tenantViaURL(addr, prefix, "0:0", suffix)
+}
+
+// tenantViaURL returns path in cluster's URL format with tenant specified in URL
+func tenantViaURL(addr, prefix, tenant, suffix string) string {
+	return fmt.Sprintf("http://%s/%s/%s/%s", addr, prefix, tenant, suffix)
+}
+
+// tenantViaHeaders returns path in cluster's URL format where tenant is omitted in URL
+// Only supported if -enableMultitenancyViaHeaders is specified
+func tenantViaHeaders(addr, prefix, suffix string) string {
+	return fmt.Sprintf("http://%s/%s/%s", addr, prefix, suffix)
+}
+
 // readAllAndClose reads everything from the response body and then closes it.
 func readAllAndClose(t *testing.T, responseBody io.ReadCloser) string {
 	t.Helper()

apptest/model.go

@@ -93,14 +93,6 @@ type QueryOpts struct {
 	Headers        http.Header
 }
 
-// getTenant returns tenant with optional default value
-func (qos *QueryOpts) getTenant() string {
-	if qos.Tenant == "" {
-		return "0"
-	}
-	return qos.Tenant
-}
-
 func (qos *QueryOpts) getHeaders() http.Header {
 	if qos.Headers == nil {
 		qos.Headers = make(http.Header)

apptest/tests/multitenancy_via_headers_test.go

@@ -0,0 +1,313 @@
+package tests
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+)
+
+func TestClusterMultiTenantSelectViaHeaders(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
+	cmpSROpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Status", "IsPartial")
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+	vmstorage := tc.MustStartVmstorage("vmstorage", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage",
+		"-retentionPeriod=100y",
+	})
+	vminsert := tc.MustStartVminsert("vminsert", []string{
+		"-storageNode=" + vmstorage.VminsertAddr(),
+		"-enableMultitenancyViaHeaders",
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmstorage.VmselectAddr(),
+		"-search.tenantCacheExpireDuration=0",
+		"-enableMultitenancyViaHeaders",
+	})
+
+	multitenant := make(http.Header)
+	multitenant.Set("AccountID", "multitenant")
+
+	// test for empty tenants request
+	got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
+		Headers: multitenant,
+		Step:    "5m",
+		Time:    "2022-05-10T08:03:00.000Z",
+	})
+	want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// ingest per tenant data and verify it with search
+	samples := []string{
+		`foo_bar 1.00 1652169600000`, // 2022-05-10T08:00:00Z
+		`foo_bar 2.00 1652169660000`, // 2022-05-10T08:01:00Z
+		`foo_bar 3.00 1652169720000`, // 2022-05-10T08:02:00Z
+	}
+	tenantHeaders := []map[string]string{
+		{"AccountID": "1", "ProjectID": "1"},
+		{"AccountID": "1", "ProjectID": "15"},
+		{"AccountID": "2"},
+		{"ProjectID": "3"},
+	}
+	instantCT := "2022-05-10T08:05:00.000Z" // 1652169900 Unix seconds
+	for _, headers := range tenantHeaders {
+		h := make(http.Header)
+		for k, v := range headers {
+			h.Set(k, v)
+		}
+		vminsert.PrometheusAPIV1ImportPrometheus(t, samples, apptest.QueryOpts{Headers: h})
+		vmstorage.ForceFlush(t)
+
+		// verify tenants are searchable via tenantID in headers
+		got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
+			Headers: h, Time: instantCT,
+		})
+		want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]}]}}`)
+		if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+			t.Errorf("unexpected response (-want, +got):\n%s", diff)
+		}
+	}
+
+	// verify all tenants searchable with multitenant header
+
+	//  /api/v1/query
+	want = apptest.NewPrometheusAPIV1QueryResponse(t,
+		`{"data":
+       {"result":[
+          {"metric":{"__name__":"foo_bar","vm_account_id":"0","vm_project_id":"3"},"value":[1652169900,"3"]},
+          {"metric":{"__name__":"foo_bar","vm_account_id":"1","vm_project_id": "1"},"value":[1652169900,"3"]},
+          {"metric":{"__name__":"foo_bar","vm_account_id":"1","vm_project_id":"15"},"value":[1652169900,"3"]},
+          {"metric":{"__name__":"foo_bar","vm_account_id":"2","vm_project_id":"0"},"value":[1652169900,"3"]}
+                 ]
+       }
+     }`,
+	)
+
+	got = vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
+		Headers: multitenant,
+		Time:    instantCT,
+	})
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// /api/v1/query_range aggregated by tenant labels
+	query := "sum(foo_bar) by(vm_account_id,vm_project_id)"
+	got = vmselect.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{
+		Headers: multitenant,
+		Start:   "2022-05-10T07:59:00.000Z",
+		End:     "2022-05-10T08:05:00.000Z",
+		Step:    "1m",
+	})
+
+	want = apptest.NewPrometheusAPIV1QueryResponse(t,
+		`{"data": 
+        {"result": [
+          {"metric": {"vm_account_id": "0","vm_project_id":"3"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]},
+          {"metric": {"vm_account_id": "1","vm_project_id":"1"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]},
+          {"metric": {"vm_account_id": "1","vm_project_id":"15"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]},
+          {"metric": {"vm_account_id": "2","vm_project_id":"0"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]}
+                   ]
+        }
+     }`)
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// verify /api/v1/series response
+
+	wantSR := apptest.NewPrometheusAPIV1SeriesResponse(t,
+		`{"data": [
+        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
+        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"},
+        {"__name__":"foo_bar", "vm_account_id":"2", "vm_project_id":"0"},
+        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"3"}
+              ]
+     }`)
+	wantSR.Sort()
+
+	gotSR := vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
+		Headers: multitenant,
+		Start:   "2022-05-10T08:03:00.000Z",
+	})
+	gotSR.Sort()
+	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// test ingestion with multitenant header, tenants must be populated from labels
+	//
+	var tenantLabelsSamples = []string{
+		`foo_bar{vm_account_id="5"} 1.00 1652169720000`,                    // 2022-05-10T08:02:00Z'
+		`foo_bar{vm_project_id="10"} 2.00 1652169660000`,                   // 2022-05-10T08:01:00Z
+		`foo_bar{vm_account_id="5",vm_project_id="15"} 3.00 1652169720000`, // 2022-05-10T08:02:00Z
+	}
+
+	vminsert.PrometheusAPIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Headers: multitenant})
+	vmstorage.ForceFlush(t)
+
+	//  /api/v1/query with query filters
+	want = apptest.NewPrometheusAPIV1QueryResponse(t,
+		`{"data":
+        {"result":[
+	         {"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id": "0"},"value":[1652169900,"1"]},
+	         {"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id":"15"},"value":[1652169900,"3"]}
+	                ]
+        }
+    }`,
+	)
+	got = vmselect.PrometheusAPIV1Query(t, `foo_bar{vm_account_id="5"}`, apptest.QueryOpts{
+		Time:    instantCT,
+		Headers: multitenant,
+	})
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// /api/v1/series with extra_filters
+
+	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
+		`{"data": [
+	       {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"15"},
+	       {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"}
+	             ]
+	   }`)
+	wantSR.Sort()
+	gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
+		Start:        "2022-05-10T08:00:00.000Z",
+		End:          "2022-05-10T08:30:00.000Z",
+		ExtraFilters: []string{`{vm_project_id="15"}`},
+		Headers:      multitenant,
+	})
+	gotSR.Sort()
+
+	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// /api/v1/label/../value with extra_filters
+
+	wantVR := apptest.NewPrometheusAPIV1LabelValuesResponse(t,
+		`{"data": [
+	       "5"
+	             ]
+	   }`)
+	// matchQuery is ignored for /api/v1/label/<labelName>/values lookups with multitenant token
+	gotVR := vmselect.PrometheusAPIV1LabelValues(t, "vm_account_id", "xxx", apptest.QueryOpts{
+		Start:        "2022-05-10T08:00:00.000Z",
+		End:          "2022-05-10T08:30:00.000Z",
+		ExtraFilters: []string{`{vm_account_id="5"}`},
+		Headers:      multitenant,
+	})
+	gotSR.Sort()
+
+	if diff := cmp.Diff(wantVR, gotVR, cmpopts.IgnoreFields(apptest.PrometheusAPIV1LabelValuesResponse{}, "Status", "IsPartial")); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Delete series from specific tenant
+	tenantID := make(http.Header)
+	tenantID.Set("AccountID", "5")
+	tenantID.Set("ProjectID", "15")
+	vmselect.APIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
+		Headers: tenantID,
+	})
+	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
+		`{"data": [
+        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"3"},
+        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
+        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
+        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"},
+        {"__name__":"foo_bar", "vm_account_id":"2", "vm_project_id":"0"},
+        {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"}
+              ]
+     }`)
+	wantSR.Sort()
+
+	gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
+		Headers: multitenant,
+		Start:   "2022-05-10T08:03:00.000Z",
+	})
+	gotSR.Sort()
+	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Delete series for multitenant with tenant filter
+	vmselect.APIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
+		Headers: multitenant,
+	})
+
+	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
+		`{"data": [
+        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"3"},
+        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
+        {"__name__":"foo_bar", "vm_account_id":"2", "vm_project_id":"0"},
+        {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"}
+              ]
+     }`)
+	wantSR.Sort()
+
+	gotSR = vmselect.PrometheusAPIV1Series(t, `foo_bar`, apptest.QueryOpts{
+		Headers: multitenant,
+		Start:   "2022-05-10T08:03:00.000Z",
+	})
+	gotSR.Sort()
+	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	if got := vmselect.GetIntMetric(t, `vm_cache_requests_total{type="multitenancy/tenants"}`); got != 0 {
+		t.Errorf("unexpected multitenancy tenants cache requests; got %d; want 0", got)
+	}
+
+	if got := vmselect.GetIntMetric(t, `vm_cache_misses_total{type="multitenancy/tenants"}`); got != 0 {
+		t.Errorf("unexpected multitenancy tenants cache misses; got %d; want 0", got)
+	}
+
+	if got := vmselect.GetIntMetric(t, `vm_cache_entries{type="multitenancy/tenants"}`); got != 0 {
+		t.Errorf("unexpected multitenancy tenants cache entries; got %d; want 0", got)
+	}
+
+	// verify that tenant in path has priority over tenant specified in headers
+
+	// /api/v1/import/prometheus
+
+	tenantInHeader := make(http.Header)
+	tenantInHeader.Set("AccountID", "42")
+	tenantInPath := "112"
+	vminsert.PrometheusAPIV1ImportPrometheus(t, samples, apptest.QueryOpts{
+		// tenants in header and path clash - path should have higher priority on ingestion
+		Headers: tenantInHeader,
+		Tenant:  "112",
+	})
+	vmstorage.ForceFlush(t)
+
+	want = apptest.NewPrometheusAPIV1QueryResponse(t,
+		`{"data":
+       {"result":[
+          {"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]}
+                 ]
+       }
+     }`,
+	)
+	got = vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
+		// tenants in header and path clash - path should have higher priority on ingestion
+		Headers: multitenant,
+		Tenant:  tenantInPath,
+		Time:    instantCT,
+	})
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+}

apptest/tests/vmagent_remotewrite_test.go

@@ -584,3 +584,69 @@ func TestClusterVMAgentForwardMetricsMetadata(t *testing.T) {
 	})
 
 }
+
+// See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy
+func TestSingleVMAgentMultitenancy(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	remoteWriteSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer remoteWriteSrv.Close()
+
+	vmagent := tc.MustStartVmagent("vmagent-multitenancy", []string{
+		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
+		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-multitenancy",
+		"-enableMultitenantHandlers",
+		"-enableMultitenancyViaHeaders",
+	}, ``)
+
+	vmagent.APIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{Tenant: "2"})
+	v := vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="2",projectID="0"}`)
+	if v != 1 {
+		t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=2")
+	}
+
+	vmagent.APIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{Tenant: "2:2"})
+	v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="2",projectID="2"}`)
+	if v != 1 {
+		t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=2, projectID=2")
+	}
+
+	headers := make(http.Header)
+	headers.Set("AccountID", "3")
+	vmagent.APIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{Headers: headers})
+	v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="3",projectID="0"}`)
+	if v != 1 {
+		t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=3, projectID=0")
+	}
+
+	headers.Set("AccountID", "3")
+	headers.Set("ProjectID", "3")
+	vmagent.APIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{Headers: headers})
+	v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="3",projectID="3"}`)
+	if v != 1 {
+		t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=3, projectID=3")
+	}
+
+	// tenants in header and path clash - path should have higher priority on ingestion
+	opts := apptest.QueryOpts{Headers: make(http.Header)}
+	opts.Headers.Set("AccountID", "4")
+	opts.Tenant = "5"
+	vmagent.APIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, opts)
+	v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="5",projectID="0"}`)
+	if v != 1 {
+		t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=5, projectID=0")
+	}
+}

apptest/vmagent.go

@@ -21,8 +21,7 @@ type Vmagent struct {
 	*app
 	*ServesMetrics
 
-	httpListenAddr           string
-	apiV1ImportPrometheusURL string
+	httpListenAddr string
 }
 
 // StartVmagent starts an instance of vmagent with the given flags. It also
@@ -52,8 +51,7 @@ func StartVmagent(instance string, flags []string, cli *Client, promScrapeConfig
 			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
 			cli:        cli,
 		},
-		httpListenAddr:           stderrExtracts[0],
-		apiV1ImportPrometheusURL: fmt.Sprintf("http://%s/api/v1/import/prometheus", stderrExtracts[0]),
+		httpListenAddr: stderrExtracts[0],
 	}, nil
 }
 
@@ -86,12 +84,33 @@ func (app *Vmagent) APIV1ImportPrometheusNoWaitFlush(t *testing.T, records []str
 	data := []byte(strings.Join(records, "\n"))
 	headers := opts.getHeaders()
 	headers.Set("Content-Type", "text/plain")
-	_, statusCode := app.cli.Post(t, app.apiV1ImportPrometheusURL, data, headers)
+	url := getVMAgentInsertPath(app.httpListenAddr, "prometheus/api/v1/import/prometheus", opts)
+	_, statusCode := app.cli.Post(t, url, data, headers)
 	if statusCode != http.StatusNoContent {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
 	}
 }
 
+// getVMAgentInsertPath returns URL path for writes.
+// If tenant is set in QueryOpts, it will return cluster-like path for ingestion.
+// If tenant is empty, it will return single-node (no tenants) path.
+func getVMAgentInsertPath(addr, suffix string, o QueryOpts) string {
+	if o.Tenant != "" {
+		// QueryOpts.Tenant has priority over headers
+		return fmt.Sprintf("http://%s/insert/%s/%s", addr, o.Tenant, suffix)
+	}
+
+	h := o.getHeaders()
+	if h.Get("AccountID") != "" || h.Get("ProjectID") != "" {
+		// vmagent supports tenantID in HTTP headers only if -enableMultitenantHandlers and -enableMultitenancyViaHeaders are set
+		// see https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy
+		return fmt.Sprintf("http://%s/insert/%s", addr, suffix)
+	}
+
+	// tenant is missing in QueryOpts and in HTTP headers. Use single-node (no tenants) path
+	return fmt.Sprintf("http://%s/%s", addr, suffix)
+}
+
 // RemoteWriteRequestsRetriesCountTotal sums up the total retries for remote write requests.
 func (app *Vmagent) RemoteWriteRequestsRetriesCountTotal(t *testing.T) int {
 	total := 0.0
@@ -168,10 +187,7 @@ func (app *Vmagent) ReloadRelabelConfigs(t *testing.T) {
 func (app *Vmagent) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
 	t.Helper()
 
-	url := fmt.Sprintf("http://%s/prometheus/api/v1/write", app.httpListenAddr)
-	if opts.Tenant != "" {
-		url = fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/write", app.httpListenAddr, opts.Tenant)
-	}
+	url := getVMAgentInsertPath(app.httpListenAddr, "prometheus/api/v1/write", opts)
 	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
 	recordsCount := len(wr.Timeseries)
 	if prommetadata.IsEnabled() {

apptest/vminsert.go

@@ -106,7 +106,7 @@ func (app *Vminsert) HTTPAddr() string {
 func (app *Vminsert) InfluxWrite(t *testing.T, records []string, opts QueryOpts) {
 	t.Helper()
 
-	url := fmt.Sprintf("http://%s/insert/%s/influx/write", app.httpListenAddr, opts.getTenant())
+	url := getClusterPath(app.httpListenAddr, "insert", "influx/write", opts)
 	uv := opts.asURLValues()
 	uvs := uv.Encode()
 	if len(uvs) > 0 {
@@ -141,7 +141,7 @@ func (app *Vminsert) GraphiteWrite(t *testing.T, records []string, _ QueryOpts)
 func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
 	t.Helper()
 
-	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/csv", app.httpListenAddr, opts.getTenant())
+	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/csv", opts)
 	uv := opts.asURLValues()
 	uvs := uv.Encode()
 	if len(uvs) > 0 {
@@ -166,7 +166,7 @@ func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, op
 func (app *Vminsert) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
 	t.Helper()
 
-	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/native", app.httpListenAddr, opts.getTenant())
+	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/native", opts)
 	uv := opts.asURLValues()
 	uvs := uv.Encode()
 	if len(uvs) > 0 {
@@ -190,7 +190,7 @@ func (app *Vminsert) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts
 func (app *Vminsert) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOpts) {
 	t.Helper()
 
-	url := fmt.Sprintf("http://%s/insert/%s/opentsdb/api/put", app.openTSDBListenAddr, opts.getTenant())
+	url := getClusterPath(app.openTSDBListenAddr, "insert", "opentsdb/api/put", opts)
 	uv := opts.asURLValues()
 	uvs := uv.Encode()
 	if len(uvs) > 0 {
@@ -213,7 +213,7 @@ func (app *Vminsert) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOp
 func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
 	t.Helper()
 
-	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/write", app.httpListenAddr, opts.getTenant())
+	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/write", opts)
 	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
 	recordsCount := len(wr.Timeseries)
 	if prommetadata.IsEnabled() {
@@ -238,7 +238,7 @@ func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest,
 func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
 	t.Helper()
 
-	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/prometheus", app.httpListenAddr, opts.getTenant())
+	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/prometheus", opts)
 	uv := opts.asURLValues()
 	uvs := uv.Encode()
 	if len(uvs) > 0 {
@@ -287,7 +287,7 @@ func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []str
 func (app *Vminsert) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
 	t.Helper()
 
-	url := fmt.Sprintf("http://%s/insert/%s/zabbixconnector/api/v1/history", app.httpListenAddr, opts.getTenant())
+	url := getClusterPath(app.httpListenAddr, "insert", "zabbixconnector/api/v1/history", opts)
 	uv := opts.asURLValues()
 	uvs := uv.Encode()
 	if len(uvs) > 0 {

apptest/vmselect.go

@@ -72,7 +72,7 @@ func (app *Vmselect) HTTPAddr() string {
 func (app *Vmselect) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
 	t.Helper()
 
-	exportURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/export", app.httpListenAddr, opts.getTenant())
+	exportURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/export", opts)
 	values := opts.asURLValues()
 	values.Add("match[]", query)
 	values.Add("format", "promapi")
@@ -88,7 +88,7 @@ func (app *Vmselect) PrometheusAPIV1Export(t *testing.T, query string, opts Quer
 func (app *Vmselect) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
 	t.Helper()
 
-	exportURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/export/native", app.httpListenAddr, opts.getTenant())
+	exportURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/export/native", opts)
 	values := opts.asURLValues()
 	values.Add("match[]", query)
 	values.Add("format", "promapi")
@@ -104,7 +104,7 @@ func (app *Vmselect) PrometheusAPIV1ExportNative(t *testing.T, query string, opt
 func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
 	t.Helper()
 
-	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/query", app.httpListenAddr, opts.getTenant())
+	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/query", opts)
 	values := opts.asURLValues()
 	values.Add("query", query)
 
@@ -120,7 +120,7 @@ func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query string, opts Query
 func (app *Vmselect) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
 	t.Helper()
 
-	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/query_range", app.httpListenAddr, opts.getTenant())
+	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/query_range", opts)
 	values := opts.asURLValues()
 	values.Add("query", query)
 
@@ -135,7 +135,7 @@ func (app *Vmselect) PrometheusAPIV1QueryRange(t *testing.T, query string, opts
 func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
 	t.Helper()
 
-	seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/series", app.httpListenAddr, opts.getTenant())
+	seriesURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/series", opts)
 	values := opts.asURLValues()
 	values.Add("match[]", matchQuery)
 
@@ -150,7 +150,7 @@ func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts
 func (app *Vmselect) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts) *PrometheusAPIV1SeriesCountResponse {
 	t.Helper()
 
-	seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/series/count", app.httpListenAddr, opts.getTenant())
+	seriesURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/series/count", opts)
 	values := opts.asURLValues()
 
 	res, _ := app.cli.PostForm(t, seriesURL, values, opts.Headers)
@@ -166,8 +166,8 @@ func (app *Vmselect) PrometheusAPIV1Labels(t *testing.T, matchQuery string, opts
 
 	values := opts.asURLValues()
 	values.Add("match[]", matchQuery)
+	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/labels", opts)
 
-	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/labels", app.httpListenAddr, opts.getTenant())
 	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
 	return NewPrometheusAPIV1LabelsResponse(t, res)
 }
@@ -181,7 +181,8 @@ func (app *Vmselect) PrometheusAPIV1LabelValues(t *testing.T, labelName, matchQu
 
 	values := opts.asURLValues()
 	values.Add("match[]", matchQuery)
-	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/label/%s/values", app.httpListenAddr, opts.getTenant(), labelName)
+	suffix := fmt.Sprintf("prometheus/api/v1/label/%s/values", labelName)
+	queryURL := getClusterPath(app.httpListenAddr, "select", suffix, opts)
 
 	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
 	return NewPrometheusAPIV1LabelValuesResponse(t, res)
@@ -195,7 +196,7 @@ func (app *Vmselect) PrometheusAPIV1Metadata(t *testing.T, metric string, limit
 	values := opts.asURLValues()
 	values.Add("metric", metric)
 	values.Add("limit", strconv.Itoa(limit))
-	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/metadata", app.httpListenAddr, opts.getTenant())
+	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/metadata", opts)
 
 	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
 	return NewPrometheusAPIV1Metadata(t, res)
@@ -208,7 +209,7 @@ func (app *Vmselect) PrometheusAPIV1Metadata(t *testing.T, metric string, limit
 func (app *Vmselect) APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) {
 	t.Helper()
 
-	queryURL := fmt.Sprintf("http://%s/delete/%s/prometheus/api/v1/admin/tsdb/delete_series", app.httpListenAddr, opts.getTenant())
+	queryURL := getClusterPath(app.httpListenAddr, "delete", "prometheus/api/v1/admin/tsdb/delete_series", opts)
 	values := opts.asURLValues()
 	values.Add("match[]", matchQuery)
 
@@ -229,7 +230,7 @@ func (app *Vmselect) MetricNamesStats(t *testing.T, limit, le, matchPattern stri
 	values.Add("limit", limit)
 	values.Add("le", le)
 	values.Add("match_pattern", matchPattern)
-	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/status/metric_names_stats", app.httpListenAddr, opts.getTenant())
+	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/status/metric_names_stats", opts)
 
 	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
 	if statusCode != http.StatusOK {
@@ -263,7 +264,7 @@ func (app *Vmselect) MetricNamesStatsReset(t *testing.T, opts QueryOpts) {
 func (app *Vmselect) APIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
 	t.Helper()
 
-	seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/status/tsdb", app.httpListenAddr, opts.getTenant())
+	url := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/status/tsdb", opts)
 	values := opts.asURLValues()
 	addNonEmpty := func(name, value string) {
 		if len(value) == 0 {
@@ -275,7 +276,7 @@ func (app *Vmselect) APIV1StatusTSDB(t *testing.T, matchQuery string, date strin
 	addNonEmpty("topN", topN)
 	addNonEmpty("date", date)
 
-	res, statusCode := app.cli.PostForm(t, seriesURL, values, opts.Headers)
+	res, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
 	}
@@ -294,8 +295,8 @@ func (app *Vmselect) APIV1StatusTSDB(t *testing.T, matchQuery string, date strin
 func (app *Vmselect) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse {
 	t.Helper()
 
-	seriesURL := fmt.Sprintf("http://%s/select/%s/graphite/metrics/index.json", app.httpListenAddr, opts.getTenant())
-	res, statusCode := app.cli.Get(t, seriesURL, opts.Headers)
+	url := getClusterPath(app.httpListenAddr, "select", "graphite/metrics/index.json", opts)
+	res, statusCode := app.cli.Get(t, url, opts.Headers)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
 	}
@@ -313,7 +314,7 @@ func (app *Vmselect) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) Graphite
 func (app *Vmselect) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
 	t.Helper()
 
-	url := fmt.Sprintf("http://%s/select/%s/graphite/tags/tagSeries", app.httpListenAddr, opts.getTenant())
+	url := getClusterPath(app.httpListenAddr, "select", "graphite/tags/tagSeries", opts)
 	values := opts.asURLValues()
 	values.Add("path", record)
 
@@ -326,7 +327,7 @@ func (app *Vmselect) GraphiteTagsTagSeries(t *testing.T, record string, opts Que
 func (app *Vmselect) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
 	t.Helper()
 
-	url := fmt.Sprintf("http://%s/select/%s/graphite/tags/tagMultiSeries", app.httpListenAddr, opts.getTenant())
+	url := getClusterPath(app.httpListenAddr, "select", "graphite/tags/tagMultiSeries", opts)
 	values := opts.asURLValues()
 	for _, rec := range records {
 		values.Add("path", rec)

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -68,8 +68,8 @@ The UI allows exploring query results via graphs and tables. See more details ab
 ## Multitenancy
 
 VictoriaMetrics cluster supports multiple isolated tenants (aka namespaces).
-Tenants are identified by `accountID` or `accountID:projectID`, which are put inside request URLs for writes and reads.
-See [these docs](#url-format) for details.
+Tenants are identified by `accountID` or `accountID:projectID` inside request URLs or HTTP headers{{% available_from "#" %}}
+for writes and reads. See [these docs](#url-format) for details.
 
 Some facts about tenants in VictoriaMetrics:
 
@@ -84,22 +84,59 @@ or [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/). [Co
 - Data for all the tenants is evenly spread among available `vmstorage` nodes. This guarantees even load among `vmstorage` nodes
 when different tenants have different amounts of data and different query load.
 
-- The database performance and resource usage doesn't depend on the number of tenants. It depends mostly on the total number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) in all the tenants. A time series is considered active if it received at least a single sample during the last hour.
+- The database performance and resource usage do not depend on the number of tenants. It depends mostly on the total number of 
+  [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) in all the tenants. 
 
 - The list of registered tenants can be obtained via `http://<vmselect>:8481/admin/tenants` url. See [these docs](#url-format).
 
 - VictoriaMetrics exposes various per-tenant statistics via metrics - see [these docs](https://docs.victoriametrics.com/victoriametrics/pertenantstatistic/).
 
-See also [multitenancy via labels](#multitenancy-via-labels).
+See also multitenancy [via headers](#multitenancy-via-headers) and [via labels](#multitenancy-via-labels).
+
+### Multitenancy via headers
+
+By default, VictoriaMetrics allows specifying `accountID` and `projectID` only in the request URL.
+
+Set `--enableMultitenancyViaHeaders` {{% available_from "#" %}} command-line flag to support 
+specifying `accountID` and `projectID` via HTTP headers `AccountID` and `ProjectID` respectively.
+This flag needs to be specified separately for vminserts and vmselects.
+
+When `--enableMultitenancyViaHeaders` is enabled, [URL format](#url-format) can be simplified to the following:
+- `http://<vminsert>:8480/insert/<suffix>` for writes
+- `http://<vmselect>:8481/select/prometheus/<suffix>` for reads
+
+For example, the following query will only select metric `up` from `accountID=2` and `projectID=3`:
+```
+curl 'https://<vmselect>:8481/select/prometheus/api/v1/query' \
+  -d 'query=up' \
+  --header "AccountID: 2" \
+  --header "ProjectID: 3"
+```
+
+The following example will ingest metric `up{instance="foo"}` to `accountID=2` and `projectID=0`:
+```
+curl --header "AccountID: 2" -d 'up{instance="foo"} 123' -X POST https://<vminsert>:8480/insert/prometheus/api/v1/import/prometheus
+```
+
+> When simplified path `/(insert|select)/<suffix>` is used and headers `AccountID`, `ProjectID` are missing, then IDs are set to `0:0` as default.
+> If tenant IDs are specified in URL, then headers are ignored.
+
+The `AccountID` header can be set to `multitenant` string: `AccountID: multitenant`. See more in [multitenancy via labels](#multitenancy-via-labels).
 
 ### Multitenancy via labels
 
-**Writes:**
+Multitenancy via labels allows specifying [tenants](#multitenancy) as labels `vm_account_id` and `vm_project_id` during
+ingestion or querying. This feature allows [ingesting workload with mixed tenants](#multitenant-writes) and [querying
+data from multiple tenants](#multitenant-reads) via the same URL.
 
-`vminsert` can accept data from multiple [tenants](#multitenancy) via a special `multitenant` endpoints `http://vminsert:8480/insert/multitenant/<suffix>`,
+#### Multitenant writes
+
+`vminsert` can accept data from multiple [tenants](#multitenancy) via special `multitenant` endpoints:
+ - `http://vminsert:8480/insert/multitenant/<suffix>`
+ - `http://vminsert:8480/insert/<suffix>` and HTTP header `AccountID: multitenant`. See how to enable headers [here](#multitenancy-via-headers).
 where `<suffix>` can be replaced with any supported suffix for data ingestion from [this list](#url-format).
-In this case the account ID and project ID are obtained from optional `vm_account_id` and `vm_project_id` labels of the incoming samples.
-If `vm_account_id` or `vm_project_id` labels are missing or invalid, then the corresponding account ID and project ID are set to 0.
+The `accountID` and `projectID` are obtained from optional `vm_account_id` and `vm_project_id` [labels](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#labels) of the incoming samples.
+If `vm_account_id` or `vm_project_id` labels are missing or invalid, then the corresponding `accountID` and `projectID` are set to 0.
 These labels are automatically removed from samples before forwarding them to `vmstorage`.
 For example, if the following samples are written into `http://vminsert:8480/insert/multitenant/prometheus/api/v1/write`:
 
@@ -119,11 +156,14 @@ such as [Graphite](https://docs.victoriametrics.com/victoriametrics/integrations
 [InfluxDB line protocol via TCP and UDP](https://docs.victoriametrics.com/victoriametrics/integrations/influxdb/) and
 [OpenTSDB telnet put protocol](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-telnet).
 
-**Reads:**
+#### Multitenant reads
 
 _For better performance prefer specifying [tenants in read URL](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format)._
 
-`vmselect` can execute {{% available_from "v1.104.0" %}} queries over multiple [tenants](#multitenancy) via special `multitenant` endpoints `http://vmselect:8481/select/multitenant/<suffix>`.
+`vmselect` can execute {{% available_from "v1.104.0" %}} queries over multiple [tenants](#multitenancy) via special `multitenant` endpoints:
+ - `http://vmselect:8481/select/multitenant/<suffix>`
+ - `http://vmselect:8481/select/<suffix>` and HTTP header `AccountID: multitenant`. See how to enable headers [here](#multitenancy-via-headers).
+
 Currently supported endpoints for `<suffix>` are:
 
 - `/prometheus/api/v1/query`
@@ -169,7 +209,7 @@ The precedence for applying filters for tenants follows this order:
  These filters have the highest priority and are applied first when provided through the query arguments.
 2. Filter tenants from labels selectors defined at metricsQL query expression.
 
-**Security considerations**
+> **Security considerations**
 It is recommended restricting access to `multitenant` endpoints only to trusted sources,
 since untrusted source may break per-tenant data by writing unwanted samples or get access to data of arbitrary tenants.
 
@@ -585,13 +625,13 @@ The metric is set to `0` when the `vmstorage` isn't in read-only mode.
 The main differences between URL formats of cluster and [Single server](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/)
 versions are that cluster has separate components for read and ingestion path, and because of multi-tenancy support.
 Also in the cluster version the `/prometheus/api/v1` endpoint ingests  `jsonl`, `csv`, `native` and `prometheus` data formats **not** only `prometheus` data.
-Check practical examples of [VictoriaMetrics API](https://docs.victoriametrics.com/victoriametrics/url-examples/).
+
+> Check practical examples of [VictoriaMetrics API](https://docs.victoriametrics.com/victoriametrics/url-examples/).
 
 - URLs for data ingestion: `http://<vminsert>:8480/insert/<accountID>/<suffix>`, where:
   - `<accountID>` is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). It is possible to set it as `accountID:projectID`,
-    where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. See [multitenancy docs](#multitenancy) for more details.
-    The `<accountID>` can be set to `multitenant` string, e.g. `http://<vminsert>:8480/insert/multitenant/<suffix>`. Such urls accept data from multiple tenants
-    specified via `vm_account_id` and `vm_project_id` labels. See [multitenancy via labels](#multitenancy-via-labels) for more details.
+    where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. See [multitenancy docs](#multitenancy) for more details 
+    about managing tenants, specifying tenant IDs via HTTP headers or labels.
   - `<suffix>` may have the following values:
     - `prometheus` and `prometheus/api/v1/write` - for ingesting data with [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
     - `prometheus/api/v1/import` - for importing data obtained via `api/v1/export` at `vmselect` (see below), JSON line format.
@@ -607,9 +647,8 @@ Check practical examples of [VictoriaMetrics API](https://docs.victoriametrics.c
     - `opentsdb/api/put` - for accepting [OpenTSDB HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html). This handler is disabled by default. It is exposed on a distinct TCP address set via `-opentsdbHTTPListenAddr` command-line flag. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http) for details.
 
 - URLs for [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/): `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:
-  - `<accountID>` is an arbitrary number identifying data namespace for the query (aka tenant). It is possible to set it as `accountID:projectID`,
-  where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. See [multitenancy docs](#multitenancy) for more details.
-  The `<accountID>` can be set to `multitenant` string, e.g. `http://<vmselect>:8481/select/multitenant/<suffix>` for querying over multiple tenants (see the full list of [supported multitenant read endpoints](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)).
+  - `<accountID>` is an arbitrary number identifying data namespace for the query (aka tenant). See [multitenancy docs](#multitenancy) for more details
+    about managing tenants, specifying tenant IDs via HTTP headers or labels.
   - `<suffix>` may have the following values:
     - `api/v1/query` - performs [PromQL instant query](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query).
     - `api/v1/query_range` - performs [PromQL range query](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query).
@@ -629,7 +668,8 @@ Check practical examples of [VictoriaMetrics API](https://docs.victoriametrics.c
     - `metric-relabel-debug` - for debugging [relabeling rules](https://docs.victoriametrics.com/victoriametrics/relabeling/).
 
 - URLs for [Graphite Metrics API](https://graphite-api.readthedocs.io/en/latest/api.html#the-metrics-api): `http://<vmselect>:8481/select/<accountID>/graphite/<suffix>`, where:
-  - `<accountID>` is an arbitrary number identifying data namespace for query (aka tenant)
+  - `<accountID>` is an arbitrary number identifying data namespace for query (aka tenant).
+    See [multitenancy docs](#multitenancy) for more details about managing tenants, specifying tenant IDs via HTTP headers or labels.
   - `<suffix>` may have the following values:
     - `render` - implements Graphite Render API. See [these docs](https://graphite.readthedocs.io/en/stable/render_api.html).
     - `metrics/find` - searches Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find).

docs/victoriametrics/vmagent.md

@@ -518,11 +518,14 @@ scrape_configs:
 
 `vmagent` can accept data via the same multitenant endpoints (`/insert/<accountID>/<suffix>`) as `vminsert` at [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/)
 does according to [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format) if `-enableMultitenantHandlers` command-line flag is set.
-In this case, vmagent automatically converts tenant identifiers from the URL to `vm_account_id` and `vm_project_id` labels and sets tenant info in metadata.
+In this case, vmagent automatically converts tenant identifiers from the URL or headers to `vm_account_id` and `vm_project_id` labels and sets tenant info in metadata.
 These tenant labels are added before applying [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/) specified via `-remoteWrite.relabelConfig`
 and `-remoteWrite.urlRelabelConfig` command-line flags. Metrics with `vm_account_id` and `vm_project_id` labels can be routed to the corresponding tenants
 when specifying `-remoteWrite.url` to [multitenant url at VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels).
 
+`vmagent` can accept tenant IDs specified via HTTP headers if both `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` command-line flags are set. 
+See more about [multitenancy via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers).
+
 ## Adding labels to metrics
 
 Extra labels can be added to metrics collected by `vmagent` via the following mechanisms:

docs/victoriametrics/vmagent_common_flags.md

@@ -34,6 +34,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Whether to check config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
   -enableMetadata
      Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true)
+  -enableMultitenancyViaHeaders
+     Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy
   -enableMultitenantHandlers
      Whether to process incoming data via multitenant insert handlers according to https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format . By default incoming data is processed via single-node insert handlers according to https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data .See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy for details
   -enableTCP6

docs/victoriametrics/vminsert_common_flags.md

@@ -38,6 +38,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      Whether to drop incoming samples if the destination vmstorage node is overloaded and/or unavailable. This prioritizes cluster availability over consistency, e.g. the cluster continues accepting all the ingested samples, but some of them may be dropped if vmstorage nodes are temporarily unavailable and/or overloaded. The drop of samples happens before the replication, so it's not recommended to use this flag with -replicationFactor enabled.
   -enableMetadata
      Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true)
+  -enableMultitenancyViaHeaders
+     Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
   -envflag.enable

docs/victoriametrics/vmselect_common_flags.md

@@ -41,6 +41,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      Flag value can be read from the given http/https url when using -deleteAuthKey=http://host/path or -deleteAuthKey=https://host/path
   -denyQueryTracing
      Whether to disable the ability to trace queries. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#query-tracing
+  -enableMultitenancyViaHeaders
+     Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
   -envflag.enable

lib/httpserver/path.go

@@ -2,20 +2,22 @@ package httpserver
 
 import (
 	"fmt"
+	"net/http"
+	"strconv"
 	"strings"
 )
 
 // Path contains the following path structure:
-// /{prefix}/{authToken}/{suffix}
-//
-// It is compatible with SaaS version.
+// - /{prefix}/{tenantID}/{suffix}
+// - /{prefix}/{suffix} -H "{tenantID}"
+// in `/{prefix}/{suffix}` format tenantID is extracted from HTTP headers
 type Path struct {
 	Prefix    string
 	AuthToken string
 	Suffix    string
 }
 
-// ParsePath parses the given path.
+// ParsePath parses the given path according to /{prefix}/{tenantID}/{suffix} format
 func ParsePath(path string) (*Path, error) {
 	// The path must have the following form:
 	// /{prefix}/{authToken}/{suffix}
@@ -56,6 +58,107 @@ func ParsePath(path string) (*Path, error) {
 	return p, nil
 }
 
+// ParsePathAndHeaders parses the given path and headers.
+//
+// The path may be one of the following forms:
+//
+//  1. /{prefix}/{tenantID}/{suffix} — tenantID is in the URL
+//  2. /{prefix}/{suffix} — tenantID is omitted and expected to be read from AccountID/ProjectID HTTP headers.
+//     If these headers are missing, tenantID is set to "0:0" to be consistent with VictoriaLogs behavior.
+//
+// prefix is "select", "insert", or "delete".
+// tenantID is "accountID[:projectID]" or "multitenant".
+// tenantID specified in path always takes priority over headers for backward compatibility.
+//
+// This function doesn't validate correctness of {tenantID} content.
+func ParsePathAndHeaders(path string, h http.Header) (*Path, error) {
+	s := skipPrefixSlashes(path)
+	n := strings.IndexByte(s, '/')
+	if n < 0 {
+		return nil, fmt.Errorf("cannot find {prefix} in %q; expecting /{prefix}/{suffix} or /{prefix}/{tenantID}/{suffix} format; "+
+			"see https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format", path)
+	}
+
+	prefix := s[:n]
+	tail := skipPrefixSlashes(s[n+1:])
+
+	if tail == "" {
+		return nil, fmt.Errorf("cannot find {suffix} in %q; expecting /{prefix}/{suffix} or /{prefix}/{tenantID}/{suffix} format; "+
+			"see https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format", path)
+	}
+
+	// Try to split tail into {tenantID}/{suffix} segments.
+	// If the first segment is a valid tenantID - consume it, ignore headers
+	// Otherwise, treat tail as {suffix} and read tenantID from HTTP headers.
+	var tenantID string
+	suffix := tail
+	n = strings.IndexByte(tail, '/')
+	if n >= 0 {
+		tenantID = tail[:n]
+	}
+	if maybeTenantID(tenantID) {
+		// cut the tenantID from suffix
+		suffix = skipPrefixSlashes(tail[n+1:])
+	} else {
+		// tenantID is not valid - assume tail is all suffix and tenantID is in headers
+		tenantID = tenantIDFromHeadersOrDefault(h, "0:0")
+	}
+
+	// Substitute double slashes with single slashes in the path, since such slashes
+	// may appear due to improper copy-pasting of the url.
+	suffix = strings.ReplaceAll(suffix, "//", "/")
+
+	return &Path{
+		Prefix:    prefix,
+		AuthToken: tenantID,
+		Suffix:    suffix,
+	}, nil
+}
+
+// maybeTenantID returns true if s is "multitenant", "<uint>" or contains ":" char.
+// It doesn't validate correctness of tenantID and is only used for quick routing.
+// It is expected that tenantID will be correctly validated later.
+func maybeTenantID(tenantID string) bool {
+	if tenantID == "" {
+		return false
+	}
+	if tenantID == "multitenant" {
+		return true
+	}
+
+	idx := strings.IndexByte(tenantID, ':')
+	if idx > 0 {
+		return true
+	}
+
+	_, err := strconv.ParseUint(tenantID, 10, 32)
+	return err == nil
+}
+
+// tenantIDFromHeaders reads AccountID and ProjectID header values from request.
+// If headers are missing, it returns defaultTenantID.
+func tenantIDFromHeadersOrDefault(h http.Header, defaultTenantID string) string {
+	aID := h.Get("AccountID")
+	pID := h.Get("ProjectID")
+	if len(aID) == 0 && len(pID) == 0 {
+		return defaultTenantID
+	}
+
+	if aID == "multitenant" {
+		// special case for multitenant
+		return "multitenant"
+	}
+
+	accountID, projectID := "0", "0"
+	if len(aID) > 0 {
+		accountID = aID
+	}
+	if len(pID) > 0 {
+		projectID = pID
+	}
+	return fmt.Sprintf("%s:%s", accountID, projectID)
+}
+
 // skipPrefixSlashes remove double slashes which may appear due
 // improper copy-pasting of the url
 func skipPrefixSlashes(s string) string {

lib/httpserver/path_test.go

@@ -0,0 +1,84 @@
+package httpserver
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestParsePathAndHeadersSuccess(t *testing.T) {
+	f := func(path, headers, prefix, authToken, suffix string) {
+		t.Helper()
+		header := make(http.Header)
+		hs := strings.Split(headers, ";")
+		for _, h := range hs {
+			if h == "" {
+				continue
+			}
+			parts := strings.Split(h, ":")
+			header.Set(parts[0], parts[1])
+		}
+		p, err := ParsePathAndHeaders(path, header)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if p.Prefix != prefix {
+			t.Fatalf("unexpected Prefix; got %q; want %q", p.Prefix, prefix)
+		}
+		if p.AuthToken != authToken {
+			t.Fatalf("unexpected AuthToken; got %q; want %q", p.AuthToken, authToken)
+		}
+		if p.Suffix != suffix {
+			t.Fatalf("unexpected Suffix; got %q; want %q", p.Suffix, suffix)
+		}
+	}
+
+	// tenant is omitted in the Path, so we try reading it from headers
+	f("/select/prometheus/api/v1/query", "AccountID:1;ProjectID:1", "select", "1:1", "prometheus/api/v1/query")
+	f("/select/prometheus/api/v1/query_range", "AccountID:1", "select", "1:0", "prometheus/api/v1/query_range")
+	f("/select/prometheus/api/v1/query_range", "ProjectID:1", "select", "0:1", "prometheus/api/v1/query_range")
+	f("/insert/prometheus", "", "insert", "0:0", "prometheus")
+	f("/insert/prometheus", "AccountID:1;ProjectID:1", "insert", "1:1", "prometheus")
+	f("/insert/prometheus", "AccountID:multitenant", "insert", "multitenant", "prometheus")
+	f("/delete/prometheus/api/v1/admin/tsdb/delete_series", "AccountID:1;ProjectID:1", "delete", "1:1", "prometheus/api/v1/admin/tsdb/delete_series")
+	f("/insert//prometheus/api/v1/import/prometheus", "AccountID:2", "insert", "2:0", "prometheus/api/v1/import/prometheus")
+
+	// If headers are empty, we assume 0:0 as default.
+	f("/insert/prometheus", "", "insert", "0:0", "prometheus")
+	f("/select/prometheus/api/v1/query", "", "select", "0:0", "prometheus/api/v1/query")
+
+	// tenant is present in the Path
+	f("/insert/123/prometheus/api/v1/write", "", "insert", "123", "prometheus/api/v1/write")
+	f("/select/1:15/prometheus/api/v1/query", "", "select", "1:15", "prometheus/api/v1/query")
+	f("/insert/multitenant/prometheus/api/v1/write", "", "insert", "multitenant", "prometheus/api/v1/write")
+	f("/insert/0/prometheus/api/v1/write", "", "insert", "0", "prometheus/api/v1/write")
+	f("/insert/0:0/prometheus/api/v1/write", "", "insert", "0:0", "prometheus/api/v1/write")
+	f("/delete/123/prometheus/api/v1/admin/tsdb/delete_series", "", "delete", "123", "prometheus/api/v1/admin/tsdb/delete_series")
+
+	// tenant in the Path takes priority over headers
+	f("/insert/123/prometheus/api/v1/write", "AccountID:1;ProjectID:1", "insert", "123", "prometheus/api/v1/write")
+	f("/insert/multitenant/prometheus/api/v1/write", "AccountID:1;ProjectID:1", "insert", "multitenant", "prometheus/api/v1/write")
+	f("/insert/123:1/prometheus/api/v1/write", "AccountID:multitenant", "insert", "123:1", "prometheus/api/v1/write")
+
+	// Double slashes in Path
+	f("//insert//123//prometheus//api/v1/write", "", "insert", "123", "prometheus/api/v1/write")
+	f("//insert//prometheus//api/v1/write", "AccountID:1;ProjectID:1", "insert", "1:1", "prometheus/api/v1/write")
+
+}
+
+func TestParsePathAndHeadersFailure(t *testing.T) {
+	f := func(path string) {
+		t.Helper()
+		p, err := ParsePathAndHeaders(path, nil)
+		if err == nil {
+			t.Fatalf("expecting non-nil error; got path %+v", p)
+		}
+	}
+
+	// No prefix or suffix
+	f("/")
+	// Only prefix, no suffix or tenant
+	f("/insert")
+	f("/select/")
+	f("/select//")
+}