diff --git a/app/vmagent/main.go b/app/vmagent/main.go index 1d6058bf52..2b827516d1 100644 --- a/app/vmagent/main.go +++ b/app/vmagent/main.go @@ -83,6 +83,9 @@ var ( maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 0, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented") maxLabelNameLen = flag.Int("maxLabelNameLen", 0, "The maximum length of label names in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented") maxLabelValueLen = flag.Int("maxLabelValueLen", 0, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented") + + enableMultitenancyViaHeaders = flag.Bool("enableMultitenancyViaHeaders", false, "Enables multitenancy via HTTP headers. "+ + "See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy") ) var ( @@ -216,7 +219,7 @@ func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error { } return func(req *http.Request) error { path := strings.ReplaceAll(req.URL.Path, "//", "/") - at, err := getAuthTokenFromPath(path) + at, err := getAuthTokenFromPath(path, req.Header) if err != nil { return fmt.Errorf("cannot obtain auth token from path %q: %w", path, err) } @@ -224,8 +227,15 @@ func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error { } } -func getAuthTokenFromPath(path string) (*auth.Token, error) { - p, err := httpserver.ParsePath(path) +func parsePath(path string, header http.Header) (*httpserver.Path, error) { + if *enableMultitenancyViaHeaders { + return httpserver.ParsePathAndHeaders(path, header) + } + return httpserver.ParsePath(path) +} + +func getAuthTokenFromPath(path string, header http.Header) (*auth.Token, error) { + p, err := parsePath(path, header) if err != nil { return nil, fmt.Errorf("cannot parse multitenant path: %w", err) } @@ -559,14 +569,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool { } func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path string) bool { - p, err := httpserver.ParsePath(path) + p, err := parsePath(path, r.Header) if err != nil { // Cannot parse multitenant path. Skip it - probably it will be parsed later. return false } if p.Prefix != "insert" { - httpserver.Errorf(w, r, `unsupported multitenant prefix: %q; expected "insert"`, p.Prefix) - return true + // processMultitenantRequest is called for all unmatched path variants, + // but we should try parsing only /insert prefixed to avoid catching all possible paths. + return false } at, err := auth.NewTokenPossibleMultitenant(p.AuthToken) if err != nil { diff --git a/apptest/client.go b/apptest/client.go index 9cb5c22231..83bee7735d 100644 --- a/apptest/client.go +++ b/apptest/client.go @@ -2,6 +2,7 @@ package apptest import ( "bytes" + "fmt" "io" "net" "net/http" @@ -105,6 +106,35 @@ func (c *Client) Write(t *testing.T, address string, data []string) { } } +// getClusterPath returns path in cluster's URL format. +// Based on QueryOpts, it will either put tenant ID into URL +// or will skip it if tenant is set via HTTP headers. +func getClusterPath(addr, prefix, suffix string, o QueryOpts) string { + if o.Tenant != "" { + // QueryOpts.Tenant has priority over headers + return tenantViaURL(addr, prefix, o.Tenant, suffix) + } + + h := o.getHeaders() + if h.Get("AccountID") != "" || h.Get("ProjectID") != "" { + return tenantViaHeaders(addr, prefix, suffix) + } + + // tenant is missing in QueryOpts and in HTTP headers. Falling back to default 0:0 tenant in URL + return tenantViaURL(addr, prefix, "0:0", suffix) +} + +// tenantViaURL returns path in cluster's URL format with tenant specified in URL +func tenantViaURL(addr, prefix, tenant, suffix string) string { + return fmt.Sprintf("http://%s/%s/%s/%s", addr, prefix, tenant, suffix) +} + +// tenantViaHeaders returns path in cluster's URL format where tenant is omitted in URL +// Only supported if -enableMultitenancyViaHeaders is specified +func tenantViaHeaders(addr, prefix, suffix string) string { + return fmt.Sprintf("http://%s/%s/%s", addr, prefix, suffix) +} + // readAllAndClose reads everything from the response body and then closes it. func readAllAndClose(t *testing.T, responseBody io.ReadCloser) string { t.Helper() diff --git a/apptest/model.go b/apptest/model.go index dfb7750ed3..50f47baf01 100644 --- a/apptest/model.go +++ b/apptest/model.go @@ -93,14 +93,6 @@ type QueryOpts struct { Headers http.Header } -// getTenant returns tenant with optional default value -func (qos *QueryOpts) getTenant() string { - if qos.Tenant == "" { - return "0" - } - return qos.Tenant -} - func (qos *QueryOpts) getHeaders() http.Header { if qos.Headers == nil { qos.Headers = make(http.Header) diff --git a/apptest/tests/multitenancy_via_headers_test.go b/apptest/tests/multitenancy_via_headers_test.go new file mode 100644 index 0000000000..1adac332f0 --- /dev/null +++ b/apptest/tests/multitenancy_via_headers_test.go @@ -0,0 +1,313 @@ +package tests + +import ( + "net/http" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/VictoriaMetrics/VictoriaMetrics/apptest" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" +) + +func TestClusterMultiTenantSelectViaHeaders(t *testing.T) { + fs.MustRemoveDir(t.Name()) + + cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType") + cmpSROpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Status", "IsPartial") + + tc := apptest.NewTestCase(t) + defer tc.Stop() + vmstorage := tc.MustStartVmstorage("vmstorage", []string{ + "-storageDataPath=" + tc.Dir() + "/vmstorage", + "-retentionPeriod=100y", + }) + vminsert := tc.MustStartVminsert("vminsert", []string{ + "-storageNode=" + vmstorage.VminsertAddr(), + "-enableMultitenancyViaHeaders", + }) + vmselect := tc.MustStartVmselect("vmselect", []string{ + "-storageNode=" + vmstorage.VmselectAddr(), + "-search.tenantCacheExpireDuration=0", + "-enableMultitenancyViaHeaders", + }) + + multitenant := make(http.Header) + multitenant.Set("AccountID", "multitenant") + + // test for empty tenants request + got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{ + Headers: multitenant, + Step: "5m", + Time: "2022-05-10T08:03:00.000Z", + }) + want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`) + if diff := cmp.Diff(want, got, cmpOpt); diff != "" { + t.Errorf("unexpected response (-want, +got):\n%s", diff) + } + + // ingest per tenant data and verify it with search + samples := []string{ + `foo_bar 1.00 1652169600000`, // 2022-05-10T08:00:00Z + `foo_bar 2.00 1652169660000`, // 2022-05-10T08:01:00Z + `foo_bar 3.00 1652169720000`, // 2022-05-10T08:02:00Z + } + tenantHeaders := []map[string]string{ + {"AccountID": "1", "ProjectID": "1"}, + {"AccountID": "1", "ProjectID": "15"}, + {"AccountID": "2"}, + {"ProjectID": "3"}, + } + instantCT := "2022-05-10T08:05:00.000Z" // 1652169900 Unix seconds + for _, headers := range tenantHeaders { + h := make(http.Header) + for k, v := range headers { + h.Set(k, v) + } + vminsert.PrometheusAPIV1ImportPrometheus(t, samples, apptest.QueryOpts{Headers: h}) + vmstorage.ForceFlush(t) + + // verify tenants are searchable via tenantID in headers + got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{ + Headers: h, Time: instantCT, + }) + want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]}]}}`) + if diff := cmp.Diff(want, got, cmpOpt); diff != "" { + t.Errorf("unexpected response (-want, +got):\n%s", diff) + } + } + + // verify all tenants searchable with multitenant header + + // /api/v1/query + want = apptest.NewPrometheusAPIV1QueryResponse(t, + `{"data": + {"result":[ + {"metric":{"__name__":"foo_bar","vm_account_id":"0","vm_project_id":"3"},"value":[1652169900,"3"]}, + {"metric":{"__name__":"foo_bar","vm_account_id":"1","vm_project_id": "1"},"value":[1652169900,"3"]}, + {"metric":{"__name__":"foo_bar","vm_account_id":"1","vm_project_id":"15"},"value":[1652169900,"3"]}, + {"metric":{"__name__":"foo_bar","vm_account_id":"2","vm_project_id":"0"},"value":[1652169900,"3"]} + ] + } + }`, + ) + + got = vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{ + Headers: multitenant, + Time: instantCT, + }) + if diff := cmp.Diff(want, got, cmpOpt); diff != "" { + t.Errorf("unexpected response (-want, +got):\n%s", diff) + } + + // /api/v1/query_range aggregated by tenant labels + query := "sum(foo_bar) by(vm_account_id,vm_project_id)" + got = vmselect.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{ + Headers: multitenant, + Start: "2022-05-10T07:59:00.000Z", + End: "2022-05-10T08:05:00.000Z", + Step: "1m", + }) + + want = apptest.NewPrometheusAPIV1QueryResponse(t, + `{"data": + {"result": [ + {"metric": {"vm_account_id": "0","vm_project_id":"3"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]}, + {"metric": {"vm_account_id": "1","vm_project_id":"1"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]}, + {"metric": {"vm_account_id": "1","vm_project_id":"15"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]}, + {"metric": {"vm_account_id": "2","vm_project_id":"0"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]} + ] + } + }`) + if diff := cmp.Diff(want, got, cmpOpt); diff != "" { + t.Errorf("unexpected response (-want, +got):\n%s", diff) + } + + // verify /api/v1/series response + + wantSR := apptest.NewPrometheusAPIV1SeriesResponse(t, + `{"data": [ + {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"}, + {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"}, + {"__name__":"foo_bar", "vm_account_id":"2", "vm_project_id":"0"}, + {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"3"} + ] + }`) + wantSR.Sort() + + gotSR := vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{ + Headers: multitenant, + Start: "2022-05-10T08:03:00.000Z", + }) + gotSR.Sort() + if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" { + t.Errorf("unexpected response (-want, +got):\n%s", diff) + } + + // test ingestion with multitenant header, tenants must be populated from labels + // + var tenantLabelsSamples = []string{ + `foo_bar{vm_account_id="5"} 1.00 1652169720000`, // 2022-05-10T08:02:00Z' + `foo_bar{vm_project_id="10"} 2.00 1652169660000`, // 2022-05-10T08:01:00Z + `foo_bar{vm_account_id="5",vm_project_id="15"} 3.00 1652169720000`, // 2022-05-10T08:02:00Z + } + + vminsert.PrometheusAPIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Headers: multitenant}) + vmstorage.ForceFlush(t) + + // /api/v1/query with query filters + want = apptest.NewPrometheusAPIV1QueryResponse(t, + `{"data": + {"result":[ + {"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id": "0"},"value":[1652169900,"1"]}, + {"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id":"15"},"value":[1652169900,"3"]} + ] + } + }`, + ) + got = vmselect.PrometheusAPIV1Query(t, `foo_bar{vm_account_id="5"}`, apptest.QueryOpts{ + Time: instantCT, + Headers: multitenant, + }) + if diff := cmp.Diff(want, got, cmpOpt); diff != "" { + t.Errorf("unexpected response (-want, +got):\n%s", diff) + } + + // /api/v1/series with extra_filters + + wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t, + `{"data": [ + {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"15"}, + {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"} + ] + }`) + wantSR.Sort() + gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{ + Start: "2022-05-10T08:00:00.000Z", + End: "2022-05-10T08:30:00.000Z", + ExtraFilters: []string{`{vm_project_id="15"}`}, + Headers: multitenant, + }) + gotSR.Sort() + + if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" { + t.Errorf("unexpected response (-want, +got):\n%s", diff) + } + + // /api/v1/label/../value with extra_filters + + wantVR := apptest.NewPrometheusAPIV1LabelValuesResponse(t, + `{"data": [ + "5" + ] + }`) + // matchQuery is ignored for /api/v1/label//values lookups with multitenant token + gotVR := vmselect.PrometheusAPIV1LabelValues(t, "vm_account_id", "xxx", apptest.QueryOpts{ + Start: "2022-05-10T08:00:00.000Z", + End: "2022-05-10T08:30:00.000Z", + ExtraFilters: []string{`{vm_account_id="5"}`}, + Headers: multitenant, + }) + gotSR.Sort() + + if diff := cmp.Diff(wantVR, gotVR, cmpopts.IgnoreFields(apptest.PrometheusAPIV1LabelValuesResponse{}, "Status", "IsPartial")); diff != "" { + t.Errorf("unexpected response (-want, +got):\n%s", diff) + } + + // Delete series from specific tenant + tenantID := make(http.Header) + tenantID.Set("AccountID", "5") + tenantID.Set("ProjectID", "15") + vmselect.APIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{ + Headers: tenantID, + }) + wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t, + `{"data": [ + {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"3"}, + {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"}, + {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"}, + {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"}, + {"__name__":"foo_bar", "vm_account_id":"2", "vm_project_id":"0"}, + {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"} + ] + }`) + wantSR.Sort() + + gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{ + Headers: multitenant, + Start: "2022-05-10T08:03:00.000Z", + }) + gotSR.Sort() + if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" { + t.Errorf("unexpected response (-want, +got):\n%s", diff) + } + + // Delete series for multitenant with tenant filter + vmselect.APIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{ + Headers: multitenant, + }) + + wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t, + `{"data": [ + {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"3"}, + {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"}, + {"__name__":"foo_bar", "vm_account_id":"2", "vm_project_id":"0"}, + {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"} + ] + }`) + wantSR.Sort() + + gotSR = vmselect.PrometheusAPIV1Series(t, `foo_bar`, apptest.QueryOpts{ + Headers: multitenant, + Start: "2022-05-10T08:03:00.000Z", + }) + gotSR.Sort() + if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" { + t.Errorf("unexpected response (-want, +got):\n%s", diff) + } + + if got := vmselect.GetIntMetric(t, `vm_cache_requests_total{type="multitenancy/tenants"}`); got != 0 { + t.Errorf("unexpected multitenancy tenants cache requests; got %d; want 0", got) + } + + if got := vmselect.GetIntMetric(t, `vm_cache_misses_total{type="multitenancy/tenants"}`); got != 0 { + t.Errorf("unexpected multitenancy tenants cache misses; got %d; want 0", got) + } + + if got := vmselect.GetIntMetric(t, `vm_cache_entries{type="multitenancy/tenants"}`); got != 0 { + t.Errorf("unexpected multitenancy tenants cache entries; got %d; want 0", got) + } + + // verify that tenant in path has priority over tenant specified in headers + + // /api/v1/import/prometheus + + tenantInHeader := make(http.Header) + tenantInHeader.Set("AccountID", "42") + tenantInPath := "112" + vminsert.PrometheusAPIV1ImportPrometheus(t, samples, apptest.QueryOpts{ + // tenants in header and path clash - path should have higher priority on ingestion + Headers: tenantInHeader, + Tenant: "112", + }) + vmstorage.ForceFlush(t) + + want = apptest.NewPrometheusAPIV1QueryResponse(t, + `{"data": + {"result":[ + {"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]} + ] + } + }`, + ) + got = vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{ + // tenants in header and path clash - path should have higher priority on ingestion + Headers: multitenant, + Tenant: tenantInPath, + Time: instantCT, + }) + if diff := cmp.Diff(want, got, cmpOpt); diff != "" { + t.Errorf("unexpected response (-want, +got):\n%s", diff) + } +} diff --git a/apptest/tests/vmagent_remotewrite_test.go b/apptest/tests/vmagent_remotewrite_test.go index 85ed2342f5..e478fc60ee 100644 --- a/apptest/tests/vmagent_remotewrite_test.go +++ b/apptest/tests/vmagent_remotewrite_test.go @@ -584,3 +584,69 @@ func TestClusterVMAgentForwardMetricsMetadata(t *testing.T) { }) } + +// See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy +func TestSingleVMAgentMultitenancy(t *testing.T) { + tc := apptest.NewTestCase(t) + defer tc.Stop() + + remoteWriteSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer remoteWriteSrv.Close() + + vmagent := tc.MustStartVmagent("vmagent-multitenancy", []string{ + fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL), + "-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-multitenancy", + "-enableMultitenantHandlers", + "-enableMultitenancyViaHeaders", + }, ``) + + vmagent.APIV1ImportPrometheus(t, []string{ + "foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z + }, apptest.QueryOpts{Tenant: "2"}) + v := vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="2",projectID="0"}`) + if v != 1 { + t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=2") + } + + vmagent.APIV1ImportPrometheus(t, []string{ + "foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z + }, apptest.QueryOpts{Tenant: "2:2"}) + v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="2",projectID="2"}`) + if v != 1 { + t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=2, projectID=2") + } + + headers := make(http.Header) + headers.Set("AccountID", "3") + vmagent.APIV1ImportPrometheus(t, []string{ + "foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z + }, apptest.QueryOpts{Headers: headers}) + v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="3",projectID="0"}`) + if v != 1 { + t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=3, projectID=0") + } + + headers.Set("AccountID", "3") + headers.Set("ProjectID", "3") + vmagent.APIV1ImportPrometheus(t, []string{ + "foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z + }, apptest.QueryOpts{Headers: headers}) + v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="3",projectID="3"}`) + if v != 1 { + t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=3, projectID=3") + } + + // tenants in header and path clash - path should have higher priority on ingestion + opts := apptest.QueryOpts{Headers: make(http.Header)} + opts.Headers.Set("AccountID", "4") + opts.Tenant = "5" + vmagent.APIV1ImportPrometheus(t, []string{ + "foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z + }, opts) + v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="5",projectID="0"}`) + if v != 1 { + t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=5, projectID=0") + } +} diff --git a/apptest/vmagent.go b/apptest/vmagent.go index db15042917..55d441b99e 100644 --- a/apptest/vmagent.go +++ b/apptest/vmagent.go @@ -21,8 +21,7 @@ type Vmagent struct { *app *ServesMetrics - httpListenAddr string - apiV1ImportPrometheusURL string + httpListenAddr string } // StartVmagent starts an instance of vmagent with the given flags. It also @@ -52,8 +51,7 @@ func StartVmagent(instance string, flags []string, cli *Client, promScrapeConfig metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]), cli: cli, }, - httpListenAddr: stderrExtracts[0], - apiV1ImportPrometheusURL: fmt.Sprintf("http://%s/api/v1/import/prometheus", stderrExtracts[0]), + httpListenAddr: stderrExtracts[0], }, nil } @@ -86,12 +84,33 @@ func (app *Vmagent) APIV1ImportPrometheusNoWaitFlush(t *testing.T, records []str data := []byte(strings.Join(records, "\n")) headers := opts.getHeaders() headers.Set("Content-Type", "text/plain") - _, statusCode := app.cli.Post(t, app.apiV1ImportPrometheusURL, data, headers) + url := getVMAgentInsertPath(app.httpListenAddr, "prometheus/api/v1/import/prometheus", opts) + _, statusCode := app.cli.Post(t, url, data, headers) if statusCode != http.StatusNoContent { t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent) } } +// getVMAgentInsertPath returns URL path for writes. +// If tenant is set in QueryOpts, it will return cluster-like path for ingestion. +// If tenant is empty, it will return single-node (no tenants) path. +func getVMAgentInsertPath(addr, suffix string, o QueryOpts) string { + if o.Tenant != "" { + // QueryOpts.Tenant has priority over headers + return fmt.Sprintf("http://%s/insert/%s/%s", addr, o.Tenant, suffix) + } + + h := o.getHeaders() + if h.Get("AccountID") != "" || h.Get("ProjectID") != "" { + // vmagent supports tenantID in HTTP headers only if -enableMultitenantHandlers and -enableMultitenancyViaHeaders are set + // see https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy + return fmt.Sprintf("http://%s/insert/%s", addr, suffix) + } + + // tenant is missing in QueryOpts and in HTTP headers. Use single-node (no tenants) path + return fmt.Sprintf("http://%s/%s", addr, suffix) +} + // RemoteWriteRequestsRetriesCountTotal sums up the total retries for remote write requests. func (app *Vmagent) RemoteWriteRequestsRetriesCountTotal(t *testing.T) int { total := 0.0 @@ -168,10 +187,7 @@ func (app *Vmagent) ReloadRelabelConfigs(t *testing.T) { func (app *Vmagent) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) { t.Helper() - url := fmt.Sprintf("http://%s/prometheus/api/v1/write", app.httpListenAddr) - if opts.Tenant != "" { - url = fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/write", app.httpListenAddr, opts.Tenant) - } + url := getVMAgentInsertPath(app.httpListenAddr, "prometheus/api/v1/write", opts) data := snappy.Encode(nil, wr.MarshalProtobuf(nil)) recordsCount := len(wr.Timeseries) if prommetadata.IsEnabled() { diff --git a/apptest/vminsert.go b/apptest/vminsert.go index 3818258b97..52f2cb9520 100644 --- a/apptest/vminsert.go +++ b/apptest/vminsert.go @@ -106,7 +106,7 @@ func (app *Vminsert) HTTPAddr() string { func (app *Vminsert) InfluxWrite(t *testing.T, records []string, opts QueryOpts) { t.Helper() - url := fmt.Sprintf("http://%s/insert/%s/influx/write", app.httpListenAddr, opts.getTenant()) + url := getClusterPath(app.httpListenAddr, "insert", "influx/write", opts) uv := opts.asURLValues() uvs := uv.Encode() if len(uvs) > 0 { @@ -141,7 +141,7 @@ func (app *Vminsert) GraphiteWrite(t *testing.T, records []string, _ QueryOpts) func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) { t.Helper() - url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/csv", app.httpListenAddr, opts.getTenant()) + url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/csv", opts) uv := opts.asURLValues() uvs := uv.Encode() if len(uvs) > 0 { @@ -166,7 +166,7 @@ func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, op func (app *Vminsert) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) { t.Helper() - url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/native", app.httpListenAddr, opts.getTenant()) + url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/native", opts) uv := opts.asURLValues() uvs := uv.Encode() if len(uvs) > 0 { @@ -190,7 +190,7 @@ func (app *Vminsert) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts func (app *Vminsert) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOpts) { t.Helper() - url := fmt.Sprintf("http://%s/insert/%s/opentsdb/api/put", app.openTSDBListenAddr, opts.getTenant()) + url := getClusterPath(app.openTSDBListenAddr, "insert", "opentsdb/api/put", opts) uv := opts.asURLValues() uvs := uv.Encode() if len(uvs) > 0 { @@ -213,7 +213,7 @@ func (app *Vminsert) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOp func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) { t.Helper() - url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/write", app.httpListenAddr, opts.getTenant()) + url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/write", opts) data := snappy.Encode(nil, wr.MarshalProtobuf(nil)) recordsCount := len(wr.Timeseries) if prommetadata.IsEnabled() { @@ -238,7 +238,7 @@ func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) { t.Helper() - url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/prometheus", app.httpListenAddr, opts.getTenant()) + url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/prometheus", opts) uv := opts.asURLValues() uvs := uv.Encode() if len(uvs) > 0 { @@ -287,7 +287,7 @@ func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []str func (app *Vminsert) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) { t.Helper() - url := fmt.Sprintf("http://%s/insert/%s/zabbixconnector/api/v1/history", app.httpListenAddr, opts.getTenant()) + url := getClusterPath(app.httpListenAddr, "insert", "zabbixconnector/api/v1/history", opts) uv := opts.asURLValues() uvs := uv.Encode() if len(uvs) > 0 { diff --git a/apptest/vmselect.go b/apptest/vmselect.go index 04c69c3ee9..cf15292e8d 100644 --- a/apptest/vmselect.go +++ b/apptest/vmselect.go @@ -72,7 +72,7 @@ func (app *Vmselect) HTTPAddr() string { func (app *Vmselect) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse { t.Helper() - exportURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/export", app.httpListenAddr, opts.getTenant()) + exportURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/export", opts) values := opts.asURLValues() values.Add("match[]", query) values.Add("format", "promapi") @@ -88,7 +88,7 @@ func (app *Vmselect) PrometheusAPIV1Export(t *testing.T, query string, opts Quer func (app *Vmselect) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte { t.Helper() - exportURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/export/native", app.httpListenAddr, opts.getTenant()) + exportURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/export/native", opts) values := opts.asURLValues() values.Add("match[]", query) values.Add("format", "promapi") @@ -104,7 +104,7 @@ func (app *Vmselect) PrometheusAPIV1ExportNative(t *testing.T, query string, opt func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse { t.Helper() - queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/query", app.httpListenAddr, opts.getTenant()) + queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/query", opts) values := opts.asURLValues() values.Add("query", query) @@ -120,7 +120,7 @@ func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query string, opts Query func (app *Vmselect) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse { t.Helper() - queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/query_range", app.httpListenAddr, opts.getTenant()) + queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/query_range", opts) values := opts.asURLValues() values.Add("query", query) @@ -135,7 +135,7 @@ func (app *Vmselect) PrometheusAPIV1QueryRange(t *testing.T, query string, opts func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse { t.Helper() - seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/series", app.httpListenAddr, opts.getTenant()) + seriesURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/series", opts) values := opts.asURLValues() values.Add("match[]", matchQuery) @@ -150,7 +150,7 @@ func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts func (app *Vmselect) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts) *PrometheusAPIV1SeriesCountResponse { t.Helper() - seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/series/count", app.httpListenAddr, opts.getTenant()) + seriesURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/series/count", opts) values := opts.asURLValues() res, _ := app.cli.PostForm(t, seriesURL, values, opts.Headers) @@ -166,8 +166,8 @@ func (app *Vmselect) PrometheusAPIV1Labels(t *testing.T, matchQuery string, opts values := opts.asURLValues() values.Add("match[]", matchQuery) + queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/labels", opts) - queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/labels", app.httpListenAddr, opts.getTenant()) res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers) return NewPrometheusAPIV1LabelsResponse(t, res) } @@ -181,7 +181,8 @@ func (app *Vmselect) PrometheusAPIV1LabelValues(t *testing.T, labelName, matchQu values := opts.asURLValues() values.Add("match[]", matchQuery) - queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/label/%s/values", app.httpListenAddr, opts.getTenant(), labelName) + suffix := fmt.Sprintf("prometheus/api/v1/label/%s/values", labelName) + queryURL := getClusterPath(app.httpListenAddr, "select", suffix, opts) res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers) return NewPrometheusAPIV1LabelValuesResponse(t, res) @@ -195,7 +196,7 @@ func (app *Vmselect) PrometheusAPIV1Metadata(t *testing.T, metric string, limit values := opts.asURLValues() values.Add("metric", metric) values.Add("limit", strconv.Itoa(limit)) - queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/metadata", app.httpListenAddr, opts.getTenant()) + queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/metadata", opts) res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers) return NewPrometheusAPIV1Metadata(t, res) @@ -208,7 +209,7 @@ func (app *Vmselect) PrometheusAPIV1Metadata(t *testing.T, metric string, limit func (app *Vmselect) APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) { t.Helper() - queryURL := fmt.Sprintf("http://%s/delete/%s/prometheus/api/v1/admin/tsdb/delete_series", app.httpListenAddr, opts.getTenant()) + queryURL := getClusterPath(app.httpListenAddr, "delete", "prometheus/api/v1/admin/tsdb/delete_series", opts) values := opts.asURLValues() values.Add("match[]", matchQuery) @@ -229,7 +230,7 @@ func (app *Vmselect) MetricNamesStats(t *testing.T, limit, le, matchPattern stri values.Add("limit", limit) values.Add("le", le) values.Add("match_pattern", matchPattern) - queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/status/metric_names_stats", app.httpListenAddr, opts.getTenant()) + queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/status/metric_names_stats", opts) res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers) if statusCode != http.StatusOK { @@ -263,7 +264,7 @@ func (app *Vmselect) MetricNamesStatsReset(t *testing.T, opts QueryOpts) { func (app *Vmselect) APIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse { t.Helper() - seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/status/tsdb", app.httpListenAddr, opts.getTenant()) + url := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/status/tsdb", opts) values := opts.asURLValues() addNonEmpty := func(name, value string) { if len(value) == 0 { @@ -275,7 +276,7 @@ func (app *Vmselect) APIV1StatusTSDB(t *testing.T, matchQuery string, date strin addNonEmpty("topN", topN) addNonEmpty("date", date) - res, statusCode := app.cli.PostForm(t, seriesURL, values, opts.Headers) + res, statusCode := app.cli.PostForm(t, url, values, opts.Headers) if statusCode != http.StatusOK { t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res) } @@ -294,8 +295,8 @@ func (app *Vmselect) APIV1StatusTSDB(t *testing.T, matchQuery string, date strin func (app *Vmselect) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse { t.Helper() - seriesURL := fmt.Sprintf("http://%s/select/%s/graphite/metrics/index.json", app.httpListenAddr, opts.getTenant()) - res, statusCode := app.cli.Get(t, seriesURL, opts.Headers) + url := getClusterPath(app.httpListenAddr, "select", "graphite/metrics/index.json", opts) + res, statusCode := app.cli.Get(t, url, opts.Headers) if statusCode != http.StatusOK { t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res) } @@ -313,7 +314,7 @@ func (app *Vmselect) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) Graphite func (app *Vmselect) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) { t.Helper() - url := fmt.Sprintf("http://%s/select/%s/graphite/tags/tagSeries", app.httpListenAddr, opts.getTenant()) + url := getClusterPath(app.httpListenAddr, "select", "graphite/tags/tagSeries", opts) values := opts.asURLValues() values.Add("path", record) @@ -326,7 +327,7 @@ func (app *Vmselect) GraphiteTagsTagSeries(t *testing.T, record string, opts Que func (app *Vmselect) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) { t.Helper() - url := fmt.Sprintf("http://%s/select/%s/graphite/tags/tagMultiSeries", app.httpListenAddr, opts.getTenant()) + url := getClusterPath(app.httpListenAddr, "select", "graphite/tags/tagMultiSeries", opts) values := opts.asURLValues() for _, rec := range records { values.Add("path", rec) diff --git a/docs/victoriametrics/Cluster-VictoriaMetrics.md b/docs/victoriametrics/Cluster-VictoriaMetrics.md index 76338e07ef..26c8098f5b 100644 --- a/docs/victoriametrics/Cluster-VictoriaMetrics.md +++ b/docs/victoriametrics/Cluster-VictoriaMetrics.md @@ -68,8 +68,8 @@ The UI allows exploring query results via graphs and tables. See more details ab ## Multitenancy VictoriaMetrics cluster supports multiple isolated tenants (aka namespaces). -Tenants are identified by `accountID` or `accountID:projectID`, which are put inside request URLs for writes and reads. -See [these docs](#url-format) for details. +Tenants are identified by `accountID` or `accountID:projectID` inside request URLs or HTTP headers{{% available_from "#" %}} +for writes and reads. See [these docs](#url-format) for details. Some facts about tenants in VictoriaMetrics: @@ -84,22 +84,59 @@ or [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/). [Co - Data for all the tenants is evenly spread among available `vmstorage` nodes. This guarantees even load among `vmstorage` nodes when different tenants have different amounts of data and different query load. -- The database performance and resource usage doesn't depend on the number of tenants. It depends mostly on the total number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) in all the tenants. A time series is considered active if it received at least a single sample during the last hour. +- The database performance and resource usage do not depend on the number of tenants. It depends mostly on the total number of + [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) in all the tenants. - The list of registered tenants can be obtained via `http://:8481/admin/tenants` url. See [these docs](#url-format). - VictoriaMetrics exposes various per-tenant statistics via metrics - see [these docs](https://docs.victoriametrics.com/victoriametrics/pertenantstatistic/). -See also [multitenancy via labels](#multitenancy-via-labels). +See also multitenancy [via headers](#multitenancy-via-headers) and [via labels](#multitenancy-via-labels). + +### Multitenancy via headers + +By default, VictoriaMetrics allows specifying `accountID` and `projectID` only in the request URL. + +Set `--enableMultitenancyViaHeaders` {{% available_from "#" %}} command-line flag to support +specifying `accountID` and `projectID` via HTTP headers `AccountID` and `ProjectID` respectively. +This flag needs to be specified separately for vminserts and vmselects. + +When `--enableMultitenancyViaHeaders` is enabled, [URL format](#url-format) can be simplified to the following: +- `http://:8480/insert/` for writes +- `http://:8481/select/prometheus/` for reads + +For example, the following query will only select metric `up` from `accountID=2` and `projectID=3`: +``` +curl 'https://:8481/select/prometheus/api/v1/query' \ + -d 'query=up' \ + --header "AccountID: 2" \ + --header "ProjectID: 3" +``` + +The following example will ingest metric `up{instance="foo"}` to `accountID=2` and `projectID=0`: +``` +curl --header "AccountID: 2" -d 'up{instance="foo"} 123' -X POST https://:8480/insert/prometheus/api/v1/import/prometheus +``` + +> When simplified path `/(insert|select)/` is used and headers `AccountID`, `ProjectID` are missing, then IDs are set to `0:0` as default. +> If tenant IDs are specified in URL, then headers are ignored. + +The `AccountID` header can be set to `multitenant` string: `AccountID: multitenant`. See more in [multitenancy via labels](#multitenancy-via-labels). ### Multitenancy via labels -**Writes:** +Multitenancy via labels allows specifying [tenants](#multitenancy) as labels `vm_account_id` and `vm_project_id` during +ingestion or querying. This feature allows [ingesting workload with mixed tenants](#multitenant-writes) and [querying +data from multiple tenants](#multitenant-reads) via the same URL. -`vminsert` can accept data from multiple [tenants](#multitenancy) via a special `multitenant` endpoints `http://vminsert:8480/insert/multitenant/`, +#### Multitenant writes + +`vminsert` can accept data from multiple [tenants](#multitenancy) via special `multitenant` endpoints: + - `http://vminsert:8480/insert/multitenant/` + - `http://vminsert:8480/insert/` and HTTP header `AccountID: multitenant`. See how to enable headers [here](#multitenancy-via-headers). where `` can be replaced with any supported suffix for data ingestion from [this list](#url-format). -In this case the account ID and project ID are obtained from optional `vm_account_id` and `vm_project_id` labels of the incoming samples. -If `vm_account_id` or `vm_project_id` labels are missing or invalid, then the corresponding account ID and project ID are set to 0. +The `accountID` and `projectID` are obtained from optional `vm_account_id` and `vm_project_id` [labels](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#labels) of the incoming samples. +If `vm_account_id` or `vm_project_id` labels are missing or invalid, then the corresponding `accountID` and `projectID` are set to 0. These labels are automatically removed from samples before forwarding them to `vmstorage`. For example, if the following samples are written into `http://vminsert:8480/insert/multitenant/prometheus/api/v1/write`: @@ -119,11 +156,14 @@ such as [Graphite](https://docs.victoriametrics.com/victoriametrics/integrations [InfluxDB line protocol via TCP and UDP](https://docs.victoriametrics.com/victoriametrics/integrations/influxdb/) and [OpenTSDB telnet put protocol](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-telnet). -**Reads:** +#### Multitenant reads _For better performance prefer specifying [tenants in read URL](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format)._ -`vmselect` can execute {{% available_from "v1.104.0" %}} queries over multiple [tenants](#multitenancy) via special `multitenant` endpoints `http://vmselect:8481/select/multitenant/`. +`vmselect` can execute {{% available_from "v1.104.0" %}} queries over multiple [tenants](#multitenancy) via special `multitenant` endpoints: + - `http://vmselect:8481/select/multitenant/` + - `http://vmselect:8481/select/` and HTTP header `AccountID: multitenant`. See how to enable headers [here](#multitenancy-via-headers). + Currently supported endpoints for `` are: - `/prometheus/api/v1/query` @@ -169,7 +209,7 @@ The precedence for applying filters for tenants follows this order: These filters have the highest priority and are applied first when provided through the query arguments. 2. Filter tenants from labels selectors defined at metricsQL query expression. -**Security considerations** +> **Security considerations** It is recommended restricting access to `multitenant` endpoints only to trusted sources, since untrusted source may break per-tenant data by writing unwanted samples or get access to data of arbitrary tenants. @@ -585,13 +625,13 @@ The metric is set to `0` when the `vmstorage` isn't in read-only mode. The main differences between URL formats of cluster and [Single server](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) versions are that cluster has separate components for read and ingestion path, and because of multi-tenancy support. Also in the cluster version the `/prometheus/api/v1` endpoint ingests `jsonl`, `csv`, `native` and `prometheus` data formats **not** only `prometheus` data. -Check practical examples of [VictoriaMetrics API](https://docs.victoriametrics.com/victoriametrics/url-examples/). + +> Check practical examples of [VictoriaMetrics API](https://docs.victoriametrics.com/victoriametrics/url-examples/). - URLs for data ingestion: `http://:8480/insert//`, where: - `` is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). It is possible to set it as `accountID:projectID`, - where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. See [multitenancy docs](#multitenancy) for more details. - The `` can be set to `multitenant` string, e.g. `http://:8480/insert/multitenant/`. Such urls accept data from multiple tenants - specified via `vm_account_id` and `vm_project_id` labels. See [multitenancy via labels](#multitenancy-via-labels) for more details. + where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. See [multitenancy docs](#multitenancy) for more details + about managing tenants, specifying tenant IDs via HTTP headers or labels. - `` may have the following values: - `prometheus` and `prometheus/api/v1/write` - for ingesting data with [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). - `prometheus/api/v1/import` - for importing data obtained via `api/v1/export` at `vmselect` (see below), JSON line format. @@ -607,9 +647,8 @@ Check practical examples of [VictoriaMetrics API](https://docs.victoriametrics.c - `opentsdb/api/put` - for accepting [OpenTSDB HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html). This handler is disabled by default. It is exposed on a distinct TCP address set via `-opentsdbHTTPListenAddr` command-line flag. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http) for details. - URLs for [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/): `http://:8481/select//prometheus/`, where: - - `` is an arbitrary number identifying data namespace for the query (aka tenant). It is possible to set it as `accountID:projectID`, - where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. See [multitenancy docs](#multitenancy) for more details. - The `` can be set to `multitenant` string, e.g. `http://:8481/select/multitenant/` for querying over multiple tenants (see the full list of [supported multitenant read endpoints](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)). + - `` is an arbitrary number identifying data namespace for the query (aka tenant). See [multitenancy docs](#multitenancy) for more details + about managing tenants, specifying tenant IDs via HTTP headers or labels. - `` may have the following values: - `api/v1/query` - performs [PromQL instant query](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query). - `api/v1/query_range` - performs [PromQL range query](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query). @@ -629,7 +668,8 @@ Check practical examples of [VictoriaMetrics API](https://docs.victoriametrics.c - `metric-relabel-debug` - for debugging [relabeling rules](https://docs.victoriametrics.com/victoriametrics/relabeling/). - URLs for [Graphite Metrics API](https://graphite-api.readthedocs.io/en/latest/api.html#the-metrics-api): `http://:8481/select//graphite/`, where: - - `` is an arbitrary number identifying data namespace for query (aka tenant) + - `` is an arbitrary number identifying data namespace for query (aka tenant). + See [multitenancy docs](#multitenancy) for more details about managing tenants, specifying tenant IDs via HTTP headers or labels. - `` may have the following values: - `render` - implements Graphite Render API. See [these docs](https://graphite.readthedocs.io/en/stable/render_api.html). - `metrics/find` - searches Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find). diff --git a/docs/victoriametrics/vmagent.md b/docs/victoriametrics/vmagent.md index 29599bf968..3a078a9dcc 100644 --- a/docs/victoriametrics/vmagent.md +++ b/docs/victoriametrics/vmagent.md @@ -518,11 +518,14 @@ scrape_configs: `vmagent` can accept data via the same multitenant endpoints (`/insert//`) as `vminsert` at [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) does according to [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format) if `-enableMultitenantHandlers` command-line flag is set. -In this case, vmagent automatically converts tenant identifiers from the URL to `vm_account_id` and `vm_project_id` labels and sets tenant info in metadata. +In this case, vmagent automatically converts tenant identifiers from the URL or headers to `vm_account_id` and `vm_project_id` labels and sets tenant info in metadata. These tenant labels are added before applying [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/) specified via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` command-line flags. Metrics with `vm_account_id` and `vm_project_id` labels can be routed to the corresponding tenants when specifying `-remoteWrite.url` to [multitenant url at VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels). +`vmagent` can accept tenant IDs specified via HTTP headers if both `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` command-line flags are set. +See more about [multitenancy via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers). + ## Adding labels to metrics Extra labels can be added to metrics collected by `vmagent` via the following mechanisms: diff --git a/docs/victoriametrics/vmagent_common_flags.md b/docs/victoriametrics/vmagent_common_flags.md index 125facf0b7..9767635006 100644 --- a/docs/victoriametrics/vmagent_common_flags.md +++ b/docs/victoriametrics/vmagent_common_flags.md @@ -34,6 +34,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ . Whether to check config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag -enableMetadata Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true) + -enableMultitenancyViaHeaders + Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy -enableMultitenantHandlers Whether to process incoming data via multitenant insert handlers according to https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format . By default incoming data is processed via single-node insert handlers according to https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data .See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy for details -enableTCP6 diff --git a/docs/victoriametrics/vminsert_common_flags.md b/docs/victoriametrics/vminsert_common_flags.md index a811cf3238..65f5123982 100644 --- a/docs/victoriametrics/vminsert_common_flags.md +++ b/docs/victoriametrics/vminsert_common_flags.md @@ -38,6 +38,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori Whether to drop incoming samples if the destination vmstorage node is overloaded and/or unavailable. This prioritizes cluster availability over consistency, e.g. the cluster continues accepting all the ingested samples, but some of them may be dropped if vmstorage nodes are temporarily unavailable and/or overloaded. The drop of samples happens before the replication, so it's not recommended to use this flag with -replicationFactor enabled. -enableMetadata Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true) + -enableMultitenancyViaHeaders + Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers -enableTCP6 Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used -envflag.enable diff --git a/docs/victoriametrics/vmselect_common_flags.md b/docs/victoriametrics/vmselect_common_flags.md index cb569fa72e..bccd1f1383 100644 --- a/docs/victoriametrics/vmselect_common_flags.md +++ b/docs/victoriametrics/vmselect_common_flags.md @@ -41,6 +41,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori Flag value can be read from the given http/https url when using -deleteAuthKey=http://host/path or -deleteAuthKey=https://host/path -denyQueryTracing Whether to disable the ability to trace queries. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#query-tracing + -enableMultitenancyViaHeaders + Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers -enableTCP6 Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used -envflag.enable diff --git a/lib/httpserver/path.go b/lib/httpserver/path.go index 3dc4f9eff1..0c4fc58c92 100644 --- a/lib/httpserver/path.go +++ b/lib/httpserver/path.go @@ -2,20 +2,22 @@ package httpserver import ( "fmt" + "net/http" + "strconv" "strings" ) // Path contains the following path structure: -// /{prefix}/{authToken}/{suffix} -// -// It is compatible with SaaS version. +// - /{prefix}/{tenantID}/{suffix} +// - /{prefix}/{suffix} -H "{tenantID}" +// in `/{prefix}/{suffix}` format tenantID is extracted from HTTP headers type Path struct { Prefix string AuthToken string Suffix string } -// ParsePath parses the given path. +// ParsePath parses the given path according to /{prefix}/{tenantID}/{suffix} format func ParsePath(path string) (*Path, error) { // The path must have the following form: // /{prefix}/{authToken}/{suffix} @@ -56,6 +58,107 @@ func ParsePath(path string) (*Path, error) { return p, nil } +// ParsePathAndHeaders parses the given path and headers. +// +// The path may be one of the following forms: +// +// 1. /{prefix}/{tenantID}/{suffix} — tenantID is in the URL +// 2. /{prefix}/{suffix} — tenantID is omitted and expected to be read from AccountID/ProjectID HTTP headers. +// If these headers are missing, tenantID is set to "0:0" to be consistent with VictoriaLogs behavior. +// +// prefix is "select", "insert", or "delete". +// tenantID is "accountID[:projectID]" or "multitenant". +// tenantID specified in path always takes priority over headers for backward compatibility. +// +// This function doesn't validate correctness of {tenantID} content. +func ParsePathAndHeaders(path string, h http.Header) (*Path, error) { + s := skipPrefixSlashes(path) + n := strings.IndexByte(s, '/') + if n < 0 { + return nil, fmt.Errorf("cannot find {prefix} in %q; expecting /{prefix}/{suffix} or /{prefix}/{tenantID}/{suffix} format; "+ + "see https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format", path) + } + + prefix := s[:n] + tail := skipPrefixSlashes(s[n+1:]) + + if tail == "" { + return nil, fmt.Errorf("cannot find {suffix} in %q; expecting /{prefix}/{suffix} or /{prefix}/{tenantID}/{suffix} format; "+ + "see https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format", path) + } + + // Try to split tail into {tenantID}/{suffix} segments. + // If the first segment is a valid tenantID - consume it, ignore headers + // Otherwise, treat tail as {suffix} and read tenantID from HTTP headers. + var tenantID string + suffix := tail + n = strings.IndexByte(tail, '/') + if n >= 0 { + tenantID = tail[:n] + } + if maybeTenantID(tenantID) { + // cut the tenantID from suffix + suffix = skipPrefixSlashes(tail[n+1:]) + } else { + // tenantID is not valid - assume tail is all suffix and tenantID is in headers + tenantID = tenantIDFromHeadersOrDefault(h, "0:0") + } + + // Substitute double slashes with single slashes in the path, since such slashes + // may appear due to improper copy-pasting of the url. + suffix = strings.ReplaceAll(suffix, "//", "/") + + return &Path{ + Prefix: prefix, + AuthToken: tenantID, + Suffix: suffix, + }, nil +} + +// maybeTenantID returns true if s is "multitenant", "" or contains ":" char. +// It doesn't validate correctness of tenantID and is only used for quick routing. +// It is expected that tenantID will be correctly validated later. +func maybeTenantID(tenantID string) bool { + if tenantID == "" { + return false + } + if tenantID == "multitenant" { + return true + } + + idx := strings.IndexByte(tenantID, ':') + if idx > 0 { + return true + } + + _, err := strconv.ParseUint(tenantID, 10, 32) + return err == nil +} + +// tenantIDFromHeaders reads AccountID and ProjectID header values from request. +// If headers are missing, it returns defaultTenantID. +func tenantIDFromHeadersOrDefault(h http.Header, defaultTenantID string) string { + aID := h.Get("AccountID") + pID := h.Get("ProjectID") + if len(aID) == 0 && len(pID) == 0 { + return defaultTenantID + } + + if aID == "multitenant" { + // special case for multitenant + return "multitenant" + } + + accountID, projectID := "0", "0" + if len(aID) > 0 { + accountID = aID + } + if len(pID) > 0 { + projectID = pID + } + return fmt.Sprintf("%s:%s", accountID, projectID) +} + // skipPrefixSlashes remove double slashes which may appear due // improper copy-pasting of the url func skipPrefixSlashes(s string) string { diff --git a/lib/httpserver/path_test.go b/lib/httpserver/path_test.go new file mode 100644 index 0000000000..e4ca513abf --- /dev/null +++ b/lib/httpserver/path_test.go @@ -0,0 +1,84 @@ +package httpserver + +import ( + "net/http" + "strings" + "testing" +) + +func TestParsePathAndHeadersSuccess(t *testing.T) { + f := func(path, headers, prefix, authToken, suffix string) { + t.Helper() + header := make(http.Header) + hs := strings.Split(headers, ";") + for _, h := range hs { + if h == "" { + continue + } + parts := strings.Split(h, ":") + header.Set(parts[0], parts[1]) + } + p, err := ParsePathAndHeaders(path, header) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if p.Prefix != prefix { + t.Fatalf("unexpected Prefix; got %q; want %q", p.Prefix, prefix) + } + if p.AuthToken != authToken { + t.Fatalf("unexpected AuthToken; got %q; want %q", p.AuthToken, authToken) + } + if p.Suffix != suffix { + t.Fatalf("unexpected Suffix; got %q; want %q", p.Suffix, suffix) + } + } + + // tenant is omitted in the Path, so we try reading it from headers + f("/select/prometheus/api/v1/query", "AccountID:1;ProjectID:1", "select", "1:1", "prometheus/api/v1/query") + f("/select/prometheus/api/v1/query_range", "AccountID:1", "select", "1:0", "prometheus/api/v1/query_range") + f("/select/prometheus/api/v1/query_range", "ProjectID:1", "select", "0:1", "prometheus/api/v1/query_range") + f("/insert/prometheus", "", "insert", "0:0", "prometheus") + f("/insert/prometheus", "AccountID:1;ProjectID:1", "insert", "1:1", "prometheus") + f("/insert/prometheus", "AccountID:multitenant", "insert", "multitenant", "prometheus") + f("/delete/prometheus/api/v1/admin/tsdb/delete_series", "AccountID:1;ProjectID:1", "delete", "1:1", "prometheus/api/v1/admin/tsdb/delete_series") + f("/insert//prometheus/api/v1/import/prometheus", "AccountID:2", "insert", "2:0", "prometheus/api/v1/import/prometheus") + + // If headers are empty, we assume 0:0 as default. + f("/insert/prometheus", "", "insert", "0:0", "prometheus") + f("/select/prometheus/api/v1/query", "", "select", "0:0", "prometheus/api/v1/query") + + // tenant is present in the Path + f("/insert/123/prometheus/api/v1/write", "", "insert", "123", "prometheus/api/v1/write") + f("/select/1:15/prometheus/api/v1/query", "", "select", "1:15", "prometheus/api/v1/query") + f("/insert/multitenant/prometheus/api/v1/write", "", "insert", "multitenant", "prometheus/api/v1/write") + f("/insert/0/prometheus/api/v1/write", "", "insert", "0", "prometheus/api/v1/write") + f("/insert/0:0/prometheus/api/v1/write", "", "insert", "0:0", "prometheus/api/v1/write") + f("/delete/123/prometheus/api/v1/admin/tsdb/delete_series", "", "delete", "123", "prometheus/api/v1/admin/tsdb/delete_series") + + // tenant in the Path takes priority over headers + f("/insert/123/prometheus/api/v1/write", "AccountID:1;ProjectID:1", "insert", "123", "prometheus/api/v1/write") + f("/insert/multitenant/prometheus/api/v1/write", "AccountID:1;ProjectID:1", "insert", "multitenant", "prometheus/api/v1/write") + f("/insert/123:1/prometheus/api/v1/write", "AccountID:multitenant", "insert", "123:1", "prometheus/api/v1/write") + + // Double slashes in Path + f("//insert//123//prometheus//api/v1/write", "", "insert", "123", "prometheus/api/v1/write") + f("//insert//prometheus//api/v1/write", "AccountID:1;ProjectID:1", "insert", "1:1", "prometheus/api/v1/write") + +} + +func TestParsePathAndHeadersFailure(t *testing.T) { + f := func(path string) { + t.Helper() + p, err := ParsePathAndHeaders(path, nil) + if err == nil { + t.Fatalf("expecting non-nil error; got path %+v", p) + } + } + + // No prefix or suffix + f("/") + // Only prefix, no suffix or tenant + f("/insert") + f("/select/") + f("/select//") +}