app/vmui: generate CSV format using /api/v1/labels (#10771)

`Export query` button on `Raw Query` tab now fetches labels of executed query and composes export `format` based on that list of labels. It ensures that all query response labels are preserved in the CSV export. 

Also, commit removes the addition of the CSV header in the frontend. Now the header is added by the backend (see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10706).

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10667
PR https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10771
Duplicate of: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10737

### Checklist

The following checks are **mandatory**:

- [x] My change adheres to [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
- [x] My change adheres to [VictoriaMetrics development
goals](https://docs.victoriametrics.com/victoriametrics/goals/).

---------

Signed-off-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: lawrence3699 <lawrence3699@users.noreply.github.com>
Co-authored-by: Max Kotliar <mkotlyar@victoriametrics.com>

app/vmselect/vmui/index.html

@@ -37,9 +37,9 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-KEOgEEMl.js"></script>
+  <script type="module" crossorigin src="./assets/index-C24BPpD_.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-COnpUsM8.js">
-  <link rel="modulepreload" crossorigin href="./assets/vendor-Mr0bmX1E.js">
+  <link rel="modulepreload" crossorigin href="./assets/vendor-BWBgVCcr.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
   <link rel="stylesheet" crossorigin href="./assets/index-D2OEy8Ra.css">
 </head>

app/vmui/packages/vmui/src/api/query-range.ts

@@ -16,23 +16,29 @@ export const getExportDataUrl = (server: string, query: string, period: TimePara
   return `${server}/api/v1/export?${params}`;
 };
 
-export const getExportCSVDataUrl = (server: string, query: string[], period: TimeParams, reduceMemUsage: boolean): string => {
+const getBaseParams = (period: TimeParams, query: string[]): URLSearchParams => {
   const params = new URLSearchParams({
     start: period.start.toString(),
     end: period.end.toString(),
-    format: "__name__,__value__,__timestamp__:unix_ms",
   });
   query.forEach((q => params.append("match[]", q)));
+  return params;
+};
+
+export const getLabelsUrl = (server: string, query: string[], period: TimeParams): string => {
+  const params = getBaseParams(period, query);
+  return `${server}/api/v1/labels?${params}`;
+};
+
+export const getExportCSVDataUrl = (server: string, query: string[], period: TimeParams, reduceMemUsage: boolean, format: string): string => {
+  const params = getBaseParams(period, query);
+  params.set("format", format);
   if (reduceMemUsage) params.set("reduce_mem_usage", "1");
   return `${server}/api/v1/export/csv?${params}`;
 };
 
 export const getExportJSONDataUrl = (server: string, query: string[], period: TimeParams, reduceMemUsage: boolean): string => {
-  const params = new URLSearchParams({
-    start: period.start.toString(),
-    end: period.end.toString(),
-  });
-  query.forEach((q => params.append("match[]", q)));
+  const params = getBaseParams(period, query);
   if (reduceMemUsage) params.set("reduce_mem_usage", "1");
   return `${server}/api/v1/export?${params}`;
 };

app/vmui/packages/vmui/src/api/raw-query.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it, vi } from "vitest";
+import { fetchRawQueryCSVExport } from "./raw-query";
+
+describe("fetchRawQueryCSVExport", () => {
+  it.skip("requests all label columns before exporting CSV data", async () => {
+    const fetchMock = vi.fn()
+      .mockResolvedValueOnce({
+        ok: true,
+        json: async () => ({ data: ["job", "__name__", "instance"] }),
+      })
+      .mockResolvedValueOnce({
+        ok: true,
+        text: async () => "up,localhost:9100,node_exporter,1,1710000000000",
+      });
+
+    const result = await fetchRawQueryCSVExport(
+      "http://localhost:8428",
+      ["up"],
+      { start: 1710000000, end: 1710000300, step: "15s", date: "2024-03-09T16:05:00Z" },
+      false,
+      fetchMock as unknown as typeof fetch,
+    );
+
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(fetchMock.mock.calls[0][0]).toBe("http://localhost:8428/api/v1/labels?start=1710000000&end=1710000300&match%5B%5D=up");
+    expect(fetchMock.mock.calls[1][0]).toBe("http://localhost:8428/api/v1/export/csv?start=1710000000&end=1710000300&match%5B%5D=up&format=__name__%2Cinstance%2Cjob%2C__value__%2C__timestamp__%3Aunix_ms");
+    expect(result).toBe("up,localhost:9100,node_exporter,1,1710000000000");
+  });
+});

app/vmui/packages/vmui/src/api/raw-query.ts

@@ -0,0 +1,31 @@
+import { getExportCSVDataUrl, getLabelsUrl } from "./query-range";
+import { TimeParams } from "../types";
+import { getCSVExportColumns } from "../utils/csv";
+
+interface LabelsResponse {
+  data?: string[];
+}
+
+export const fetchRawQueryCSVExport = async (
+  serverUrl: string,
+  query: string[],
+  period: TimeParams,
+  reduceMemUsage: boolean,
+  fetchFn: typeof fetch = fetch,
+): Promise<string> => {
+  const labelsResponse = await fetchFn(getLabelsUrl(serverUrl, query, period));
+  if (!labelsResponse.ok) {
+    throw new Error(await labelsResponse.text());
+  }
+
+  const { data = [] } = (await labelsResponse.json()) as LabelsResponse;
+  const columns = getCSVExportColumns(data);
+  const format = columns.join(",");
+
+  const response = await fetchFn(getExportCSVDataUrl(serverUrl, query, period, reduceMemUsage, format));
+  if (!response.ok) {
+    throw new Error(await response.text());
+  }
+
+  return await response.text();
+};

app/vmui/packages/vmui/src/pages/RawQueryPage/hooks/useFetchExport.ts

@@ -6,10 +6,11 @@ import { useTimeState } from "../../../state/time/TimeStateContext";
 import { useAppState } from "../../../state/common/StateContext";
 import { useCustomPanelState } from "../../../state/customPanel/CustomPanelStateContext";
 import { isValidHttpUrl } from "../../../utils/url";
-import { getExportCSVDataUrl, getExportDataUrl, getExportJSONDataUrl } from "../../../api/query-range";
+import { getExportDataUrl, getExportJSONDataUrl } from "../../../api/query-range";
 import { parseLineToJSON } from "../../../utils/json";
 import { downloadCSV, downloadJSON } from "../../../utils/file";
 import { useSnack } from "../../../contexts/Snackbar";
+import { fetchRawQueryCSVExport } from "../../../api/raw-query";
 
 interface FetchQueryParams {
   hideQuery?: number[];
@@ -67,11 +68,8 @@ export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams):
     const getFilename = (format: ExportFormats) => `vmui_export_${query.join("_")}_${period.start}_${period.end}.${format}`;
     return {
       csv: async () => {
-        const url = getExportCSVDataUrl(serverUrl, query, period, reduceMemUsage);
-        const response = await fetch(url);
         try {
-          let text = await response.text();
-          text = "name,value,timestamp\n" + text;
+          const text = await fetchRawQueryCSVExport(serverUrl, query, period, reduceMemUsage);
           downloadCSV(text, getFilename("csv"));
         } catch (e) {
           console.error(e);

app/vmui/packages/vmui/src/utils/csv.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { formatValueToCSV } from "./csv";
+import { formatValueToCSV, getCSVExportColumns } from "./csv";
 
 describe("formatValueToCSV", () => {
   it("should wrap value in quotes if it contains a comma", () => {
@@ -32,3 +32,10 @@ describe("formatValueToCSV", () => {
     expect(result).toBe("");
   });
 });
+
+describe("getCSVExportColumns", () => {
+  it("should prepend metric name and append value and timestamp columns", () => {
+    const result = getCSVExportColumns(["instance", "__name__", "job", "instance"]);
+    expect(result.join(",")).toEqual("__name__,instance,job,__value__,__timestamp__:unix_ms");
+  });
+});

app/vmui/packages/vmui/src/utils/csv.ts

@@ -2,3 +2,8 @@ export const formatValueToCSV= (value: string) =>
   (value.includes(",") || value.includes("\n") || value.includes("\""))
     ? "\"" + value.replace(/"/g, "\"\"") + "\""
     : value;
+
+export const getCSVExportColumns = (labelNames: string[]) => {
+  const labels = Array.from(new Set(labelNames.filter((label) => label && label !== "__name__"))).sort();
+  return ["__name__", ...labels, "__value__", "__timestamp__:unix_ms"];
+};

docs/victoriametrics/changelog/CHANGELOG.md

@@ -41,13 +41,13 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): optimize vminsert buffer size per vmstorage node based on available CPU, memory and storage node count to reduce OOM risk. See [#10725](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10725).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): allow setting `-1` value for `-storage.maxHourlySeries` and `-storage.maxDailySeries` command-line flags. This automatically sets limits to the highest possible value in order to enable tracking without enforcing any limits. This is helpful for estimating current usage before applying real limits. See [#9614](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9614).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): expose `vmalert_remotewrite_queue_size` and `vmalert_remotewrite_queue_capacity` to facilitate monitoring of remote write queue usage. See [#10765](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10765).
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): CSV export on the `Raw Query` tab now includes all labels from the executed query. VMUI no longer prepends a header row, as it is now provided by the backend. See [#10667](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10667) and [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666). Thanks to @lawrence3699 for the contribution.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add header row to `/api/v1/export/csv` output and auto-detect header rows during import via `/api/v1/import/csv`. See [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666). Thanks to @andriibeee for the contribution.
 
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry the requests that failed with unexpected EOF due to unstable network to S3 service. See [#10699](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10699).
 * BUGFIX: All VictoriaMetrics components: Fix an issue where `unsupported` metric metadata type was exposed for summaries and quantiles if a summary wasn't updated within a certain time window. See [metrics#120](https://github.com/VictoriaMetrics/metrics/issues/120) and [metrics#121](https://github.com/VictoriaMetrics/metrics/pull/121).
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): align request body buffering flags - `maxRequestBodySizeToRetry` and `requestBufferSize` to the same `16KB` value. Allow disabling request buffering by setting `requestBufferSize=0`. See [#10675](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10675)
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix `scrape_series_added` metric to update only on successful scrapes, aligning its behavior with Prometheus. See [#10653](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10653).
-
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent partial responses from second-level vmselect nodes in [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). Ensures response completeness and correctness, and avoids cache pollution in top-level vmselect. See [#10678](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10678)
 
 ## [v1.139.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.139.0)