Compare commits

..

39 Commits

Author SHA1 Message Date
Zakhar Bessarab
3e5527ae57 apptest/tests: add test to verify sparse cache usage
Sparse cache is only used for "final merges" - merge of data of previous months, so tests verify that.

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
2025-01-29 17:19:40 +04:00
Github Actions
0e08a7a125 Automatic update Grafana datasource docs from VictoriaMetrics/victoriametrics-datasource@0d6e500 (#8180) 2025-01-29 13:27:11 +01:00
Roman Khavronenko
13c4324bb5 lib/cgroup: warn users about using fractional CPU quotas (#8175)
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7988

### Describe Your Changes

Please provide a brief description of the changes you made. Be as
specific as possible to help others understand the purpose and impact of
your modifications.

### Checklist

The following checks are **mandatory**:

- [ ] My change adheres [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/contributing/).

---------

Signed-off-by: hagen1778 <roman@victoriametrics.com>
2025-01-29 13:19:08 +01:00
Aliaksandr Valialkin
95f182053b lib/logstorage: remove unnecesary abstraction - RowsFormatter
It is better to use the AppendFieldsToJSON function directly
instead of hiding it under RowsFormatter abstraction.
2025-01-28 18:03:18 +01:00
Aliaksandr Valialkin
ec64a1fd7c docs/Articles.md: add a link to https://tanmay-bhat.medium.com/reducing-inter-az-traffic-in-victoriametrics-with-zonekeeper-3bd7e1526796
Thanks to Tanmay Bhat ( https://tanmay-bhat.medium.com/ ) for the great article about VictoriaMetrics and vmagent!
2025-01-28 17:03:45 +01:00
Aliaksandr Valialkin
f61b5da617 go.mod: update Go from v1.23.5 to 1.23.5
This is a follow-up for 489631b227
2025-01-28 16:55:48 +01:00
Aliaksandr Valialkin
3c036e0d31 lib/logstorage: ignore logs with too long field names during data ingestion
Previously too long field names were silently truncated. This is not what most users expect.
It is better ignoring the whole log entry in this case and logging it with the WARNING message,
so human operator could notice and fix the ingestion of incorrect logs ASAP.

The commit also adds and updates the following entries to VictoriaLogs faq:

- https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain
- https://docs.victoriametrics.com/victorialogs/faq/#what-is-the-maximum-supported-field-name-length
- https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have

These entries are referred at `-insert.maxLineSizeBytes` and `-insert.maxFieldsPerLine` command-line descriptions
and at the WARNING messages, which are emitted when log entries are ignored because of some of these limits
are exceeded.
2025-01-28 16:55:48 +01:00
Aliaksandr Valialkin
aacb9b2726 app/vlogscli: show compact output mode line in the help output to be consistent with the naming for other output modes 2025-01-28 16:55:47 +01:00
Alexander Marshalov
c753f75e18 vmcloud docs: actualize parameters description due to changes in v1.108.0 (#6928) (#8172)
vmcloud docs: actualize parameters description due to changes in
v1.108.0 (#6928)
2025-01-28 16:05:07 +01:00
f41gh7
330461c635 docs/changelog: mention v1.102.12 and v1.97.17 releases
Signed-off-by: f41gh7 <nik@victoriametrics.com>
2025-01-28 14:27:44 +01:00
Roman Khavronenko
5a1a28ba87 dashboards: consistently use process_cpu_cores_available for CPU usage
Related issue:
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7988

---------

Signed-off-by: hagen1778 <roman@victoriametrics.com>
2025-01-28 13:58:06 +01:00
hagen1778
5b18dc0214 docs: re-order changes for better readability
Signed-off-by: hagen1778 <roman@victoriametrics.com>
2025-01-28 09:57:15 +01:00
Aliaksandr Valialkin
10abdf34ab vendor: run make vendor-update 2025-01-27 23:10:31 +01:00
Fred Navruzov
e55b5fc605 docs/vmanomaly: release v1.19.2 (#8165)
### Describe Your Changes

docs/vmanomaly: release v1.19.2 (patch that addresses some of the bugs
found in 1.19.0)

### Checklist

The following checks are **mandatory**:

- [x] My change adheres [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/contributing/).

---------

Co-authored-by: Mathias Palmersheim <mathias@victoriametrics.com>
2025-01-27 21:03:34 +02:00
Aliaksandr Valialkin
3b34241380 lib/fs/fsutil: move lib/envutil to the more appropriate place at lib/fs/fsutil
This is a follow-up for 043d066133
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6871
2025-01-27 18:47:39 +01:00
Zakhar Bessarab
de144899b2 docs/{vmauth,VictoriaLogs}: add examples of using headers per-user
### Describe Your Changes
- add example a generic example to vmauth docs
- add an multi-tenancy usage example to VictoriaLogs docs
### Checklist

The following checks are **mandatory**:

- [x] My change adheres [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/contributing/).

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
2025-01-27 20:14:43 +04:00
Aliaksandr Valialkin
dc9dba71b2 lib/storage: open per-month partitions in parallel
This should reduce the time needed for opening the storage with retentions exceeding a few months.

While at at, limit the concurrency of opening partitions in parallel to the number of available CPU cores,
since higher concurrency may increase RAM usage and CPU usage without performance improvements
if opening a single partition is CPU-bound task.

This is a follow-up for 17988942ab
2025-01-27 16:07:14 +01:00
Aliaksandr Valialkin
fd1568531d lib/filestream: use correct formatting option for error type in the error message 2025-01-27 15:23:52 +01:00
f41gh7
46bb2d4308 docs/changelog: mention lts releases
Signed-off-by: f41gh7 <nik@victoriametrics.com>
2025-01-27 11:05:50 +01:00
f41gh7
8fa8101677 docs: point apps versions to the latest releases
v1.110.0
v1.102.11
v1.97.16

Signed-off-by: f41gh7 <nik@victoriametrics.com>
2025-01-27 11:05:50 +01:00
f41gh7
204cec7913 CHANGELOG.md: cut v1.110.0 release 2025-01-27 11:05:50 +01:00
Aliaksandr Valialkin
17988942ab lib/logstorage: open per-day partitions in parallel during startup
This significantly reduces startup times when the storage contains large partitions over many days.
2025-01-27 00:34:02 +01:00
Aliaksandr Valialkin
ed05ae12c4 lib/logstorage: optimize unmarshalColumnNames a bit
This should reduce the time needed for opening a large storage with many partitions,
which contain logs with big number of fields (aka wide events).

Thanks to @kiriklo for the initial idea at the pull request https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8061

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7937
2025-01-27 00:12:46 +01:00
Aliaksandr Valialkin
256924e2d6 lib/logstorage: improve error message by adding a link with the explanation why VictoriaLogs ignores logs with the size exceeding 2MB
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7972
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7984
2025-01-26 22:50:26 +01:00
Aliaksandr Valialkin
b5392337bf lib/logstorage: block_stat pipe: return the path to the part where the block is stored 2025-01-26 22:36:47 +01:00
Aliaksandr Valialkin
fa4e3607c3 docs/VictoriaLogs: small updates 2025-01-26 22:03:21 +01:00
Aliaksandr Valialkin
043d066133 lib/{fs,filestream}: unconditionally disable fsync in tests
Use the testing.Testing() function in order to determine whether the code runs in test.
This allows running tests and fast speed without the need to specify DISABLE_FSYNC_FOR_TESTING
environment variable.

This is a follow-up for the commit 334cd92a6c
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6871
2025-01-26 21:47:20 +01:00
Aliaksandr Valialkin
900159a2d3 lib/logstorage: remove unneeded code after 202eb429a7
readerWithStats isn't used when reading column names from file
2025-01-26 20:04:15 +01:00
Aliaksandr Valialkin
4464c5a254 docs/VictoriaLogs/sql-to-logsql.md: pay attention to the fact that stats() pipe at LogsQL has better usability than GROUP BY at SQL 2025-01-24 19:58:37 +01:00
Aliaksandr Valialkin
90fed18b83 deployment/docker: update VictoriaLogs from v1.7.0-victorialogs to v1.8.0-victorialogs
See https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.8.0-victorialogs
2025-01-24 19:16:27 +01:00
Aliaksandr Valialkin
1480ecc129 docs/VictoriaLogs/CHANGELOG.md: cut v1.8.0-victorialogs release 2025-01-24 19:06:44 +01:00
Aliaksandr Valialkin
c67f4d4d86 app/vlselect/vmui: run make vmui-logs-update after the commit 87739bbbef 2025-01-24 19:05:47 +01:00
Aliaksandr Valialkin
c2f5088adc docs/VictoriaLogs/CHANGELOG.md: move the changes from 87739bbbef to the correct place
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7750
2025-01-24 19:04:35 +01:00
Yury Molodov
87739bbbef vmui/logs: update hits chart legend (#7750)
### Describe Your Changes

- Added the `fields_limit` parameter for the `hits` query to limit the
number of returned fields. This reduces the response size and decreases
memory usage.

#### Legend Menu:
A context menu has been added for legend items, which includes:  
1. Metric name
2. **Copy _stream name** - copies the full stream name in the format
`{field1="value1", ..., fieldN="valueN"}`.
3. **Add _stream to filter** - adds the full stream value to the current
filter:
   `_stream: {field1="value1", ..., fieldN="valueN"} AND (old_expr)`.  
4. **Exclude _stream from filter** - excludes the stream from the
current filter:
`(NOT _stream: {field1="value1", ..., fieldN="valueN"}) AND (old_expr)`.
5. List of fields with options:  
   - Copy as `field: "value"`.  
   - Add to filter: `field: "value" AND (old_expr)`.  
   - Exclude from filter: `-field: "value" AND (old_expr)`.  
6. Total number of hits for the stream.  

Related issue: #7552

<details>
  <summary>UI Demo - Legend Menu</summary>
  
<img width="400"
src="https://github.com/user-attachments/assets/ee1954b2-fdce-44b4-a2dc-aa73096a5414"/>
<img width="400"
src="https://github.com/user-attachments/assets/19d71f04-c207-4143-a176-c5f221592e3d"/>

</details>

---

#### Legend:
1. Displays the total number of hits for the stream.  
2. Added hints below the legend for total hits and graph interactions.
3. Click behavior is now the same as in other vmui charts:  
   - `click` - shows only the selected series.  
   - `click + ctrl/cmd` - hides the selected series.  
   
<details>
  <summary>UI Demo - Legend</summary>

before: 
<img
src="https://github.com/user-attachments/assets/18270842-0c39-4f63-bcda-da62e15c3c73"/>

after:
<img
src="https://github.com/user-attachments/assets/351cad3a-f763-4b1d-b3be-b569b5472a7c"/>
  
</details>

---

#### Tooltip:  
1. The `other` label is moved to the end, and others are sorted by
value.
2. Values are aligned to the right.  
3. Labels are truncated and always shown in a single line for better
readability; the full name is available in the legend.

<details>
  <summary>UI Demo - Tooltip</summary>
  
| before     | after        |
|----------|----------|
| <img
src="https://github.com/user-attachments/assets/adccff38-e2e6-46e4-a69e-21381982489c"/>
| <img
src="https://github.com/user-attachments/assets/81008897-d816-4aed-92cb-749ea7e0ff1e"/>
|
  
</details>

---

#### Group View (tab Group):
Groups are now sorted by the number of records in descending order.

<details>
  <summary>UI Demo - Group View</summary>

before: 
<img width="800"
src="https://github.com/user-attachments/assets/15b4ca72-7e5d-421f-913b-c5ff22c340cb"/>

after:
<img width="800"
src="https://github.com/user-attachments/assets/32ff627b-6f30-4195-bfe7-8c9b4aa11f6b"/>
  
</details>
2025-01-24 19:02:04 +01:00
Aliaksandr Valialkin
dce4dc0a33 docs/VictoriaLogs/CHANGELOG.md: typo fix after ad6c587494: ignore_global_time_range -> ignore_global_time_filter 2025-01-24 18:59:40 +01:00
Aliaksandr Valialkin
ad6c587494 lib/logstorage: properly propagate extra filters to all the subqueries
The purpose of extra filters ( https://docs.victoriametrics.com/victorialogs/querying/#extra-filters )
is to limit the subset of logs, which can be queried. For example, it is expected that all the queries
with `extra_filters={tenant=123}` can access only logs, which contain `123` value for the `tenant` field.

Previously this wasn't the case, since the provided extra filters weren't applied to subqueries.
For example, the following query could be used to select all the logs outside `tenant=123`, for any `extra_filters` arg:

    * | union({tenant!=123})

This commit fixes this by propagating extra filters to all the subqueries.

While at it, this commit also properly propagates [start, end] time range filter from HTTP querying APIs
into all the subqueries, since this is what most users expect. This behaviour can be overriden on per-subquery
basis with the `options(ignore_global_time_filter=true)` option - see https://docs.victoriametrics.com/victorialogs/logsql/#query-options

Also properly apply apply optimizations across all the subqueries. Previously the optimizations at Query.optimize()
function were applied only to the top-level query.
2025-01-24 18:49:25 +01:00
Aliaksandr Valialkin
467cdd8a3d lib: consistently use logger.Panicf("BUG: ...") for logging programming bugs
logger.Fatalf("BUG: ...") complicates investigating the bug, since it doesn't show the call stack,
which led to the bug. So it is better to consistently use logger.Panicf("BUG: ...") for logging programming bugs.
2025-01-24 16:39:21 +01:00
Aliaksandr Valialkin
026894054b docs/VictoriaLogs/LogsQL.md: show how to unroll the returned histogram buckets into separate rows at histogram pipe docs 2025-01-24 16:39:21 +01:00
Aliaksandr Valialkin
db9107acef docs/VictoriaLogs/sql-to-logsql.md: show how to substitute complex SQL query with top pipe 2025-01-24 16:39:20 +01:00
378 changed files with 5442 additions and 2050 deletions

View File

@@ -513,19 +513,19 @@ check-all: fmt vet golangci-lint govulncheck
clean-checkers: remove-golangci-lint remove-govulncheck
test:
DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/... ./app/...
go test ./lib/... ./app/...
test-race:
DISABLE_FSYNC_FOR_TESTING=1 go test -race ./lib/... ./app/...
go test -race ./lib/... ./app/...
test-pure:
DISABLE_FSYNC_FOR_TESTING=1 CGO_ENABLED=0 go test ./lib/... ./app/...
CGO_ENABLED=0 go test ./lib/... ./app/...
test-full:
DISABLE_FSYNC_FOR_TESTING=1 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
test-full-386:
DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
integration-test: victoria-metrics vmagent vmalert vmauth
go test ./apptest/... -skip="^TestCluster.*"

View File

@@ -199,8 +199,8 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []l
lmp.bytesIngestedTotal.Add(n)
if len(fields) > *MaxFieldsPerLine {
rf := logstorage.RowFormatter(fields)
logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
line := logstorage.MarshalFieldsToJSON(nil, fields)
logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, line)
rowsDroppedTotalTooManyFields.Inc()
return
}

View File

@@ -8,8 +8,10 @@ import (
var (
// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers; "+
"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have")
// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers; "+
"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain")
)

View File

@@ -270,7 +270,7 @@ func printCommandsHelp(w io.Writer) {
\h - show this help
\s - singleline json output mode
\m - multiline json output mode
\c - compact output
\c - compact output mode
\logfmt - logfmt output mode
\wrap_long_lines - toggles wrapping long lines
\tail <query> - live tail <query> results

View File

@@ -1,12 +1,12 @@
{
"files": {
"main.css": "./static/css/main.3134e778.css",
"main.js": "./static/js/main.82cd6930.js",
"main.css": "./static/css/main.02a1c6cb.css",
"main.js": "./static/js/main.55c8060b.js",
"static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
"index.html": "./index.html"
},
"entrypoints": [
"static/css/main.3134e778.css",
"static/js/main.82cd6930.js"
"static/css/main.02a1c6cb.css",
"static/js/main.55c8060b.js"
]
}

View File

@@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.82cd6930.js"></script><link href="./static/css/main.3134e778.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.55c8060b.js"></script><link href="./static/css/main.02a1c6cb.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -1,3 +1,6 @@
import uPlot from "uplot";
import { ReactNode } from "react";
export interface MetricBase {
group: number;
metric: {
@@ -6,13 +9,13 @@ export interface MetricBase {
}
export interface MetricResult extends MetricBase {
values: [number, string][]
values: [number, string][];
}
export interface InstantMetricResult extends MetricBase {
value?: [number, string]
values?: [number, string][]
value?: [number, string];
values?: [number, string][];
}
export interface ExportMetricResult extends MetricBase {
@@ -44,9 +47,23 @@ export interface LogHits {
timestamps: string[];
values: number[];
total?: number;
fields: {
[key: string]: string;
};
fields: { [key: string]: string; };
_isOther: boolean;
}
export interface LegendLogHits {
label: string;
total: number;
totalHits: number;
isOther: boolean;
fields: { [key: string]: string; };
stroke?: uPlot.Series.Stroke;
}
export interface LegendLogHitsMenu {
title: string;
icon?: ReactNode;
handler?: () => void;
}
export interface ReportMetaData {

View File

@@ -1,22 +1,23 @@
import React, { FC, useMemo, useRef, useState } from "preact/compat";
import React, { FC, useCallback, useMemo, useRef, useState } from "preact/compat";
import "./style.scss";
import "uplot/dist/uPlot.min.css";
import useElementSize from "../../../hooks/useElementSize";
import uPlot, { AlignedData } from "uplot";
import { useEffect } from "react";
import useBarHitsOptions from "./hooks/useBarHitsOptions";
import useBarHitsOptions, { getLabelFromLogHit } from "./hooks/useBarHitsOptions";
import BarHitsTooltip from "./BarHitsTooltip/BarHitsTooltip";
import { TimeParams } from "../../../types";
import usePlotScale from "../../../hooks/uplot/usePlotScale";
import useReadyChart from "../../../hooks/uplot/useReadyChart";
import useZoomChart from "../../../hooks/uplot/useZoomChart";
import classNames from "classnames";
import { LogHits } from "../../../api/types";
import { LegendLogHits, LogHits } from "../../../api/types";
import { addSeries, delSeries, setBand } from "../../../utils/uplot";
import { GraphOptions, GRAPH_STYLES } from "./types";
import BarHitsOptions from "./BarHitsOptions/BarHitsOptions";
import stack from "../../../utils/uplot/stack";
import BarHitsLegend from "./BarHitsLegend/BarHitsLegend";
import { calculateTotalHits, sortLogHits } from "../../../utils/logs";
interface Props {
logHits: LogHits[];
@@ -57,6 +58,29 @@ const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onAp
graphOptions
});
const prepareLegend = useCallback((hits: LogHits[], totalHits: number): LegendLogHits[] => {
return hits.map((hit) => {
const label = getLabelFromLogHit(hit);
const legendItem: LegendLogHits = {
label,
isOther: hit._isOther,
fields: hit.fields,
total: hit.total || 0,
totalHits,
stroke: series.find((s) => s.label === label)?.stroke,
};
return legendItem;
}).sort(sortLogHits("total"));
}, [series]);
const legendDetails: LegendLogHits[] = useMemo(() => {
const totalHits = calculateTotalHits(logHits);
return prepareLegend(logHits, totalHits);
}, [logHits, prepareLegend]);
useEffect(() => {
if (!uPlotInst) return;
delSeries(uPlotInst);
@@ -121,6 +145,7 @@ const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onAp
<BarHitsLegend
uPlotInst={uPlotInst}
onApplyFilter={onApplyFilter}
legendDetails={legendDetails}
/>
)}
</div>

View File

@@ -1,83 +1,53 @@
import React, { FC, useCallback, useEffect, useState } from "preact/compat";
import React, { FC, useEffect, useState } from "preact/compat";
import uPlot, { Series } from "uplot";
import "./style.scss";
import "../../Line/Legend/style.scss";
import classNames from "classnames";
import { MouseEvent } from "react";
import { isMacOs } from "../../../../utils/detect-device";
import Tooltip from "../../../Main/Tooltip/Tooltip";
import { getStreamPairs } from "../../../../utils/logs";
import BarHitsLegendItem from "./BarHitsLegendItem";
import { LegendLogHits } from "../../../../api/types";
interface Props {
uPlotInst: uPlot;
legendDetails: LegendLogHits[];
onApplyFilter: (value: string) => void;
}
const BarHitsLegend: FC<Props> = ({ uPlotInst, onApplyFilter }) => {
const BarHitsLegend: FC<Props> = ({ uPlotInst, legendDetails, onApplyFilter }) => {
const [series, setSeries] = useState<Series[]>([]);
const [pairs, setPairs] = useState<string[][]>([]);
const totalHits = legendDetails[0]?.totalHits || 0;
const updateSeries = useCallback(() => {
const series = uPlotInst.series.filter(s => s.scale !== "x");
setSeries(series);
setPairs(series.map(s => getStreamPairs(s.label || "")));
const getSeries = () => {
return uPlotInst.series.filter(s => s.scale !== "x");
};
const handleRedrawGraph = () => {
uPlotInst.redraw();
setSeries(getSeries());
};
useEffect(() => {
setSeries(getSeries());
}, [uPlotInst]);
const handleClickByValue = (value: string) => (e: MouseEvent<HTMLDivElement>) => {
const metaKey = e.metaKey || e.ctrlKey;
if (!metaKey) return;
onApplyFilter(`{${value}}` || "");
updateSeries();
uPlotInst.redraw();
};
const handleClickByStream = (target: Series) => (e: MouseEvent<HTMLDivElement>) => {
const metaKey = e.metaKey || e.ctrlKey;
if (metaKey) return;
target.show = !target.show;
updateSeries();
uPlotInst.redraw();
};
useEffect(updateSeries, [uPlotInst]);
return (
<div className="vm-bar-hits-legend">
{series.map((s, i) => (
<Tooltip
key={s.label}
title={(
<ul className="vm-bar-hits-legend-info">
<li>Click to {s.show ? "hide" : "show"} the _stream.</li>
<li>{isMacOs() ? "Cmd" : "Ctrl"} + Click to filter by the _stream.</li>
</ul>
)}
>
<div
className={classNames({
"vm-bar-hits-legend-item": true,
"vm-bar-hits-legend-item_hide": !s.show,
})}
onClick={handleClickByStream(s)}
>
<div
className="vm-bar-hits-legend-item__marker"
style={{ backgroundColor: `${(s?.stroke as () => string)?.()}` }}
/>
<div className="vm-bar-hits-legend-item-pairs">
{pairs[i].map(value => (
<span
className="vm-bar-hits-legend-item-pairs__value"
key={value}
onClick={handleClickByValue(value)}
>
{value}
</span>
))}
</div>
</div>
</Tooltip>
{legendDetails.map((legend) => (
<BarHitsLegendItem
key={legend.label}
legend={legend}
series={series}
onRedrawGraph={handleRedrawGraph}
onApplyFilter={onApplyFilter}
/>
))}
<div className="vm-bar-hits-legend-info">
<div>
Total hits: <b>{totalHits.toLocaleString("en-US")}</b>
</div>
<div>
<code>L-Click</code> toggles visibility.&nbsp;
<code>R-Click</code> opens menu.
</div>
</div>
</div>
);
};

View File

@@ -0,0 +1,92 @@
import React, { FC, useMemo, useRef, useState } from "preact/compat";
import classNames from "classnames";
import { Series } from "uplot";
import { MouseEvent } from "react";
import { LegendLogHits } from "../../../../api/types";
import { getStreamPairs } from "../../../../utils/logs";
import { formatNumberShort } from "../../../../utils/math";
import Popper from "../../../Main/Popper/Popper";
import useBoolean from "../../../../hooks/useBoolean";
import LegendHitsMenu from "../LegendHitsMenu/LegendHitsMenu";
interface Props {
legend: LegendLogHits;
series: Series[];
onRedrawGraph: () => void;
onApplyFilter: (value: string) => void;
}
const BarHitsLegendItem: FC<Props> = ({ legend, series, onRedrawGraph, onApplyFilter }) => {
const {
value: openContextMenu,
setTrue: handleOpenContextMenu,
setFalse: handleCloseContextMenu,
} = useBoolean(false);
const legendRef = useRef<HTMLDivElement>(null);
const [clickPosition, setClickPosition] = useState<{ top: number; left: number } | null>(null);
const targetSeries = useMemo(() => series.find(s => s.label === legend.label), [series]);
const fields = useMemo(() => getStreamPairs(legend.label), [legend.label]);
const label = fields.join(", ");
const totalShortFormatted = formatNumberShort(legend.total);
const handleClickByStream = (e: MouseEvent<HTMLDivElement>) => {
if (!targetSeries) return;
if (e.metaKey || e.ctrlKey) {
targetSeries.show = !targetSeries.show;
} else {
const isOnlyTargetVisible = series.every(s => s === targetSeries || !s.show);
series.forEach(s => {
s.show = isOnlyTargetVisible || (s === targetSeries);
});
}
onRedrawGraph();
};
const handleContextMenu = (e: MouseEvent<HTMLDivElement>) => {
e.preventDefault();
setClickPosition({ top: e.clientY, left: e.clientX });
handleOpenContextMenu();
};
return (
<div
ref={legendRef}
className={classNames({
"vm-bar-hits-legend-item": true,
"vm-bar-hits-legend-item_other": legend.isOther,
"vm-bar-hits-legend-item_hide": !targetSeries?.show,
})}
onClick={handleClickByStream}
onContextMenu={handleContextMenu}
>
<div
className="vm-bar-hits-legend-item__marker"
style={{ backgroundColor: `${legend.stroke}` }}
/>
<div className="vm-bar-hits-legend-item__label">{label}</div>
<span className="vm-bar-hits-legend-item__total">({totalShortFormatted})</span>
<Popper
placement="fixed"
open={openContextMenu}
buttonRef={legendRef}
placementPosition={clickPosition}
onClose={handleCloseContextMenu}
>
<LegendHitsMenu
legend={legend}
fields={fields}
onApplyFilter={onApplyFilter}
onClose={handleCloseContextMenu}
/>
</Popper>
</div>
);
};
export default BarHitsLegendItem;

View File

@@ -3,16 +3,16 @@
.vm-bar-hits-legend {
display: flex;
flex-wrap: wrap;
gap: $padding-small;
padding: 0 $padding-small $padding-small;
color: $color-text;
&-item {
display: grid;
grid-template-columns: auto 1fr;
max-width: 50%;
display: flex;
align-items: center;
gap: $padding-small;
font-size: 12px;
padding: 0 $padding-small;
font-size: $font-size-small;
padding: $padding-small $padding-global;
border-radius: $border-radius-small;
cursor: pointer;
transition: 0.2s;
@@ -27,34 +27,44 @@
}
&__marker {
width: 14px;
min-width: 14px;
max-width: 14px;
height: 14px;
border: $color-background-block;
}
&-pairs {
display: flex;
gap: $padding-small;
&__label {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
&__value {
padding: $padding-small 0;
&:hover {
text-decoration: underline;
}
&:after {
content: ",";
}
&:last-child:after {
content: "";
}
}
&__total {
color: $color-text-secondary;
font-style: italic;
grid-column: 2;
}
}
&-info {
list-style-position: inside;
flex-grow: 1;
width: 100%;
display: flex;
align-items: center;
justify-content: space-between;
padding-top: $padding-small;
color: $color-text-secondary;
font-size: $font-size-small;
code {
display: inline-block;
padding: calc($padding-small / 2) $padding-small;
font-size: $font-size-small;
text-align: center;
background-color: $color-background-body;
background-repeat: repeat-x;
border: $border-divider;
border-radius: 4px;
}
}
}

View File

@@ -5,6 +5,7 @@ import { DATE_TIME_FORMAT } from "../../../../constants/date";
import classNames from "classnames";
import "./style.scss";
import "../../ChartTooltip/style.scss";
import { sortLogHits } from "../../../../utils/logs";
interface Props {
data: AlignedData;
@@ -26,7 +27,7 @@ const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
const tooltipItems = values.map((value, i) => {
const targetSeries = series[i + 1];
const stroke = (targetSeries?.stroke as () => string)?.();
const label = targetSeries?.label || "other";
const label = targetSeries?.label;
const show = targetSeries?.show;
return {
label,
@@ -34,7 +35,7 @@ const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
value,
show
};
}).filter(item => item.value > 0 && item.show).sort((a, b) => b.value - a.value);
}).filter(item => item.value > 0 && item.show).sort(sortLogHits("value"));
const point = {
top: tooltipItems[0] ? uPlotInst?.valToPos?.(tooltipItems[0].value, "y") || 0 : 0,
@@ -104,16 +105,19 @@ const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
className="vm-chart-tooltip-data__marker"
style={{ background: item.stroke }}
/>
<p>
{item.label}: <b>{item.value}</b>
<p className="vm-bar-hits-tooltip-item">
<span className="vm-bar-hits-tooltip-item__label">{item.label}</span>
<span>{item.value.toLocaleString("en-US")}</span>
</p>
</div>
))}
</div>
{tooltipData.values.length > 1 && (
<div className="vm-chart-tooltip-data">
<p>
Total records: <b>{tooltipData.total}</b>
<span/>
<p className="vm-bar-hits-tooltip-item">
<span className="vm-bar-hits-tooltip-item__label">Total</span>
<span>{tooltipData.total.toLocaleString("en-US")}</span>
</p>
</div>
)}

View File

@@ -9,4 +9,19 @@
opacity: 1;
pointer-events: auto;
}
&-item {
display: grid;
grid-template-columns: 1fr auto;
align-items: center;
gap: $padding-global;
max-width: 100%;
&__label {
display: inline-block;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
}
}

View File

@@ -0,0 +1,50 @@
import React, { FC } from "preact/compat";
import "./style.scss";
import { LegendLogHits } from "../../../../api/types";
import LegendHitsMenuStats from "./LegendHitsMenuStats";
import LegendHitsMenuBase from "./LegendHitsMenuBase";
import LegendHitsMenuRow from "./LegendHitsMenuRow";
import LegendHitsMenuFields from "./LegendHitsMenuFields";
import { LOGS_LIMIT_HITS } from "../../../../constants/logs";
const otherDescription = `aggregated results for fields not in the top ${LOGS_LIMIT_HITS}`;
interface Props {
legend: LegendLogHits;
fields: string[];
onApplyFilter: (value: string) => void;
onClose: () => void;
}
const LegendHitsMenu: FC<Props> = ({ legend, fields, onApplyFilter, onClose }) => {
return (
<div className="vm-legend-hits-menu">
<div className="vm-legend-hits-menu-section">
<LegendHitsMenuRow
className="vm-legend-hits-menu-row_info"
title={legend.isOther ? otherDescription : legend.label}
/>
</div>
{!legend.isOther && (
<LegendHitsMenuBase
legend={legend}
onApplyFilter={onApplyFilter}
onClose={onClose}
/>
)}
{!legend.isOther && (
<LegendHitsMenuFields
fields={fields}
onApplyFilter={onApplyFilter}
onClose={onClose}
/>
)}
<LegendHitsMenuStats legend={legend}/>
</div>
);
};
export default LegendHitsMenu;

View File

@@ -0,0 +1,64 @@
import React, { FC } from "preact/compat";
import LegendHitsMenuRow from "./LegendHitsMenuRow";
import useCopyToClipboard from "../../../../hooks/useCopyToClipboard";
import { CopyIcon, FilterIcon, FilterOffIcon } from "../../../Main/Icons";
import { LegendLogHits, LegendLogHitsMenu } from "../../../../api/types";
import { LOGS_GROUP_BY } from "../../../../constants/logs";
interface Props {
legend: LegendLogHits;
onApplyFilter: (value: string) => void;
onClose: () => void;
}
const LegendHitsMenuBase: FC<Props> = ({ legend, onApplyFilter, onClose }) => {
const copyToClipboard = useCopyToClipboard();
const handleAddStreamToFilter = () => {
onApplyFilter(`${LOGS_GROUP_BY}: ${legend.label}`);
onClose();
};
const handleExcludeStreamToFilter = () => {
onApplyFilter(`(NOT ${LOGS_GROUP_BY}: ${legend.label})`);
onClose();
};
const handlerCopyLabel = async () => {
await copyToClipboard(legend.label, `${legend.label} has been copied`);
onClose();
};
const options: LegendLogHitsMenu[] = [
{
title: `Copy ${LOGS_GROUP_BY} name`,
icon: <CopyIcon/>,
handler: handlerCopyLabel,
},
{
title: `Add ${LOGS_GROUP_BY} to filter`,
icon: <FilterIcon/>,
handler: handleAddStreamToFilter,
},
{
title: `Exclude ${LOGS_GROUP_BY} to filter`,
icon: <FilterOffIcon/>,
handler: handleExcludeStreamToFilter,
}
];
return (
<div className="vm-legend-hits-menu-section">
{options.map(({ icon, title, handler }) => (
<LegendHitsMenuRow
key={title}
iconStart={icon}
title={title}
handler={handler}
/>
))}
</div>
);
};
export default LegendHitsMenuBase;

View File

@@ -0,0 +1,74 @@
import React, { FC, useMemo } from "preact/compat";
import LegendHitsMenuRow from "./LegendHitsMenuRow";
import { CopyIcon, FilterIcon, FilterOffIcon } from "../../../Main/Icons";
import { convertToFieldFilter } from "../../../../utils/logs";
import { LegendLogHitsMenu } from "../../../../api/types";
import useCopyToClipboard from "../../../../hooks/useCopyToClipboard";
interface Props {
fields: string[];
onApplyFilter: (value: string) => void;
onClose: () => void;
}
const LegendHitsMenuFields: FC<Props> = ({ fields, onApplyFilter, onClose }) => {
const copyToClipboard = useCopyToClipboard();
const handleCopy = (field: string) => async () => {
await copyToClipboard(field, `${field} has been copied`);
onClose();
};
const handleAddToFilter = (field: string) => () => {
onApplyFilter(field);
onClose();
};
const handleExcludeToFilter = (field: string) => () => {
onApplyFilter(`-${field}`);
onClose();
};
const generateFieldMenu = (field: string): LegendLogHitsMenu[] => {
return [
{
title: "Copy",
icon: <CopyIcon/>,
handler: handleCopy(field),
},
{
title: "Add to filter",
icon: <FilterIcon/>,
handler: handleAddToFilter(field),
},
{
title: "Exclude to filter",
icon: <FilterOffIcon/>,
handler: handleExcludeToFilter(field),
}
];
};
const fieldsWithMenu: LegendLogHitsMenu[] = useMemo(() => {
return fields.map(field => {
const title = convertToFieldFilter(field);
return {
title,
submenu: generateFieldMenu(title),
};
});
}, [fields]);
return (
<div className="vm-legend-hits-menu-section">
{fieldsWithMenu?.map((field) => (
<LegendHitsMenuRow
key={field.title}
{...field}
/>
))}
</div>
);
};
export default LegendHitsMenuFields;

View File

@@ -0,0 +1,116 @@
import React, { FC, useRef, useState } from "preact/compat";
import classNames from "classnames";
import { ReactNode, useEffect } from "react";
import Tooltip from "../../../Main/Tooltip/Tooltip";
import { LegendLogHitsMenu } from "../../../../api/types";
import { ArrowDropDownIcon } from "../../../Main/Icons";
import useClickOutside from "../../../../hooks/useClickOutside";
interface Props {
title: string | ReactNode;
handler?: () => void;
iconStart?: ReactNode;
iconEnd?: ReactNode;
className?: string;
submenu?: LegendLogHitsMenu[];
}
const LegendHitsMenuRow: FC<Props> = ({ title, handler, iconStart, iconEnd, className, submenu }) => {
const containerRef = useRef<HTMLDivElement>(null);
const titleRef = useRef<HTMLDivElement>(null);
const submenuRef = useRef<HTMLDivElement>(null);
const [isOverflownTitle, setIsOverflownTitle] = useState(false);
const [openSubmenu, setOpenSubmenu] = useState(false);
const [posSubmenuLeft, setPosSubmenuLeft] = useState(false);
const hasSubmenu = !!submenu?.length;
const handleToggleContextMenu = () => {
setOpenSubmenu(prev => !prev);
};
const handleCloseContextMenu = () => {
setOpenSubmenu(false);
};
const handleClick = () => {
handler && handler();
hasSubmenu && handleToggleContextMenu();
};
useEffect(() => {
if (!titleRef.current) return;
setIsOverflownTitle(titleRef.current.scrollWidth > titleRef.current.clientWidth);
}, [title, titleRef]);
useEffect(() => {
requestAnimationFrame(() => {
if (!openSubmenu || !submenuRef.current) {
setPosSubmenuLeft(false);
return;
}
const { left, width } = submenuRef.current.getBoundingClientRect();
setPosSubmenuLeft(left + width > window.innerWidth);
});
}, [submenuRef, openSubmenu]);
useClickOutside(containerRef, handleCloseContextMenu);
const titleContent = (
<div
ref={titleRef}
className="vm-legend-hits-menu-row__title"
>
{title}
</div>
);
return (
<div
ref={containerRef}
className={classNames({
"vm-legend-hits-menu-row": true,
"vm-legend-hits-menu-row_interactive": !!handler || hasSubmenu,
[`${className}`]: className
})}
onClick={handleClick}
>
{iconStart && <div className="vm-legend-hits-menu-row__icon">{iconStart}</div>}
{isOverflownTitle ? (<Tooltip title={title}>{titleContent}</Tooltip>) : titleContent}
{iconEnd && !hasSubmenu && <div className="vm-legend-hits-menu-row__icon">{iconEnd}</div>}
{hasSubmenu && (
<div className="vm-legend-hits-menu-row__icon vm-legend-hits-menu-row__icon_drop">
<ArrowDropDownIcon/>
</div>
)}
{openSubmenu && submenu && (
<div
ref={submenuRef}
className={classNames({
"vm-legend-hits-menu": true,
"vm-legend-hits-menu_submenu": true,
"vm-legend-hits-menu_submenu_left": posSubmenuLeft
})}
>
<div className="vm-legend-hits-menu-section">
{submenu.map(({ icon, title, handler }) => (
<LegendHitsMenuRow
key={title}
iconStart={icon}
title={title}
handler={handler}
/>
))}
</div>
</div>
)}
</div>
);
};
export default LegendHitsMenuRow;

View File

@@ -0,0 +1,23 @@
import React, { FC } from "preact/compat";
import { LegendLogHits } from "../../../../api/types";
interface Props {
legend: LegendLogHits;
}
const LegendHitsMenuStats: FC<Props> = ({ legend }) => {
const totalFormatted = legend.total.toLocaleString("en-US");
const percentage = Math.round((legend.total / legend.totalHits) * 100);
return (
<div className="vm-legend-hits-menu-section">
<div className="vm-legend-hits-menu-row">
<div className="vm-legend-hits-menu-row__title">
Total: {totalFormatted} ({percentage}%)
</div>
</div>
</div>
);
};
export default LegendHitsMenuStats;

View File

@@ -0,0 +1,178 @@
@use "src/styles/variables" as *;
.vm-legend-hits-menu {
min-width: 160px;
z-index: 1;
&_submenu {
position: absolute;
top: calc(-1 * $padding-small);
background-color: $color-background-block;
left: calc(100% + ($padding-small / 2));
box-shadow: $box-shadow-popper;
border-radius: $border-radius-small;
animation: vm-submenu-show 150ms cubic-bezier(0.280, 0.840, 0.2, 1);
transform-origin: top left;
&_left {
left: auto;
right: calc(100% + ($padding-small / 2));
transform-origin: top right;
}
}
&-section {
border-bottom: $border-divider;
&:last-child {
border-bottom: none;
}
}
&-row {
position: relative;
display: flex;
gap: $padding-small;
align-items: center;
justify-content: flex-start;
padding: 0 $padding-global;
transition: background-color 0.3s;
color: $color-text;
&_interactive {
cursor: pointer;
&:hover {
background-color: rgba(0, 0, 0, 0.05);
}
}
&_info {
font-size: $font-size-small;
font-weight: 500;
padding-block: $padding-small;
}
&_info &__icon {
color: $color-info;
}
&__icon {
display: flex;
align-items: center;
justify-content: center;
width: 14px;
height: 14px;
&_drop {
transform: rotate(-90deg);
}
}
&__title {
flex-grow: 1;
padding: $padding-global 0;
position: relative;
max-width: 400px;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
}
&-other-list {
width: 80vw;
height: 80vh;
overflow: auto;
&__search {
position: sticky;
top: 0;
padding: $padding-small 0;
background-color: $color-background-block;
border-bottom: $border-divider;
z-index: 2;
}
&-row {
border-bottom: $border-divider;
&_header {
border-bottom: none;
position: sticky;
top: 65px;
background-color: $color-background-block;
z-index: 1;
width: 100%;
&:after {
content: '';
position: absolute;
bottom: 0;
left: 0;
width: 100%;
height: 1px;
border-bottom: $border-divider;
}
}
}
&-cell {
padding: calc($padding-small / 2) 0;
text-align: left;
&_header {
padding: $padding-small;
font-weight: 500;
}
&_number {
padding: $padding-small;
text-align: right;
font-variant-numeric: tabular-nums;
}
&_fields {
width: 100%;
}
}
&-fields {
display: flex;
flex-wrap: wrap;
align-items: center;
justify-content: flex-start;
&__field {
padding: calc($padding-small / 2) $padding-small;
border-radius: $border-radius-small;
transition: background-color 0.3s;
&:hover {
background-color: $color-hover-black;
}
&:not(:last-child) {
&:after {
content: ',';
}
}
}
}
&-actions {
display: flex;
align-items: center;
justify-content: center;
}
}
}
@keyframes vm-submenu-show {
0% {
opacity: 0;
}
100% {
opacity: 1;
}
}

View File

@@ -36,6 +36,14 @@ interface UseGetBarHitsOptionsArgs {
graphOptions: GraphOptions;
}
export const OTHER_HITS_LABEL = "other";
export const getLabelFromLogHit = (logHit: LogHits) => {
if (logHit?._isOther) return OTHER_HITS_LABEL;
const fields = Object.values(logHit?.fields || {});
return fields.map((value) => value || "\"\"").join(", ");
};
const useBarHitsOptions = ({
data,
logHits,
@@ -59,12 +67,12 @@ const useBarHitsOptions = ({
let colorN = 0;
return data.map((_d, i) => {
if (i === 0) return {}; // 0 index is xAxis(timestamps)
const fields = Object.values(logHits?.[i - 1]?.fields || {});
const label = fields.map((value) => value || "\"\"").join(", ");
const color = getCssVariable(label ? seriesColors[colorN] : "color-log-hits-bar-0");
if (label) colorN++;
const target = logHits?.[i - 1];
const label = getLabelFromLogHit(target);
const color = getCssVariable(target?._isOther ? "color-log-hits-bar-0" : seriesColors[colorN]);
if (!target?._isOther) colorN++;
return {
label: label || "other",
label,
width: strokeWidth[graphOptions.graphStyle],
spanGaps: true,
stroke: color,

View File

@@ -32,6 +32,11 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
max-width: calc(100vw/3);
}
&_hits &-data {
display: grid;
grid-template-columns: $font-size 1fr;
}
&_sticky {
pointer-events: auto;
z-index: 99;
@@ -90,6 +95,8 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
}
&__marker {
min-width: $font-size;
max-width: $font-size;
width: $font-size;
height: $font-size;
border: 1px solid rgba($color-white, 0.5);

View File

@@ -581,3 +581,45 @@ export const CommentIcon = () => (
></path>
</svg>
);
export const FilterIcon = () => (
<svg
viewBox="0 0 24 24"
fill="currentColor"
>
<path
d="M4.25 5.61C6.27 8.2 10 13 10 13v6c0 .55.45 1 1 1h2c.55 0 1-.45 1-1v-6s3.72-4.8 5.74-7.39c.51-.66.04-1.61-.79-1.61H5.04c-.83 0-1.3.95-.79 1.61"
></path>
</svg>
);
export const FilterOffIcon = () => (
<svg
viewBox="0 0 24 24"
fill="currentColor"
>
<path
d="M19.79 5.61C20.3 4.95 19.83 4 19 4H6.83l7.97 7.97zM2.81 2.81 1.39 4.22 10 13v6c0 .55.45 1 1 1h2c.55 0 1-.45 1-1v-2.17l5.78 5.78 1.41-1.41z"
></path>
</svg>
);
export const OpenNewIcon = () => (
<svg
viewBox="0 0 24 24"
fill="currentColor"
>
<path
d="M19 19H5V5h7V3H5c-1.11 0-2 .9-2 2v14c0 1.1.89 2 2 2h14c1.1 0 2-.9 2-2v-7h-2zM14 3v2h3.59l-9.83 9.83 1.41 1.41L19 6.41V10h2V3z"
></path>
</svg>
);
export const ModalIcon = () => (
<svg
viewBox="0 0 24 24"
fill="currentColor"
>
<path d="M19 4H5c-1.11 0-2 .9-2 2v12c0 1.1.89 2 2 2h14c1.1 0 2-.9 2-2V6c0-1.1-.89-2-2-2m0 14H5V8h14z"></path>
</svg>
);

View File

@@ -15,9 +15,10 @@ interface PopperProps {
open: boolean
onClose: () => void
buttonRef: React.RefObject<HTMLElement>
placement?: "bottom-right" | "bottom-left" | "top-left" | "top-right"
placement?: "bottom-right" | "bottom-left" | "top-left" | "top-right" | "fixed"
placementPosition?: { top: number, left: number } | null
animation?: string
offset?: {top: number, left: number}
offset?: { top: number, left: number }
clickOutside?: boolean,
fullWidth?: boolean
title?: string
@@ -29,6 +30,7 @@ const Popper: FC<PopperProps> = ({
children,
buttonRef,
placement = "bottom-left",
placementPosition,
open = false,
onClose,
offset = { top: 6, left: 0 },
@@ -92,13 +94,18 @@ const Popper: FC<PopperProps> = ({
if (needAlignRight) position.left = buttonPos.right - popperSize.width;
if (needAlignTop) position.top = buttonPos.top - popperSize.height - offsetTop;
const { innerWidth, innerHeight } = window;
const margin = 20;
if (placement === "fixed" && placementPosition) {
position.top = Math.max(placementPosition.top + offset.top, 0);
position.left = Math.max(placementPosition.left + offset.left, 0);
return position;
}
const isOverflowBottom = (position.top + popperSize.height + margin) > innerHeight;
const isOverflowTop = (position.top - margin) < 0;
const isOverflowRight = (position.left + popperSize.width + margin) > innerWidth;
const isOverflowLeft = (position.left - margin) < 0;
const { innerWidth, innerHeight } = window;
const isOverflowBottom = (position.top + popperSize.height) > innerHeight;
const isOverflowTop = (position.top) < 0;
const isOverflowRight = (position.left + popperSize.width) > innerWidth;
const isOverflowLeft = (position.left) < 0;
if (isOverflowBottom) position.top = buttonPos.top - popperSize.height - offsetTop;
if (isOverflowTop) position.top = buttonPos.height + buttonPos.top + offsetTop;
@@ -106,11 +113,11 @@ const Popper: FC<PopperProps> = ({
if (isOverflowLeft) position.left = buttonPos.left + offsetLeft;
if (fullWidth) position.width = `${buttonPos.width}px`;
if (position.top < 0) position.top = 20;
if (position.left < 0) position.left = 20;
if (position.top < 0) position.top = 0;
if (position.left < 0) position.left = 0;
return position;
},[buttonRef, placement, isOpen, children, fullWidth]);
}, [buttonRef, placement, isOpen, children, fullWidth]);
const handleClickClose = (e: ReactMouseEvent<HTMLButtonElement, MouseEvent>) => {
e.stopPropagation();
@@ -131,10 +138,10 @@ const Popper: FC<PopperProps> = ({
if (!popperRef.current || !isOpen || (isMobile && !disabledFullScreen)) return;
const { right, width } = popperRef.current.getBoundingClientRect();
if (right > window.innerWidth) {
const left = window.innerWidth - 20 - width;
popperRef.current.style.left = left < window.innerWidth ? "0" : `${left}px`;
const left = window.innerWidth - width;
popperRef.current.style.left = `${left}px`;
}
}, [isOpen, popperRef]);
}, [isOpen, popperRef, placementPosition]);
const handlePopstate = useCallback(() => {
if (isOpen && isMobile && !disabledFullScreen) {

View File

@@ -11,7 +11,7 @@
border-radius: $border-radius-small;
&_open {
z-index: 101;
z-index: 100;
opacity: 1;
transform-origin: top center;
animation: vm-slider 150ms cubic-bezier(0.280, 0.840, 0.420, 1.1);

View File

@@ -2,6 +2,7 @@ import { DATE_TIME_FORMAT } from "./date";
export const LOGS_ENTRIES_LIMIT = 50;
export const LOGS_BARS_VIEW = 100;
export const LOGS_LIMIT_HITS = 5;
// "Ungrouped" is a string that is used as a value for the "groupBy" parameter.
export const WITHOUT_GROUPING = "Ungrouped";

View File

@@ -69,7 +69,7 @@ const ExploreLogs: FC = () => {
};
const handleApplyFilter = (val: string) => {
setQuery(prev => `_stream: ${val === "other" ? "{}" : val} AND (${prev})`);
setQuery(prev => `${val} AND (${prev})`);
setIsUpdatingQuery(true);
};

View File

@@ -43,7 +43,7 @@ const GroupLogs: FC<Props> = ({ logs, settingsRef }) => {
values,
pairs,
};
}).sort((a, b) => a.keysString.localeCompare(b.keysString)); // groups sorting
}).sort((a, b) => b.values.length - a.values.length); // groups sorting
}, [logs, groupBy]);
const handleToggleExpandAll = useCallback(() => {

View File

@@ -6,6 +6,7 @@ import { MouseEvent, useState } from "react";
import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
import { useSearchParams } from "react-router-dom";
import { LOGS_GROUP_BY, LOGS_URL_PARAMS } from "../../../constants/logs";
import { convertToFieldFilter } from "../../../utils/logs";
interface Props {
pair: string;
@@ -23,8 +24,7 @@ const GroupLogsHeaderItem: FC<Props> = ({ pair, isHide }) => {
const handleClickByPair = (value: string) => async (e: MouseEvent<HTMLDivElement>) => {
e.stopPropagation();
const isKeyValue = /(.+)?=(".+")/.test(value);
const copyValue = isKeyValue ? `${value.replace(/=/, ": ")}` : `${groupBy}: "${value}"`;
const copyValue = convertToFieldFilter(value, groupBy);
const isCopied = await copyToClipboard(copyValue);
if (isCopied) {
setCopied(value);

View File

@@ -268,7 +268,7 @@ $font-size-logs: var(--font-size-logs, $font-size-small);
border: $border-divider;
border-radius: $border-radius-small;
overflow: auto;
height: 300px;
max-height: 300px;
resize: vertical;
font-family: $font-family-monospace;
font-size: $font-size-logs;

View File

@@ -4,6 +4,8 @@ import { ErrorTypes, TimeParams } from "../../../types";
import { LogHits } from "../../../api/types";
import { useSearchParams } from "react-router-dom";
import { getHitsTimeParams } from "../../../utils/logs";
import { LOGS_GROUP_BY, LOGS_LIMIT_HITS } from "../../../constants/logs";
import { isEmptyObject } from "../../../utils/object";
export const useFetchLogHits = (server: string, query: string) => {
const [searchParams] = useSearchParams();
@@ -30,46 +32,12 @@ export const useFetchLogHits = (server: string, query: string) => {
step: `${step}ms`,
start: start.toISOString(),
end: end.toISOString(),
field: "_stream" // In the future, this field can be made configurable
fields_limit: `${LOGS_LIMIT_HITS}`,
field: LOGS_GROUP_BY,
})
};
};
const accumulateHits = (resultHit: LogHits, hit: LogHits) => {
resultHit.total = (resultHit.total || 0) + (hit.total || 0);
hit.timestamps.forEach((timestamp, i) => {
const index = resultHit.timestamps.findIndex(t => t === timestamp);
if (index === -1) {
resultHit.timestamps.push(timestamp);
resultHit.values.push(hit.values[i]);
} else {
resultHit.values[index] += hit.values[i];
}
});
return resultHit;
};
const getHitsWithTop = (hits: LogHits[]) => {
const topN = 5;
const defaultHit = { fields: {}, timestamps: [], values: [], total: 0 };
const hitsByTotal = hits.sort((a, b) => (b.total || 0) - (a.total || 0));
const result = [];
const otherHits: LogHits = hitsByTotal.slice(topN).reduce(accumulateHits, defaultHit);
if (otherHits.total) {
result.push(otherHits);
}
const topHits: LogHits[] = hitsByTotal.slice(0, topN);
if (topHits.length) {
result.push(...topHits);
}
return result;
};
const fetchLogHits = useCallback(async (period: TimeParams) => {
abortControllerRef.current.abort();
abortControllerRef.current = new AbortController();
@@ -98,7 +66,7 @@ export const useFetchLogHits = (server: string, query: string) => {
setError(error);
}
setLogHits(!hits ? [] : getHitsWithTop(hits));
setLogHits(hits.map(hit => ({ ...hit, _isOther: isEmptyObject(hit.fields) })));
} catch (e) {
if (e instanceof Error && e.name !== "AbortError") {
setError(String(e));

View File

@@ -1,6 +1,8 @@
import { TimeParams } from "../types";
import dayjs from "dayjs";
import { LOGS_BARS_VIEW } from "../constants/logs";
import { LOGS_BARS_VIEW, LOGS_GROUP_BY } from "../constants/logs";
import { LogHits } from "../api/types";
import { OTHER_HITS_LABEL } from "../components/Chart/BarHitsChart/hooks/useBarHitsOptions";
export const getStreamPairs = (value: string): string[] => {
const pairs = /^{.+}$/.test(value) ? value.slice(1, -1).split(",") : [value];
@@ -14,3 +16,27 @@ export const getHitsTimeParams = (period: TimeParams) => {
const step = Math.ceil(totalSeconds / LOGS_BARS_VIEW) || 1;
return { start, end, step };
};
export const convertToFieldFilter = (value: string, field = LOGS_GROUP_BY) => {
const isKeyValue = /(.+)?=(".+")/.test(value);
if (isKeyValue) {
return value.replace(/=/, ": ");
}
return `${field}: "${value}"`;
};
export const calculateTotalHits = (hits: LogHits[]): number => {
return hits.reduce((acc, item) => acc + (item.total || 0), 0);
};
export const sortLogHits = <T extends { label?: string }>(key: keyof T) => (a: T, b: T): number => {
if (a.label === OTHER_HITS_LABEL) return 1;
if (b.label === OTHER_HITS_LABEL) return -1;
const aValue = a[key] as unknown as number;
const bValue = b[key] as unknown as number;
return bValue - aValue;
};

View File

@@ -57,3 +57,15 @@ export const getLastFromArray = (a: number[]) => {
}
}
};
export const formatNumberShort = (value: number) => {
if (value >= 1_000_000_000) {
return (value / 1_000_000_000).toFixed(1).replace(/\.0$/, "") + "B"; // Миллиарды
} else if (value >= 1_000_000) {
return (value / 1_000_000).toFixed(1).replace(/\.0$/, "") + "M"; // Миллионы
} else if (value >= 1_000) {
return (value / 1_000).toFixed(1).replace(/\.0$/, "") + "K"; // Тысячи
} else {
return value.toString(); // Для чисел меньше 1000
}
};

View File

@@ -14,3 +14,7 @@ export function filterObject<T extends object>(
export function compactObject<T extends object>(obj: T) {
return filterObject(obj, (entry) => !!entry[1] || typeof entry[1] === "number");
}
export function isEmptyObject(obj: object) {
return Object.keys(obj).length === 0;
}

View File

@@ -0,0 +1,76 @@
package tests
import (
"fmt"
"math/rand/v2"
"testing"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
)
func TestStorageUsesSparseCacheForFinalMerge(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
sut := tc.MustStartVmsingle("sparse-cache-final-merge", []string{`-retentionPeriod=100y`, `-downsampling.period={__name__=~"metric.*"}:5m:1s`})
// insert metrics daily over past 2 months
const metricsPerStep = 10
const steps = 35
const stepSize = int64(1 * 24 * 60 * 60 * 1000)
records := make([]string, metricsPerStep)
ts := time.Now().Add(-2 * 30 * 24 * time.Hour).UnixMilli()
for range steps {
for i := range metricsPerStep {
name := fmt.Sprintf("metric_%d", i)
records[i] = fmt.Sprintf("%s %d %d", name, rand.IntN(1000), ts)
}
sut.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{})
ts += stepSize
}
sut.ForceFlush(t)
sut.ForceMerge(t)
// todo: replace with a more reliable way to check if the merge is completed
// wait for merge to be completed
time.Sleep(5 * time.Second)
v := sut.GetIntMetric(t, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`)
if v <= 0 {
t.Fatalf(`unexpected vm_cache_requests_total{type="indexdb/dataBlocksSparse"} value: %d`, v)
}
}
func TestStorageDoesNotUseSparseCacheForRegularMerge(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
sut := tc.MustStartVmsingle("sparse-cache-regular-merge", []string{`-retentionPeriod=100y`, `-downsampling.period={__name__=~"metric.*"}:5m:1s`})
// insert metrics into current month only
const metricsPerStep = 10
const steps = 2
const stepSize = 60 * 1000
records := make([]string, metricsPerStep)
ts := time.Now().Add(-1 * time.Hour).UnixMilli()
for range steps {
for i := range metricsPerStep {
name := fmt.Sprintf("metric_%d", i)
records[i] = fmt.Sprintf("%s %d %d", name, rand.IntN(1000), ts)
}
sut.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{})
ts += stepSize
}
sut.ForceFlush(t)
sut.ForceMerge(t)
// todo: replace with a more reliable way to check if the merge is completed
// wait for merge to be completed
time.Sleep(5 * time.Second)
v := sut.GetIntMetric(t, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`)
if v == 0 {
t.Fatalf(`unexpected vm_cache_requests_total{type="indexdb/dataBlocksSparse"} value: %d`, v)
}
}

View File

@@ -24,6 +24,7 @@ type Vmsingle struct {
// vmstorage URLs.
forceFlushURL string
forceMergeURL string
// vminsert URLs.
influxLineWriteURL string
@@ -65,6 +66,7 @@ func StartVmsingle(instance string, flags []string, cli *Client) (*Vmsingle, err
httpListenAddr: stderrExtracts[1],
forceFlushURL: fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
forceMergeURL: fmt.Sprintf("http://%s/internal/force_merge", stderrExtracts[1]),
influxLineWriteURL: fmt.Sprintf("http://%s/influx/write", stderrExtracts[1]),
prometheusAPIV1ImportPrometheusURL: fmt.Sprintf("http://%s/prometheus/api/v1/import/prometheus", stderrExtracts[1]),
prometheusAPIV1WriteURL: fmt.Sprintf("http://%s/prometheus/api/v1/write", stderrExtracts[1]),
@@ -83,6 +85,13 @@ func (app *Vmsingle) ForceFlush(t *testing.T) {
app.cli.Get(t, app.forceFlushURL, http.StatusOK)
}
// ForceMerge is a test helper function that forces the merging of parts.
func (app *Vmsingle) ForceMerge(t *testing.T) {
t.Helper()
app.cli.Get(t, app.forceMergeURL, http.StatusOK)
}
// InfluxWrite is a test helper function that inserts a
// collection of records in Influx line format by sending a HTTP
// POST request to /influx/write vmsingle endpoint.

View File

@@ -21,6 +21,7 @@ type Vmstorage struct {
vmselectAddr string
forceFlushURL string
forceMergeURL string
}
// StartVmstorage starts an instance of vmstorage with the given flags. It also
@@ -57,6 +58,7 @@ func StartVmstorage(instance string, flags []string, cli *Client) (*Vmstorage, e
vmselectAddr: stderrExtracts[3],
forceFlushURL: fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
forceMergeURL: fmt.Sprintf("http://%s/internal/force_merge", stderrExtracts[1]),
}, nil
}
@@ -80,6 +82,12 @@ func (app *Vmstorage) ForceFlush(t *testing.T) {
app.cli.Get(t, app.forceFlushURL, http.StatusOK)
}
// ForceMerge is a test helper function that forces the merging of parts.
func (app *Vmstorage) ForceMerge(t *testing.T) {
t.Helper()
app.cli.Get(t, app.forceMergeURL, http.StatusOK)
}
// String returns the string representation of the vmstorage app state.
func (app *Vmstorage) String() string {
return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q vminsertAddr: %q vmselectAddr: %q}", []any{

View File

@@ -1686,7 +1686,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -1933,7 +1933,7 @@
},
"editorMode": "code",
"exemplar": true,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -1981,7 +1981,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -1687,7 +1687,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -1934,7 +1934,7 @@
},
"editorMode": "code",
"exemplar": true,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -1982,7 +1982,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -1666,7 +1666,7 @@
},
"editorMode": "code",
"exemplar": true,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -1473,7 +1473,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / \n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / \n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -1361,7 +1361,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -1665,7 +1665,7 @@
},
"editorMode": "code",
"exemplar": true,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -1472,7 +1472,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / \n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / \n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -1360,7 +1360,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

View File

@@ -4,7 +4,7 @@ services:
# And forward them to --remoteWrite.url
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.109.1
image: victoriametrics/vmagent:v1.110.0
depends_on:
- "vminsert"
ports:
@@ -39,7 +39,7 @@ services:
# where N is number of vmstorages (2 in this case).
vmstorage-1:
container_name: vmstorage-1
image: victoriametrics/vmstorage:v1.109.1-cluster
image: victoriametrics/vmstorage:v1.110.0-cluster
ports:
- 8482
- 8400
@@ -51,7 +51,7 @@ services:
restart: always
vmstorage-2:
container_name: vmstorage-2
image: victoriametrics/vmstorage:v1.109.1-cluster
image: victoriametrics/vmstorage:v1.110.0-cluster
ports:
- 8482
- 8400
@@ -66,7 +66,7 @@ services:
# pre-process them and distributes across configured vmstorage shards.
vminsert:
container_name: vminsert
image: victoriametrics/vminsert:v1.109.1-cluster
image: victoriametrics/vminsert:v1.110.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -81,7 +81,7 @@ services:
# vmselect collects results from configured `--storageNode` shards.
vmselect-1:
container_name: vmselect-1
image: victoriametrics/vmselect:v1.109.1-cluster
image: victoriametrics/vmselect:v1.110.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -94,7 +94,7 @@ services:
restart: always
vmselect-2:
container_name: vmselect-2
image: victoriametrics/vmselect:v1.109.1-cluster
image: victoriametrics/vmselect:v1.110.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -112,7 +112,7 @@ services:
# It can be used as an authentication proxy.
vmauth:
container_name: vmauth
image: victoriametrics/vmauth:v1.109.1
image: victoriametrics/vmauth:v1.110.0
depends_on:
- "vmselect-1"
- "vmselect-2"
@@ -127,7 +127,7 @@ services:
# vmalert executes alerting and recording rules
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.109.1
image: victoriametrics/vmalert:v1.110.0
depends_on:
- "vmauth"
ports:

View File

@@ -45,7 +45,7 @@ services:
# storing logs and serving read queries.
victorialogs:
container_name: victorialogs
image: victoriametrics/victoria-logs:v1.7.0-victorialogs
image: victoriametrics/victoria-logs:v1.8.0-victorialogs
command:
- "--storageDataPath=/vlogs"
- "--httpListenAddr=:9428"
@@ -60,7 +60,7 @@ services:
# scraping, storing metrics and serve read requests.
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.109.1
image: victoriametrics/victoria-metrics:v1.110.0
ports:
- 8428:8428
volumes:
@@ -79,7 +79,7 @@ services:
# depending on the requested path.
vmauth:
container_name: vmauth
image: victoriametrics/vmauth:v1.109.1
image: victoriametrics/vmauth:v1.110.0
depends_on:
- "victoriametrics"
- "victorialogs"
@@ -96,7 +96,7 @@ services:
# vmalert executes alerting and recording rules according to given rule type.
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.109.1
image: victoriametrics/vmalert:v1.110.0
depends_on:
- "vmauth"
- "alertmanager"

View File

@@ -4,7 +4,7 @@ services:
# And forward them to --remoteWrite.url
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.109.1
image: victoriametrics/vmagent:v1.110.0
depends_on:
- "victoriametrics"
ports:
@@ -22,7 +22,7 @@ services:
# storing metrics and serve read requests.
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.109.1
image: victoriametrics/victoria-metrics:v1.110.0
ports:
- 8428:8428
- 8089:8089
@@ -65,7 +65,7 @@ services:
# vmalert executes alerting and recording rules
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.109.1
image: victoriametrics/vmalert:v1.110.0
depends_on:
- "victoriametrics"
- "alertmanager"

View File

@@ -1,7 +1,7 @@
services:
# meta service will be ignored by compose
.victorialogs:
image: docker.io/victoriametrics/victoria-logs:v1.7.0-victorialogs
image: docker.io/victoriametrics/victoria-logs:v1.8.0-victorialogs
command:
- -storageDataPath=/vlogs
- -loggerFormat=json
@@ -19,7 +19,7 @@ services:
retries: 10
dd-proxy:
image: docker.io/victoriametrics/vmauth:v1.109.1
image: docker.io/victoriametrics/vmauth:v1.110.0
restart: on-failure
volumes:
- ./:/etc/vmauth
@@ -45,7 +45,7 @@ services:
replicas: 0
victoriametrics:
image: victoriametrics/victoria-metrics:v1.109.1
image: victoriametrics/victoria-metrics:v1.110.0
ports:
- '8428:8428'
command:

View File

@@ -1,7 +1,7 @@
services:
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.109.1
image: victoriametrics/vmagent:v1.110.0
depends_on:
- "victoriametrics"
ports:
@@ -18,7 +18,7 @@ services:
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.109.1
image: victoriametrics/victoria-metrics:v1.110.0
ports:
- 8428:8428
volumes:
@@ -50,7 +50,7 @@ services:
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.109.1
image: victoriametrics/vmalert:v1.110.0
depends_on:
- "victoriametrics"
ports:
@@ -72,7 +72,7 @@ services:
restart: always
vmanomaly:
container_name: vmanomaly
image: victoriametrics/vmanomaly:v1.19.1
image: victoriametrics/vmanomaly:v1.19.2
depends_on:
- "victoriametrics"
ports:

View File

@@ -3,7 +3,7 @@ version: "3"
services:
# Run `make package-victoria-logs` to build victoria-logs image
vlogs:
image: docker.io/victoriametrics/victoria-logs:v1.7.0-victorialogs
image: docker.io/victoriametrics/victoria-logs:v1.8.0-victorialogs
volumes:
- vlogs:/vlogs
ports:

View File

@@ -86,6 +86,7 @@ See also [case studies](https://docs.victoriametrics.com/casestudies/).
* [Persistent Data Structures in VictoriaMetrics (Part 2): vmselect](https://medium.com/@jiekun/persistent-data-structures-in-victoriametrics-part-2-vmselect-9e3de39a4d20)
* [Migrating to VictoriaMetrics (by Zomato): A Complete Overhaul for Enhanced Observability](https://blog.zomato.com/migrating-to-victoriametrics-a-complete-overhaul-for-enhanced-observability)
* [Harness the Power of VictoriaMetrics and Grafana Operators for Metrics Management](https://blog.ogenki.io/post/series/observability/metrics/)
* [Reducing Inter-AZ traffic in VictoriaMetrics with Zonekeeper](https://tanmay-bhat.medium.com/reducing-inter-az-traffic-in-victoriametrics-with-zonekeeper-3bd7e1526796)
## Our articles

View File

@@ -48,6 +48,21 @@ VictoriaMetrics can run also on MacOS for testing and development purposes.
* **MacOS**: amd64, arm64 (for testing and development purposes)
* **Windows**: amd64
## Kubernetes
VictoriaMetrics natively supports deployment in Kubernetes via [helm charts](https://docs.victoriametrics.com/helm/)
and [kubernetes operator](https://docs.victoriametrics.com/operator/). See how to [start using k8s operator](https://docs.victoriametrics.com/guides/getting-started-with-vm-operator/).
Common recommendations:
1. Prefer setting [requests equal to limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits)
for stateful components like [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/#architecture-overview) to avoid unnecessary
component restarts.
1. Avoid using [fractional CPU units](https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units)
when setting resources for optimal performance. VictoriaMetrics is written in Go and its runtime requires specifying
[integer number](https://pkg.go.dev/runtime#GOMAXPROCS) of concurrently running threads.
When fractional CPU unit is specified, VictoriaMetrics will automatically round it down.
## Upgrade procedure
It is safe to upgrade VictoriaMetrics to new versions unless the [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) say otherwise.

View File

@@ -22,5 +22,5 @@ to [the latest available releases](https://docs.victoriametrics.com/changelog/).
## Currently supported LTS release lines
- v1.102.x - the latest one is [v1.102.10 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.10)
- v1.97.x - the latest one is [v1.97.15 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.15)
- v1.102.x - the latest one is [v1.102.11 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.11)
- v1.97.x - the latest one is [v1.97.16 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.16)

View File

@@ -55,8 +55,8 @@ under the current directory:
```sh
docker pull victoriametrics/victoria-metrics:v1.109.1
docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 victoriametrics/victoria-metrics:v1.109.1
docker pull victoriametrics/victoria-metrics:v1.110.0
docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 victoriametrics/victoria-metrics:v1.110.0
```

View File

@@ -16,9 +16,28 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
## tip
* FEATURE: [`block_stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#block_stats-pipe): return the path to the part where every data block is stored. The path to the part is returned in the `part_path` field. This allows investigating the distribution of data blocks among parts.
* FEATURE: reduce VictoriaLogs startup time by multiple times when it opens a large datastore with big [retention](https://docs.victoriametrics.com/victorialogs/#retention).
* BUGFIX: [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/): drop log entries with too long field names and log the dropped log entries with the `ignoring log entry with too long field name` message, so human operators could notice and fix the ingestion of incorrect logs ASAP. Previously too long field names were silently truncated to shorter values. This isn't what most users expect. See [why VictoriaLogs has a limit on the field name length](https://docs.victoriametrics.com/victorialogs/faq/#what-is-the-maximum-supported-field-name-length).
## [v1.8.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.8.0-victorialogs)
Released at 2025-01-24
* FEATURE: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): add an ability to limit query concurrency for the `<q>` [query](https://docs.victoriametrics.com/victorialogs/logsql/#query-syntax) via `options(concurrency=N) <q>` syntax. This may be needed for reducing RAM and CPU usage at the cost of longer query execution times. See [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#query-options) for details.
* FEATURE: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): allow overriding the global time range filter at subqueries inside [`in` filter](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter), [`join` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#join-pipe) and [`union` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#union-pipe) via `options(ignore_global_time_filter=true) <q>` syntax. See [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#query-options) for details.
* FEATURE: add [`hash` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#hash-pipe) for calculating hashes over the selected log fields. This may be useful for splitting the selected logs into distinct buckets. For example, the following query splits `user_id` fields into 4 buckets with the help of `hash` pipe: `_time:5m | hash(user_id) as h | math h%4 as bucket | stats by (bucket) count()`.
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): reflect column settings for the table view in URL, so the table view can be shared via link. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7662).
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): added context menu for legend items with options to copy and filter streams and fields. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7750).
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): improved legend functionality with consistent click behavior across all vmui charts. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7750).
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): added sorting for group view by record count in descending order. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7750).
* BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): removed the ability to filter by other in the legend, as other represents an aggregated series of all streams not included in the top results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7552).
* BUGFIX: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): properly apply optimizations to all the subqueries. Previously only the top-level query was optimized, while subqueries inside [`in` filter](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter), [`join` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#join-pipe) and [`union` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#union-pipe) weren't optimized.
* BUGFIX: [HTTP querying API](https://docs.victoriametrics.com/victorialogs/querying/#http-api): properly apply [extra filters](https://docs.victoriametrics.com/victorialogs/querying/#extra-filters) to all the subqueries. Previously extra filters were applied only to the top-level query and weren't applied to sub-queries inside [`in` filter](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter), [`join` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#join-pipe) and [`union` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#union-pipe).
* BUGFIX: [HTTP querying API](https://docs.victoriametrics.com/victorialogs/querying/#http-api): properly apply time range filter from `start` and `end` args to all the subqueries. Previously the time range filters were applied only to the top-level query and weren't applied to sub-queries inside [`in` filter](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter), [`join` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#join-pipe) and [`union` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#union-pipe). The global time range filter can be ignored on a per-subquery basis via the `ignore_global_time_filter=true` option - see [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#query-options) for details.
## [v1.7.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.7.0-victorialogs)

View File

@@ -99,11 +99,13 @@ VictoriaLogs is designed solely for logs. VictoriaLogs uses [similar design idea
ClickHouse needs an intermediate applications for converting the ingested logs into `INSERT` SQL statements for the particular database schema.
This may increase the complexity of the system and, subsequently, increase its' maintenance costs.
- VictoriaLogs provides [built-in Web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui) for logs' exploration.
## How does VictoriaLogs work?
VictoriaLogs accepts logs as [JSON entries](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
Then it stores log fields into a distinct data block. E.g. values for the same log field across multiple log entries
Then it stores log fields into distinct data blocks. E.g. values for the same log field across multiple log entries
are stored in a single data block. This allows reading data blocks only for the needed fields during querying.
Data blocks are compressed before being saved to persistent storage. This allows saving disk space and improving query performance
@@ -123,7 +125,7 @@ On top of this, VictoriaLogs employs additional optimizations for achieving high
- It uses [bloom filters](https://en.wikipedia.org/wiki/Bloom_filter) for skipping blocks without the given
[word](https://docs.victoriametrics.com/victorialogs/logsql/#word-filter) or [phrase](https://docs.victoriametrics.com/victorialogs/logsql/#phrase-filter).
- It uses custom encoding and compression for fields with different data types.
For example, it encodes IP addresses int 4 bytes. Custom fields' encoding reduces data size on disk and improves query performance.
For example, it encodes IP addresses into 4 bytes. Custom fields' encoding reduces data size on disk and improves query performance.
- It physically groups logs for the same [log stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
close to each other in the storage. This improves compression ratio, which helps reducing disk space usage. This also improves query performance
by skipping blocks for unneeded streams when [stream filter](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter) is used.
@@ -182,16 +184,36 @@ VictoriaLogs works optimally with log records of up to `10KB`. It works OK with
log records of up to `100KB`. It works not so optimal with log records exceeding
`100KB`.
The max size of a log record VictoriaLogs can handle is `2MB`. This is
because VictoriaLogs stores log records in blocks and `2MB` is the max size of a
block. Blocks of this size fit the L2 cache of a typical CPU, which gives an
optimal processing performance.
The max size of a log record VictoriaLogs can accept during [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/)
is `2MB`, because log records are stored in blocks of up to `2MB` size.
Blocks of this size fit the L2 cache of a typical CPU, which gives an
optimal performance during data ingestion and querying.
However, log records whose size is close to `2MB` aren't handled efficiently by
Note that log records with sizes close to `2MB` aren't handled efficiently by
VictoriaLogs because per-block overhead translates to a single log record, and
this overhead is big.
this overhead is big.
The `2MB` limit is hadrcoded and is unlikely to change.
The `2MB` limit is hadrcoded and is unlikely to increase.
The limit can be set to the lower value during [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/)
via `-insert.maxLineSizeBytes` command-line flag.
## What is the maximum supported field name length
VictoriaLogs limits [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) name length to 128 bytes -
Log entries with longer field names are ignored during [date ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/).
The maximum length of a field name is hardcoded and is unikely to increase, since this may increase RAM and CPU usage.
## How many fields a single log entry may contain
A single log entry may contain up to 2000 fields. This fits well the majority of use cases for structured logs and
for [wide events](https://jeremymorrell.dev/blog/a-practitioners-guide-to-wide-events/).
The maximum number of fields per log entry is hardcoded and is unlikely to increase, since this may increase RAM and CPU usage.
The limit can be set to the lower value during [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/)
via `-insert.maxFieldsPerLine` command-line flag.
## How to determine which log fields occupy the most of disk space?

View File

@@ -1364,6 +1364,7 @@ LogsQL supports the following pipes:
- `bloom_bytes` - on-disk size of bloom filter data for the given `field`
- `dict_bytes` - on-disk size of the dictionary data for the given `field`
- `dict_items` - the number of unique values in the dictionary for the given `field`
- `part_path` - the path to the data part where the block is stored
The `block_stats` pipe is needed mostly for debugging purposes.
@@ -3152,6 +3153,9 @@ For example, the following query unrolls `timestamp` and `value` [log fields](ht
_time:5m | unroll (timestamp, value)
```
If the unrolled JSON array contains JSON objects, then it may be handy using [`unpack_json`](#unpack_json-pipe) for unpacking
the unrolled array items into separate fields for further processing.
See also:
- [`unpack_json` pipe](#unpack_json-pipe)
@@ -3341,9 +3345,31 @@ If the field contains [duration value](#duration-values), then `histogram` norma
If the field contains [short numeric value](#short-numeric-values), then `histogram` normalizes it to numeric value without any suffixes. For example, `1KiB` is converted to `1024`.
Histogram buckets are returned as the following JSON array:
```json
[{"vmrange":"...","hits":...},...,{"vmrange":"...","hits":...}]
```
Every `vmrange` value contains value range for the corresponding [VictoriaMetrics histogram bucket](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350),
while `hits` contains the number of values, which hit the given bucket.
It may be handy to unroll the returned histogram buckets for further processing during the query. For example, the following query
calculates a histogram over the `response_size` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
and then unrolls it into distinct rows with `vmrange` and `hits` fields with the help of [`unroll`](#unroll-pipe) and [`unpack_json`](#unpack_json-pipe) pipes:
```logsql
_time:5m
| stats histogram(response_size) as buckets
| unroll (buckets)
| unpack_json from buckets
```
See also:
- [`quantile`](#quantile-stats)
- [`unroll` pipe](#unroll-pipe)
- [`unpack_json` pipe](#unpack_json-pipe)
### max stats
@@ -3766,3 +3792,19 @@ VictoriaLogs supports the following options, which can be passed in the beginnin
```logsql
options(concurrency=2) _time:1d | count_uniq(user_id)
```
- `ignore_global_time_filter` - allows ignoring time filter from `start` and `end` args of [HTTP querying API](https://docs.victoriametrics.com/victorialogs/querying/#http-api)
for the given (sub)query. For example, the following query returns the number of logs with `user_id` values seen in logs during December 2024, on the `[start...end]`
time range passed to [`/api/v1/query`](https://docs.victoriametrics.com/victorialogs/querying/#querying-logs):
```logsql
user_id:in(options(ignore_global_time_filter=true) _time:2024-12Z | keep user_id) | count()
```
The [`in`](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter) query without `options(ignore_global_time_filter=true)`
takes into account only `user_id` values on the intersection of December 2024 and `[start...end]` time range pased
to [`/api/v1/query`](https://docs.victoriametrics.com/victorialogs/querying/#querying-logs):
```logsql
user_id:in(_time:2024-12Z | keep user_id) | count()
```

View File

@@ -33,8 +33,8 @@ Just download archive for the needed Operating system and architecture, unpack i
For example, the following commands download VictoriaLogs archive for Linux/amd64, unpack and run it:
```sh
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.7.0-victorialogs/victoria-logs-linux-amd64-v1.7.0-victorialogs.tar.gz
tar xzf victoria-logs-linux-amd64-v1.7.0-victorialogs.tar.gz
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.8.0-victorialogs/victoria-logs-linux-amd64-v1.8.0-victorialogs.tar.gz
tar xzf victoria-logs-linux-amd64-v1.8.0-victorialogs.tar.gz
./victoria-logs-prod
```
@@ -58,7 +58,7 @@ Here is the command to run VictoriaLogs in a Docker container:
```sh
docker run --rm -it -p 9428:9428 -v ./victoria-logs-data:/victoria-logs-data \
docker.io/victoriametrics/victoria-logs:v1.7.0-victorialogs
docker.io/victoriametrics/victoria-logs:v1.8.0-victorialogs
```
See also:

View File

@@ -4,28 +4,30 @@ from [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/).
VictoriaLogs provides the following features:
- It can accept logs from popular log collectors. See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/).
- It is much easier to set up and operate compared to Elasticsearch and Grafana Loki.
- It is much easier to set up and operate compared to Elasticsearch and Grafana Loki, since it is basically zero-config.
See [these docs](https://docs.victoriametrics.com/victorialogs/quickstart/).
- It provides easy yet powerful query language with full-text search across
- It provides easy yet powerful query language with full-text search capabilities across
all the [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
See [LogsQL docs](https://docs.victoriametrics.com/victorialogs/logsql/).
- It provides interactive command-line tool for querying VictoriaLogs - [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/).
- It provides [built-in web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui) for logs' exploration.
- Ir provides [Grafana plugin](https://docs.victoriametrics.com/victorialogs/victorialogs-datasource/) for building arbitrary dashboards in Grafana.
- It provides [interactive command-line tool for querying VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/).
- It can be seamlessly combined with good old Unix tools for log analysis such as `grep`, `less`, `sort`, `jq`, etc.
See [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) for details.
- VictoriaLogs' capacity and performance scales linearly with the available resources (CPU, RAM, disk IO, disk space).
It runs smoothly on Raspberry PI and on servers with hundreds of CPU cores and terabytes of RAM.
- It can handle up to 30x bigger data volumes than Elasticsearch and Grafana Loki when running on the same hardware.
See [these docs](#benchmarks) and [this article](https://itnext.io/how-do-open-source-solutions-for-logs-work-elasticsearch-loki-and-victorialogs-9f7097ecbc2f) for details.
- It provides fast full-text search out of the box for [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
with high cardinality (e.g. high number of unique values) such as `trace_id`, `user_id` and `ip`.
- It support [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) with high cardinality (e.g. high number of unique values) such as `trace_id`, `user_id` and `ip`.
- It is optimized for logs with hundreds of fields (aka [`wide events`](https://jeremymorrell.dev/blog/a-practitioners-guide-to-wide-events/)).
- It supports multitenancy - see [these docs](#multitenancy).
- It supports out-of-order logs' ingestion aka backfilling.
- It supports live tailing for newly ingested logs. See [these docs](https://docs.victoriametrics.com/victorialogs/querying/#live-tailing).
- It supports selecting surrounding logs in front and after the selected logs. See [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#stream_context-pipe).
- It provides web UI for querying logs - see [these docs](https://docs.victoriametrics.com/victorialogs/querying/#web-ui).
- It provides [Grafana plugin for querying logs](https://docs.victoriametrics.com/victorialogs/victorialogs-datasource/).
- It supports alerting - see [these docs](https://docs.victoriametrics.com/victorialogs/vmalert/).
You can play with VictoriaLogs web UI at [this playground](https://play-vmlogs.victoriametrics.com/).
If you have questions about VictoriaLogs, then read [this FAQ](https://docs.victoriametrics.com/victorialogs/faq/).
Also feel free asking any questions at [VictoriaMetrics community Slack chat](https://victoriametrics.slack.com/),
you can join it via [Slack Inviter](https://slack.victoriametrics.com/).
@@ -247,6 +249,37 @@ VictoriaLogs has very low overhead for per-tenant management, so it is OK to hav
VictoriaLogs doesn't perform per-tenant authorization. Use [vmauth](https://docs.victoriametrics.com/vmauth/) or similar tools for per-tenant authorization.
### Multitenancy access control
Enforce access control for tenants by using [vmauth](https://docs.victoriametrics.com/vmauth/). Access control can be configured for each tenant by setting up the following rules:
```yaml
users:
- username: "foo"
password: "bar"
url_map:
- src_paths:
- "/select/.*"
- "/insert/.*"
headers:
- "AccountID: 1"
- "ProjectID: 0"
url_prefix:
- "http://localhost:9428/"
- username: "baz"
password: "bar"
url_map:
- src_paths: ["/select/.*"]
headers:
- "AccountID: 2"
- "ProjectID: 0"
url_prefix:
- "http://localhost:9428/"
```
This configuration allows `foo` to use the `/select/.*` and `/insert/.*` endpoints with `AccountID: 1` and `ProjectID: 0`, while `baz` can only use the `/select/.*` endpoint with `AccountID: 2` and `ProjectID: 0`.
## Security
It is expected that VictoriaLogs runs in a protected environment, which is unreachable from the Internet without proper authorization.
@@ -256,6 +289,8 @@ or similar authorization proxies.
## Benchmarks
See [the comparison of VictoriaLogs with Elasticsearch, MongoDB, TimescaleDB, PostgreSQL, MySQL and SQLite](https://benchmark.clickhouse.com/#eyJzeXN0ZW0iOnsiQWxsb3lEQiI6ZmFsc2UsIkFsbG95REIgKHR1bmVkKSI6ZmFsc2UsIkF0aGVuYSAocGFydGl0aW9uZWQpIjpmYWxzZSwiQXRoZW5hIChzaW5nbGUpIjpmYWxzZSwiQXVyb3JhIGZvciBNeVNRTCI6ZmFsc2UsIkF1cm9yYSBmb3IgUG9zdGdyZVNRTCI6ZmFsc2UsIkJ5Q29uaXR5IjpmYWxzZSwiQnl0ZUhvdXNlIjpmYWxzZSwiY2hEQiAoRGF0YUZyYW1lKSI6ZmFsc2UsImNoREIgKFBhcnF1ZXQsIHBhcnRpdGlvbmVkKSI6ZmFsc2UsImNoREIiOmZhbHNlLCJDaXR1cyI6ZmFsc2UsIkNsaWNrSG91c2UgQ2xvdWQgKGF3cykiOmZhbHNlLCJDbGlja0hvdXNlIENsb3VkIChhenVyZSkiOmZhbHNlLCJDbGlja0hvdXNlIENsb3VkIChnY3ApIjpmYWxzZSwiQ2xpY2tIb3VzZSAoZGF0YSBsYWtlLCBwYXJ0aXRpb25lZCkiOmZhbHNlLCJDbGlja0hvdXNlIChkYXRhIGxha2UsIHNpbmdsZSkiOmZhbHNlLCJDbGlja0hvdXNlIChQYXJxdWV0LCBwYXJ0aXRpb25lZCkiOmZhbHNlLCJDbGlja0hvdXNlIChQYXJxdWV0LCBzaW5nbGUpIjpmYWxzZSwiQ2xpY2tIb3VzZSAod2ViKSI6ZmFsc2UsIkNsaWNrSG91c2UiOmZhbHNlLCJDbGlja0hvdXNlICh0dW5lZCkiOmZhbHNlLCJDbGlja0hvdXNlICh0dW5lZCwgbWVtb3J5KSI6ZmFsc2UsIkNsb3VkYmVycnkiOmZhbHNlLCJDcmF0ZURCIjpmYWxzZSwiQ3J1bmNoeSBCcmlkZ2UgZm9yIEFuYWx5dGljcyAoUGFycXVldCkiOmZhbHNlLCJEYXRhYmVuZCI6ZmFsc2UsIkRhdGFGdXNpb24gKFBhcnF1ZXQsIHBhcnRpdGlvbmVkKSI6ZmFsc2UsIkRhdGFGdXNpb24gKFBhcnF1ZXQsIHNpbmdsZSkiOmZhbHNlLCJBcGFjaGUgRG9yaXMiOmZhbHNlLCJEcmlsbCI6ZmFsc2UsIkRydWlkIjpmYWxzZSwiRHVja0RCIChEYXRhRnJhbWUpIjpmYWxzZSwiRHVja0RCIChtZW1vcnkpIjpmYWxzZSwiRHVja0RCIChQYXJxdWV0LCBwYXJ0aXRpb25lZCkiOmZhbHNlLCJEdWNrREIiOmZhbHNlLCJFbGFzdGljc2VhcmNoIjp0cnVlLCJFbGFzdGljc2VhcmNoICh0dW5lZCkiOmZhbHNlLCJHbGFyZURCIjpmYWxzZSwiR3JlZW5wbHVtIjpmYWxzZSwiSGVhdnlBSSI6ZmFsc2UsIkh5ZHJhIjpmYWxzZSwiSW5mb2JyaWdodCI6ZmFsc2UsIktpbmV0aWNhIjpmYWxzZSwiTWFyaWFEQiBDb2x1bW5TdG9yZSI6ZmFsc2UsIk1hcmlhREIiOmZhbHNlLCJNb25ldERCIjpmYWxzZSwiTW9uZ29EQiI6dHJ1ZSwiTW90aGVyRHVjayI6ZmFsc2UsIk15U1FMIChNeUlTQU0pIjpmYWxzZSwiTXlTUUwiOnRydWUsIk9jdG9TUUwiOmZhbHNlLCJPeGxhIjpmYWxzZSwiUGFuZGFzIChEYXRhRnJhbWUpIjpmYWxzZSwiUGFyYWRlREIgKFBhcnF1ZXQsIHBhcnRpdGlvbmVkKSI6ZmFsc2UsIlBhcmFkZURCIChQYXJxdWV0LCBzaW5nbGUpIjpmYWxzZSwicGdfZHVja2RiIChNb3RoZXJEdWNrIGVuYWJsZWQpIjpmYWxzZSwicGdfZHVja2RiIjpmYWxzZSwiUGlub3QiOmZhbHNlLCJQb2xhcnMgKERhdGFGcmFtZSkiOmZhbHNlLCJQb2xhcnMgKFBhcnF1ZXQpIjpmYWxzZSwiUG9zdGdyZVNRTCAodHVuZWQpIjpmYWxzZSwiUG9zdGdyZVNRTCI6dHJ1ZSwiUXVlc3REQiI6ZmFsc2UsIlJlZHNoaWZ0IjpmYWxzZSwiU2VsZWN0REIiOmZhbHNlLCJTaW5nbGVTdG9yZSI6ZmFsc2UsIlNub3dmbGFrZSI6ZmFsc2UsIlNwYXJrIjpmYWxzZSwiU1FMaXRlIjp0cnVlLCJTdGFyUm9ja3MiOmZhbHNlLCJUYWJsZXNwYWNlIjpmYWxzZSwiVGVtYm8gT0xBUCAoY29sdW1uYXIpIjpmYWxzZSwiVGltZXNjYWxlIENsb3VkIjpmYWxzZSwiVGltZXNjYWxlREIgKG5vIGNvbHVtbnN0b3JlKSI6ZmFsc2UsIlRpbWVzY2FsZURCIjp0cnVlLCJUaW55YmlyZCAoRnJlZSBUcmlhbCkiOmZhbHNlLCJVbWJyYSI6ZmFsc2UsIlZpY3RvcmlhTG9ncyI6dHJ1ZX0sInR5cGUiOnsiQyI6dHJ1ZSwiY29sdW1uLW9yaWVudGVkIjp0cnVlLCJQb3N0Z3JlU1FMIGNvbXBhdGlibGUiOnRydWUsIm1hbmFnZWQiOnRydWUsImdjcCI6dHJ1ZSwic3RhdGVsZXNzIjp0cnVlLCJKYXZhIjp0cnVlLCJDKysiOnRydWUsIk15U1FMIGNvbXBhdGlibGUiOnRydWUsInJvdy1vcmllbnRlZCI6dHJ1ZSwiQ2xpY2tIb3VzZSBkZXJpdmF0aXZlIjp0cnVlLCJlbWJlZGRlZCI6dHJ1ZSwic2VydmVybGVzcyI6dHJ1ZSwiZGF0YWZyYW1lIjp0cnVlLCJhd3MiOnRydWUsImF6dXJlIjp0cnVlLCJhbmFseXRpY2FsIjp0cnVlLCJSdXN0Ijp0cnVlLCJzZWFyY2giOnRydWUsImRvY3VtZW50Ijp0cnVlLCJHbyI6dHJ1ZSwic29tZXdoYXQgUG9zdGdyZVNRTCBjb21wYXRpYmxlIjp0cnVlLCJEYXRhRnJhbWUiOnRydWUsInBhcnF1ZXQiOnRydWUsInRpbWUtc2VyaWVzIjp0cnVlfSwibWFjaGluZSI6eyIxNiB2Q1BVIDEyOEdCIjpmYWxzZSwiOCB2Q1BVIDY0R0IiOmZhbHNlLCJzZXJ2ZXJsZXNzIjpmYWxzZSwiMTZhY3UiOmZhbHNlLCJjNmEuNHhsYXJnZSwgNTAwZ2IgZ3AyIjp0cnVlLCJMIjpmYWxzZSwiTSI6ZmFsc2UsIlMiOmZhbHNlLCJYUyI6ZmFsc2UsImM2YS5tZXRhbCwgNTAwZ2IgZ3AyIjpmYWxzZSwiMTkyR0IiOmZhbHNlLCIyNEdCIjpmYWxzZSwiMzYwR0IiOmZhbHNlLCI0OEdCIjpmYWxzZSwiNzIwR0IiOmZhbHNlLCI5NkdCIjpmYWxzZSwiZGV2IjpmYWxzZSwiNzA4R0IiOmZhbHNlLCJjNW4uNHhsYXJnZSwgNTAwZ2IgZ3AyIjpmYWxzZSwiQW5hbHl0aWNzLTI1NkdCICg2NCB2Q29yZXMsIDI1NiBHQikiOmZhbHNlLCJjNS40eGxhcmdlLCA1MDBnYiBncDIiOmZhbHNlLCJjNmEuNHhsYXJnZSwgMTUwMGdiIGdwMiI6dHJ1ZSwiY2xvdWQiOmZhbHNlLCJkYzIuOHhsYXJnZSI6ZmFsc2UsInJhMy4xNnhsYXJnZSI6ZmFsc2UsInJhMy40eGxhcmdlIjpmYWxzZSwicmEzLnhscGx1cyI6ZmFsc2UsIlMyIjpmYWxzZSwiUzI0IjpmYWxzZSwiMlhMIjpmYWxzZSwiM1hMIjpmYWxzZSwiNFhMIjpmYWxzZSwiWEwiOmZhbHNlLCJMMSAtIDE2Q1BVIDMyR0IiOmZhbHNlLCJjNmEuNHhsYXJnZSwgNTAwZ2IgZ3AzIjpmYWxzZSwiMTYgdkNQVSA2NEdCIjpmYWxzZSwiNCB2Q1BVIDE2R0IiOmZhbHNlLCI4IHZDUFUgMzJHQiI6ZmFsc2V9LCJjbHVzdGVyX3NpemUiOnsiMSI6dHJ1ZSwiMiI6ZmFsc2UsIjQiOmZhbHNlLCI4IjpmYWxzZSwiMTYiOmZhbHNlLCIzMiI6ZmFsc2UsIjY0IjpmYWxzZSwiMTI4IjpmYWxzZSwic2VydmVybGVzcyI6ZmFsc2UsInVuZGVmaW5lZCI6ZmFsc2V9LCJtZXRyaWMiOiJob3QiLCJxdWVyaWVzIjpbdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZSx0cnVlLHRydWUsdHJ1ZV19).
Here is a [benchmark suite](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/logs-benchmark) for comparing data ingestion performance
and resource usage between VictoriaLogs and Elasticsearch or Loki.

View File

@@ -942,7 +942,7 @@ The arg passed to `extra_filters` and `extra_stream_filters` must be properly en
## Web UI
VictoriaLogs provides Web UI for logs [querying](https://docs.victoriametrics.com/victorialogs/logsql/) and exploration
at `http://localhost:9428/select/vmui`.
at `http://localhost:9428/select/vmui`. Try [VictoriaLogs web UI playground](https://play-vmlogs.victoriametrics.com/).
There are three modes of displaying query results:

View File

@@ -23,8 +23,8 @@ or from [docker images](https://hub.docker.com/r/victoriametrics/vlogscli/tags).
### Running `vlogscli` from release binary
```sh
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.7.0-victorialogs/vlogscli-linux-amd64-v1.7.0-victorialogs.tar.gz
tar xzf vlogscli-linux-amd64-v1.7.0-victorialogs.tar.gz
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.8.0-victorialogs/vlogscli-linux-amd64-v1.8.0-victorialogs.tar.gz
tar xzf vlogscli-linux-amd64-v1.8.0-victorialogs.tar.gz
./vlogscli-prod
```

View File

@@ -39,7 +39,7 @@ FROM <table>
<optional GROUP BY>
<optional HAVING>
<optional ORDER BY>
<optional LIMIT and OFFSET>
<optional LIMIT / OFFSET>
<optional UNION>
```
@@ -69,7 +69,7 @@ The following rules must be used for converting SQL query into LogsQL query:
Otherwise just start LogsQL query with [`*`](https://docs.victoriametrics.com/victorialogs/logsql/#any-value-filter).
For example, `SELECT * FROM table WHERE field1=value1 AND field2<>value2` is converted into `field1:=value1 field2:!=value2`,
while `SELECT * FROM table` is converted into `*`.
* Subqueries inside `WHERE` must be converted into [`in` filters](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter).
* `IN` subqueries inside `WHERE` must be converted into [`in` filters](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter).
For example, `SELECT * FROM table WHERE id IN (SELECT id2 FROM table)` is converted into `id:in(* | fields id2)`.
* If the `SELECT` part isn't equal to `*` and there are no `GROUP BY` / aggregate functions in the SQL query, then enumerate
the selected columns at [`fields` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#fields-pipe).
@@ -77,7 +77,9 @@ The following rules must be used for converting SQL query into LogsQL query:
* If the SQL query contains `JOIN`, then convert it into [`join` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#join-pipe).
* If the SQL query contains `GROUP BY` / aggregate functions, then convert them to [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe).
For example, `SELECT count(*) FROM table` is converted into `* | count()`, while `SELECT user_id, count(*) FROM table GROUP BY user_id`
is converted to `* | stats by (user_id) count()`.
is converted to `* | stats by (user_id) count()`. Note how the LogsQL query mentions the `GROUP BY` fields only once,
while SQL forces mentioning these fields twice - at the `SELECT` and at the `GROUP BY`. How many times did you hit the discrepancy
between `SELECT` and `GROUP BY` fields?
* If the SQL query contains additional calculations and/or transformations at the `SELECT`, which aren't covered yet by `GROUP BY`,
then convert them into the corresponding [LogsQL pipes](https://docs.victoriametrics.com/victorialogs/logsql/#pipes).
The most frequently used pipes are [`math`](https://docs.victoriametrics.com/victorialogs/logsql/#math-pipe)
@@ -90,6 +92,25 @@ The following rules must be used for converting SQL query into LogsQL query:
* If the SQL query contains `UNION`, then convert it into [`union` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#union-pipe).
For example `SELECT * FROM table WHERE filters1 UNION ALL SELECT * FROM table WHERE filters2` is converted into `filters1 | union (filters2)`.
SQL queries are frequently used for obtaining top N column values, which are the most frequently seen in the selected rows.
For example, the query below returns top 5 `user_id` values, which present in the biggest number of rows:
```sql
SELECT user_id, count(*) hits FROM table GROUP BY user_id ORDER BY hits DESC LIMIT 5
```
LogsQL provides a shortcut syntax with [`top` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#top-pipe) for this case:
```logsql
* | top 5 (user_id)
```
It is equivalent to the longer LogsQL query:
```logsql
* | by (user_id) count() hits | sort by (hits desc) limit 5
```
[LogsQL pipes](https://docs.victoriametrics.com/victorialogs/logsql/#pipes) support much wider functionality comparing to SQL,
so spend your spare time by reading [pipe docs](https://docs.victoriametrics.com/victorialogs/logsql/) and playing with them
at [VictoriaLogs playground](https://play-vmlogs.victoriametrics.com/).

View File

@@ -11,14 +11,29 @@ aliases:
---
Please find the changelog for VictoriaMetrics Anomaly Detection below.
## v1.19.2
Released: 2025-01-27
- IMPROVEMENT: Added the `complete` option to the `--splitBy` argument in `config_splitter.py` [util](https://docs.victoriametrics.com/anomaly-detection/faq/index.html#splitting-the-config). This allows splitting a parent configuration into the smallest possible sub-configurations, each containing exactly one scheduler, one model, and either one or multiple queries (depending on whether the model is [multivariate](https://docs.victoriametrics.com/anomaly-detection/components/models/#multivariate-models) or not).
- FIX: Resolved an issue where duplicate log messages were generated during sub-config validation of the parent configuration.
- FIX: Corrected usage of `AccountID` and `ProjectID` extracted from `tenant_id`, which are appended as labels `vm_account_id` and `vm_project_id`, respectively (previously swapped) by `VmReader` when using the per-query `tenant_id` feature. **This issue affected versions [v1.19.0](#v1190) and [v1.19.1](#v1191).**
- FIX: Resolved an issue with the `VmReader` instance string representation that caused errors when `vmanomaly` was run with `--loggerLevel DEBUG`.
## v1.19.1
Released: 2025-01-21
> **Note**: There is a known bug in [v1.19.0](#v1190) - the `AccountID` and `ProjectID` are swapped when they are extracted from the `tenant_id` argument in `VMReader`. This can cause correctly read results being written to the wrong tenant when using the per-query `tenant_id` feature with `AccountID` != `ProjectID`. Please update to patch [v1.19.2](#v1192), which resolves this issue.
- FIX: Resolved writer warnings for configurations where `reader.tenant_id` equals `writer.tenant_id` and **is not** `multitenant`, as this is a valid setup. Enhanced tenant_id-related log messages across config validation, reader, and writer for improved clarity.
## v1.19.0
Released: 2025-01-20
> **Note**: There is a known bug in [v1.19.0](#v1190) - the `AccountID` and `ProjectID` are swapped when they are extracted from the `tenant_id` argument in `VMReader`. This can cause correctly read results being written to the wrong tenant when using the per-query `tenant_id` feature with `AccountID` != `ProjectID`. Please update to patch [v1.19.2](#v1192), which resolves this issue.
- FEATURE: Added support for per-query `tenant_id` in the [`VmReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader). This allows overriding the reader-level `tenant_id` within a single global `vmanomaly` configuration on a *per-query* basis, enabling isolation of data for different tenants in separate queries when querying the [VictoriaMetrics cluster version](https://docs.victoriametrics.com/cluster-victoriametrics/). For details, see the [documentation](https://docs.victoriametrics.com/anomaly-detection/components/reader/?highlight=tenant_id#per-query-parameters).
- IMPROVEMEMT: Speedup the model infer stage on multicore systems.
- IMPROVEMEMT: Speedup the model fitting stage by 1.253x, depending on configuration complexity.

View File

@@ -158,7 +158,7 @@ services:
# ...
vmanomaly:
container_name: vmanomaly
image: victoriametrics/vmanomaly:v1.19.1
image: victoriametrics/vmanomaly:v1.19.2
# ...
ports:
- "8490:8490"
@@ -337,10 +337,10 @@ For **horizontal** scalability, `vmanomaly` can be deployed as multiple independ
### Splitting the config
CLI utility named `config_splitter` is available in `vmanomaly` {{% available_from "v1.18.5" anomaly %}}. The config splitter tool enables splitting a parent vmanomaly YAML configuration file into multiple sub-configurations based on logical entities such as `schedulers`, `queries`, `models`, `extra_filters`. The resulting sub-configurations are fully validated, functional, account for many-to-many relationships between models and their associated queries, and the schedulers they are linked to. These sub-configurations can then be saved to a specified directory for further use:
CLI utility named `config_splitter` is available in `vmanomaly` {{% available_from "v1.18.5" anomaly %}}. The config splitter tool enables splitting a parent vmanomaly YAML configuration file into multiple sub-configurations based on logical entities such as `schedulers`, `queries`, `models`, `extra_filters` and `complete` {{% available_from "v1.19.2" anomaly %}}. The resulting sub-configurations are fully validated, functional, account for many-to-many relationships between models and their associated queries, and the schedulers they are linked to. These sub-configurations can then be saved to a specified directory for further use:
```shellhelp
usage: config_splitter.py [-h] --splitBy {schedulers,models,queries,extra_filters} --outputDir OUTPUT_DIR [--fileNameFormat {raw,hash,int}] [--loggerLevel {WARNING,INFO,ERROR,FATAL,DEBUG}]
usage: config_splitter.py [-h] --splitBy {schedulers,models,queries,extra_filters,complete} --outputDir OUTPUT_DIR [--fileNameFormat {raw,hash,int}] [--loggerLevel {WARNING,ERROR,FATAL,INFO,DEBUG}]
config [config ...]
Splits the configuration of VictoriaMetrics Anomaly Detection service by a logical entity.
@@ -350,21 +350,22 @@ positional arguments:
options:
-h show this help message and exit
--splitBy {schedulers,models,queries,extra_filters}
The logical entity to split by. Choices: ['schedulers', 'models', 'queries', 'extra_filters'].
--splitBy {schedulers,models,queries,extra_filters,complete}
The logical entity to split by. Choices: ['schedulers', 'models', 'queries', 'extra_filters', 'complete']. `complete` produces configurations based on combinations of
(scheduler, model, queries). Default: complete.
--outputDir output_dir
Directory where the split configuration files will be saved.
--fileNameFormat {raw,hash,int}
The naming format for the output configuration files. Choices: raw (use the entity alias), hash (use hashed alias), int (use a sequential integer from 0 to N for N
produced sub-configs). Default: raw.
--loggerLevel {WARNING,INFO,ERROR,FATAL,DEBUG}
The naming format for the output configuration files. Choices: raw (use the entity alias), hash (use hashed alias), int (use a sequential integer from 0 to N for N produced
sub-configs). Default: raw.
--loggerLevel {WARNING,ERROR,FATAL,INFO,DEBUG}
Minimum level to log. Default: INFO
```
Heres an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
```sh
docker pull victoriametrics/vmanomaly:v1.19.1 && docker image tag victoriametrics/vmanomaly:v1.19.1 vmanomaly
docker pull victoriametrics/vmanomaly:v1.19.2 && docker image tag victoriametrics/vmanomaly:v1.19.2 vmanomaly
```
```sh

View File

@@ -101,13 +101,13 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
1. Pull Docker image:
```sh
docker pull victoriametrics/vmanomaly:v1.19.1
docker pull victoriametrics/vmanomaly:v1.19.2
```
2. (Optional step) tag the `vmanomaly` Docker image:
```sh
docker image tag victoriametrics/vmanomaly:v1.19.1 vmanomaly
docker image tag victoriametrics/vmanomaly:v1.19.2 vmanomaly
```
3. Start the `vmanomaly` Docker container with a *license file*, use the command below.
@@ -141,7 +141,7 @@ docker run -it --user 1000:1000 \
services:
# ...
vmanomaly:
image: victoriametrics/vmanomaly:v1.19.1
image: victoriametrics/vmanomaly:v1.19.2
volumes:
$YOUR_LICENSE_FILE_PATH:/license
$YOUR_CONFIG_FILE_PATH:/config.yml

View File

@@ -994,7 +994,7 @@ monitoring:
Let's pull the docker image for `vmanomaly`:
```sh
docker pull victoriametrics/vmanomaly:v1.19.1
docker pull victoriametrics/vmanomaly:v1.19.2
```
Now we can run the docker container putting as volumes both config and model file:
@@ -1008,7 +1008,7 @@ docker run -it \
-v $(PWD)/license:/license \
-v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
-v $(PWD)/custom.yaml:/config.yaml \
victoriametrics/vmanomaly:v1.19.1 /config.yaml \
victoriametrics/vmanomaly:v1.19.2 /config.yaml \
--licenseFile=/license
```

View File

@@ -2,9 +2,9 @@
- To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
- In the tutorial, we'll be using the following VictoriaMetrics components:
- [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/single-server-victoriametrics) (v1.109.1)
- [vmalert](https://docs.victoriametrics.com/vmalert/) (v1.109.1)
- [vmagent](https://docs.victoriametrics.com/vmagent/) (v1.109.1)
- [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/single-server-victoriametrics) (v1.110.0)
- [vmalert](https://docs.victoriametrics.com/vmalert/) (v1.110.0)
- [vmagent](https://docs.victoriametrics.com/vmagent/) (v1.110.0)
- [Grafana](https://grafana.com/) (v.10.2.1)
- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.27.0)
@@ -315,7 +315,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
services:
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.109.1
image: victoriametrics/vmagent:v1.110.0
depends_on:
- "victoriametrics"
ports:
@@ -332,7 +332,7 @@ services:
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.109.1
image: victoriametrics/victoria-metrics:v1.110.0
ports:
- 8428:8428
volumes:
@@ -365,7 +365,7 @@ services:
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.109.1
image: victoriametrics/vmalert:v1.110.0
depends_on:
- "victoriametrics"
ports:
@@ -387,7 +387,7 @@ services:
restart: always
vmanomaly:
container_name: vmanomaly
image: victoriametrics/vmanomaly:v1.19.1
image: victoriametrics/vmanomaly:v1.19.2
depends_on:
- "victoriametrics"
ports:

View File

@@ -18,6 +18,30 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
## tip
* FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): improve startup times when opening a storage with the [retention](https://docs.victoriametrics.com/#retention) exceeding a few months.
## [v1.102.12](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.12)
Released at 2025-01-28
**v1.102.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.102.x line will be supported for at least 12 months since [v1.102.0](https://docs.victoriametrics.com/changelog/#v11020) release**
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): log metric names for signals with unsupported delta temporality on ingestion via [OpenTelemetry protocol for metrics](https://docs.victoriametrics.com/#sending-data-via-opentelemetry). Thanks to @chenlujjj for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8018).
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): respect staleness detection in increase, increase_pure and delta functions when time series has gaps and `-search.maxStalenessInterval` is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8072) for details.
## [v1.97.17](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.17)
Released at 2025-01-28
**v1.97.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.97.x line will be supported for at least 12 months since [v1.97.0](https://docs.victoriametrics.com/CHANGELOG.html#v1970) release**
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): log metric names for signals with unsupported delta temporality on ingestion via [OpenTelemetry protocol for metrics](https://docs.victoriametrics.com/#sending-data-via-opentelemetry). Thanks to @chenlujjj for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8018).
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): respect staleness detection in increase, increase_pure and delta functions when time series has gaps and `-search.maxStalenessInterval` is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8072) for details.
## [v1.110.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.0)
Released at 2025-01-24
@@ -29,14 +53,42 @@ Released at 2025-01-24
* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards): all dashboards that use [VictoriaMetrics Grafana datasource](https://github.com/VictoriaMetrics/victoriametrics-datasource) were updated to use a [new datasource ID](https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/tag/v0.12.0).
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): reflect column settings for the table view in URL, so the table view can be shared via link. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7662).
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): allow ingesting histograms with missing `_sum` metric via [OpenTelemetry ingestion protocol](https://docs.victoriametrics.com/#sending-data-via-opentelemetry) in the same way as Prometheus does.
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): respect staleness detection in increase, increase_pure and delta functions when time series has gaps and `-search.maxStalenessInterval` is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8072) for details.
* BUGFIX: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) components: properly trim whitespaces at the end of license provided via `-license` and `-licenseFile` command-line flags. Previously, the trailing whitespaces could cause the license verification to fail.
* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): fix possible runtime panic during requests processing under heavy load. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051) for details.
* BUGFIX: [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): fix panic when trying to delete series by using [multitenant read](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy-via-labels) endpoint. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8126) for the details.
* BUGFIX: [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): prevent panic when `vmselect` receives an error response from `vmstorage` during the query execution and request processing for other `vmstorage` nodes is still in progress. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8114) for the details.
* BUGFIX: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) components: properly trim whitespaces at the end of license provided via `-license` and `-licenseFile` command-line flags. Previously, the trailing whitespaces could cause the license verification to fail.
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): respect staleness detection in increase, increase_pure and delta functions when time series has gaps and `-search.maxStalenessInterval` is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8072) for details.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): allow ingesting histograms with missing `_sum` metric via [OpenTelemetry ingestion protocol](https://docs.victoriametrics.com/#sending-data-via-opentelemetry) in the same way as Prometheus does.
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix an issue where pressing the "Enter" key in the query editor did not execute the query. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8058).
## [v1.102.11](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.11)
Released at 2025-01-24
**v1.102.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.102.x line will be supported for at least 12 months since [v1.102.0](https://docs.victoriametrics.com/changelog/#v11020) release**
* SECURITY: upgrade Go builder from Go1.23.4 to Go1.23.5. See the list of issues addressed in [Go1.23.5](https://github.com/golang/go/issues?q=milestone%3AGo1.23.5+label%3ACherryPickApproved).
* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): fix possible runtime panic during requests processing under heavy load. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051) for details.
* BUGFIX: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) components: properly trim whitespaces at the end of license provided via `-license` and `-licenseFile` command-line flags. Previously, the trailing whitespaces could cause the license verification to fail.
* BUGFIX: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) components: remove unnecessary delay before failing if all online verification attempts have failed. This should reduce the time required for the component to proceed if all online verification attempts have failed.
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): don't take into account the last raw sample before the lookbehind window is sample exceeds the staleness interval. This affects correctness of increase, increase_pure, delta functions when preforming calculations on time series with gaps. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8002) for details.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): allow ingesting histograms with missing `_sum` metric via [OpenTelemetry ingestion protocol](https://docs.victoriametrics.com/#sending-data-via-opentelemetry) in the same way as Prometheus does.
## [v1.97.16](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.16)
Released at 2025-01-24
**v1.97.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.97.x line will be supported for at least 12 months since [v1.97.0](https://docs.victoriametrics.com/CHANGELOG.html#v1970) release**
* BUGFIX: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) components: remove unnecessary delay before failing if all online verification attempts have failed. This should reduce the time required for the component to proceed if all online verification attempts have failed.
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): don't take into account the last raw sample before the lookbehind window is sample exceeds the staleness interval. This affects correctness of increase, increase_pure, delta functions when preforming calculations on time series with gaps. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8002) for details.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): allow ingesting histograms with missing `_sum` metric via [OpenTelemetry ingestion protocol](https://docs.victoriametrics.com/#sending-data-via-opentelemetry) in the same way as Prometheus does.
## [v1.109.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.109.1)
Released at 2025-01-17

View File

@@ -82,7 +82,7 @@ VictoriaMetrics Enterprise components are available in the following forms:
It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
Binary releases of VictoriaMetrics Enterprise are available [at the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.109.1-enterprise.tar.gz`.
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.110.0-enterprise.tar.gz`.
In order to run binary release of VictoriaMetrics Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
from the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) and unpack it. Then run the unpacked binary.
@@ -100,8 +100,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
```sh
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.109.1/victoria-metrics-linux-amd64-v1.109.1-enterprise.tar.gz
tar -xzf victoria-metrics-linux-amd64-v1.109.1-enterprise.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.110.0/victoria-metrics-linux-amd64-v1.110.0-enterprise.tar.gz
tar -xzf victoria-metrics-linux-amd64-v1.110.0-enterprise.tar.gz
./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
```
@@ -116,7 +116,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
Docker images for VictoriaMetrics Enterprise are available [at VictoriaMetrics DockerHub](https://hub.docker.com/u/victoriametrics).
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.109.1-enterprise`.
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.110.0-enterprise`.
In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via command-line
flag as described [here](#binary-releases).
@@ -126,13 +126,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
```sh
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.109.1-enterprise -license=BASE64_ENCODED_LICENSE_KEY
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.110.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
```
Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
```sh
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.109.1-enterprise -licenseFile=/path/to/vm-license
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.110.0-enterprise -licenseFile=/path/to/vm-license
```
Example docker-compose configuration:
@@ -141,7 +141,7 @@ version: "3.5"
services:
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.109.1
image: victoriametrics/victoria-metrics:v1.110.0
ports:
- 8428:8428
volumes:
@@ -173,7 +173,7 @@ is used to provide key in plain-text:
```yaml
server:
image:
tag: v1.109.1-enterprise
tag: v1.110.0-enterprise
license:
key: {BASE64_ENCODED_LICENSE_KEY}
@@ -184,7 +184,7 @@ In order to provide key via existing secret, the following values file is used:
```yaml
server:
image:
tag: v1.109.1-enterprise
tag: v1.110.0-enterprise
license:
secret:
@@ -233,7 +233,7 @@ spec:
license:
key: {BASE64_ENCODED_LICENSE_KEY}
image:
tag: v1.109.1-enterprise
tag: v1.110.0-enterprise
```
In order to provide key via existing secret, the following custom resource is used:
@@ -250,7 +250,7 @@ spec:
name: vm-license
key: license
image:
tag: v1.109.1-enterprise
tag: v1.110.0-enterprise
```
Example secret with license key:

View File

@@ -236,27 +236,27 @@ services:
- grafana_data:/var/lib/grafana/
vmsingle:
image: victoriametrics/victoria-metrics:v1.109.1
image: victoriametrics/victoria-metrics:v1.110.0
command:
- -httpListenAddr=0.0.0.0:8429
vmstorage:
image: victoriametrics/vmstorage:v1.109.1-cluster
image: victoriametrics/vmstorage:v1.110.0-cluster
vminsert:
image: victoriametrics/vminsert:v1.109.1-cluster
image: victoriametrics/vminsert:v1.110.0-cluster
command:
- -storageNode=vmstorage:8400
- -httpListenAddr=0.0.0.0:8480
vmselect:
image: victoriametrics/vmselect:v1.109.1-cluster
image: victoriametrics/vmselect:v1.110.0-cluster
command:
- -storageNode=vmstorage:8401
- -httpListenAddr=0.0.0.0:8481
vmagent:
image: victoriametrics/vmagent:v1.109.1
image: victoriametrics/vmagent:v1.110.0
volumes:
- ./scrape.yaml:/etc/vmagent/config.yaml
command:
@@ -265,7 +265,7 @@ services:
- -remoteWrite.url=http://vmsingle:8429/api/v1/write
vmgateway-cluster:
image: victoriametrics/vmgateway:v1.109.1-enterprise
image: victoriametrics/vmgateway:v1.110.0-enterprise
ports:
- 8431:8431
volumes:
@@ -281,7 +281,7 @@ services:
- -auth.oidcDiscoveryEndpoints=http://keycloak:8080/realms/master/.well-known/openid-configuration
vmgateway-single:
image: victoriametrics/vmgateway:v1.109.1-enterprise
image: victoriametrics/vmgateway:v1.110.0-enterprise
ports:
- 8432:8431
volumes:
@@ -393,7 +393,7 @@ Once iDP configuration is done, vmagent configuration needs to be updated to use
```yaml
vmagent:
image: victoriametrics/vmagent:v1.109.1
image: victoriametrics/vmagent:v1.110.0
volumes:
- ./scrape.yaml:/etc/vmagent/config.yaml
- ./vmagent-client-secret:/etc/vmagent/oauth2-client-secret

View File

@@ -30,8 +30,8 @@ scrape_configs:
After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/) to the same directory:
```
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.109.1/victoria-metrics-linux-amd64-v1.109.1.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.109.1.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.110.0/victoria-metrics-linux-amd64-v1.110.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.110.0.tar.gz
```
Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -146,8 +146,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/) acco
```yaml
# Download and unpack single-node VictoriaMetrics
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.109.1/victoria-metrics-linux-amd64-v1.109.1.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.109.1.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.110.0/victoria-metrics-linux-amd64-v1.110.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.110.0.tar.gz
# Run single-node VictoriaMetrics with the given scrape.yaml
./victoria-metrics-prod -promscrape.config=scrape.yaml

View File

@@ -24,10 +24,10 @@ For a detailed explanation of each parameter, visit the guide on [Understanding
## Flag Parameters Configuration
| **Flag** | **Default Value** | **Description** |
|-----------------------------------|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| **Max Label Value Length** | `<= 1kb` (Default: `4kb`) | Maximum length of label values. Longer values are truncated. Large label values can lead to high RAM consumption. This can be adjusted via [support](mailto:support-cloud@victoriametrics.com). |
| **Max Labels per Time Series** | `<= 30` | Maximum number of labels per time series. Excess labels are dropped. Higher values can increase [cardinality](https://docs.victoriametrics.com/keyconcepts/#cardinality) and resource usage. This can be configured in [deployment settings](https://docs.victoriametrics.com/victoriametrics-cloud/quickstart/#modifying-an-existing-deployment). |
| **Flag** | **Default Value** | **Description** |
|-----------------------------------|---------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| **Max Label Value Length** | `<= 1kb` (Default: `4kb`) | Maximum length of label values. Time series with longer values are dropped. Large label values can lead to high RAM consumption. This can be adjusted via [support](mailto:support-cloud@victoriametrics.com). |
| **Max Labels per Time Series** | `<= 30` | Maximum number of labels per time series. Time series with excess labels are dropped. Higher values can increase [cardinality](https://docs.victoriametrics.com/keyconcepts/#cardinality) and resource usage. This can be configured in [deployment settings](https://docs.victoriametrics.com/victoriametrics-cloud/quickstart/#modifying-an-existing-deployment). |
## Terms and definitions:

View File

@@ -46,7 +46,7 @@ Installing VictoriaMetrics Grafana datasource [requires](https://grafana.com/doc
``` ini
[plugins]
allow_loading_unsigned_plugins = victoriametrics-datasource
allow_loading_unsigned_plugins = victoriametrics-metrics-datasource
```
For `grafana-operator` users, please adjust `config:` section in your `kind=Grafana` resource as below
@@ -54,7 +54,7 @@ For `grafana-operator` users, please adjust `config:` section in your `kind=Graf
```
config:
plugins:
allow_loading_unsigned_plugins: "victoriametrics-datasource"
allow_loading_unsigned_plugins: "victoriametrics-metrics-datasource"
```
See [why VictoriaMetrics datasource is unsigned](#why-victoriaMetrics-datasource-is-unsigned).
@@ -83,7 +83,7 @@ datasources:
# displayed in Grafana panels and queries.
- name: VictoriaMetrics
# <string, required> Sets the data source type.
type: victoriametrics-datasource
type: victoriametrics-metrics-datasource
# <string, required> Sets the access mode, either
# proxy or direct (Server or Browser in the UI).
# Some data sources are incompatible with any setting
@@ -99,7 +99,7 @@ datasources:
# displayed in Grafana panels and queries.
- name: VictoriaMetrics - cluster
# <string, required> Sets the data source type.
type: victoriametrics-datasource
type: victoriametrics-metrics-datasource
# <string, required> Sets the access mode, either
# proxy or direct (Server or Browser in the UI).
# Some data sources are incompatible with any setting
@@ -124,8 +124,8 @@ Please find the example of provisioning Grafana instance with VictoriaMetrics da
grafana:
image: grafana/grafana:11.0.0
environment:
- GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.3/victoriametrics-datasource-v0.10.3.zip;victoriametrics-datasource
- GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-datasource
- GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.13.0/victoriametrics-metrics-datasource-v0.13.0.zip;victoriametrics-metrics-datasource
- GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-metrics-datasource
ports:
- 3000:3000/tcp
volumes:
@@ -152,14 +152,14 @@ Option 1. Using Grafana provisioning:
``` yaml
env:
GF_INSTALL_PLUGINS: "https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.3/victoriametrics-datasource-v0.10.3.zip;victoriametrics-datasource"
GF_INSTALL_PLUGINS: "https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.13.0/victoriametrics-metrics-datasource-v0.13.0.zip;victoriametrics-metrics-datasource"
```
Option 2. Using Grafana plugins section in `values.yaml`:
``` yaml
plugins:
- https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.3/victoriametrics-datasource-v0.10.3.zip;victoriametrics-datasource
- https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.13.0/victoriametrics-metrics-datasource-v0.13.0.zip;victoriametrics-metrics-datasource
```
Option 3. Using init container:
@@ -179,7 +179,7 @@ extraInitContainers:
set -ex
mkdir -p /var/lib/grafana/plugins/
ver=$(curl -s -L https://api.github.com/repos/VictoriaMetrics/victoriametrics-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-metrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
tar -xf /var/lib/grafana/plugins/vm-plugin.tar.gz -C /var/lib/grafana/plugins/
rm /var/lib/grafana/plugins/vm-plugin.tar.gz
volumeMounts:
@@ -239,7 +239,7 @@ spec:
set -ex
mkdir -p /var/lib/grafana/plugins/
ver=$(curl -s https://api.github.com/repos/VictoriaMetrics/victoriametrics-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-metrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
tar -xf /var/lib/grafana/plugins/vm-plugin.tar.gz -C /var/lib/grafana/plugins/
rm /var/lib/grafana/plugins/vm-plugin.tar.gz
volumeMounts:
@@ -247,7 +247,7 @@ spec:
mountPath: /var/lib/grafana
config:
plugins:
allow_loading_unsigned_plugins: victoriametrics-datasource
allow_loading_unsigned_plugins: victoriametrics-metrics-datasource
```
See [Grafana operator reference](https://grafana-operator.github.io/grafana-operator/docs/grafana/) to find more about Grafana operator.
@@ -259,7 +259,7 @@ This example uses init container to download and install plugin.
```sh
ver=$(curl -s https://api.github.com/repos/VictoriaMetrics/victoriametrics-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-metrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
tar -xf /var/lib/grafana/plugins/vm-plugin.tar.gz -C /var/lib/grafana/plugins/
rm /var/lib/grafana/plugins/vm-plugin.tar.gz
```
@@ -279,7 +279,7 @@ plugins = {{path to directory with plugin}}
``` ini
[plugins]
allow_loading_unsigned_plugins = victoriametrics-datasource
allow_loading_unsigned_plugins = victoriametrics-metrics-datasource
```
### 2. Run the plugin

View File

@@ -661,6 +661,19 @@ unauthorized_user:
headers:
- "TenantID: foobar"
- "X-Forwarded-For:"
users:
- username: "foo"
password: "bar"
dump_request_on_errors: true
url_map:
- src_paths: ["/select/.*"]
headers:
- "AccountID: 1"
- "ProjectID: 0"
url_prefix:
- "http://backend:9428/"
```
`vmauth` also supports the ability to set and remove HTTP response headers before returning the response from the backend to client.

87
go.mod
View File

@@ -1,9 +1,6 @@
module github.com/VictoriaMetrics/VictoriaMetrics
go 1.23.3
// See // See https://github.com/googleapis/google-cloud-go/issues/11283#issuecomment-2558515586
exclude google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a
go 1.23.5
// This is needed in order to avoid vmbackup and vmrestore binary size increase by 20MB
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8008
@@ -13,19 +10,19 @@ replace cloud.google.com/go/storage => cloud.google.com/go/storage v1.43.0
require (
cloud.google.com/go/storage v1.50.0
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0
github.com/VictoriaMetrics/easyproto v0.1.4
github.com/VictoriaMetrics/fastcache v1.12.2
github.com/VictoriaMetrics/metrics v1.35.1
github.com/VictoriaMetrics/metricsql v0.82.0
github.com/aws/aws-sdk-go-v2 v1.32.8
github.com/aws/aws-sdk-go-v2/config v1.28.10
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48
github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2
github.com/bmatcuk/doublestar/v4 v4.7.1
github.com/aws/aws-sdk-go-v2 v1.34.0
github.com/aws/aws-sdk-go-v2/config v1.29.2
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54
github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1
github.com/bmatcuk/doublestar/v4 v4.8.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/cheggaaa/pb/v3 v3.1.5
github.com/cheggaaa/pb/v3 v3.1.6
github.com/ergochat/readline v0.1.3
github.com/gogo/protobuf v1.3.2
github.com/golang/snappy v0.0.4
@@ -45,7 +42,7 @@ require (
golang.org/x/net v0.34.0
golang.org/x/oauth2 v0.25.0
golang.org/x/sys v0.29.0
google.golang.org/api v0.216.0
google.golang.org/api v0.218.0
gopkg.in/yaml.v2 v2.4.0
)
@@ -59,22 +56,22 @@ require (
github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.51 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 // indirect
github.com/aws/smithy-go v1.22.1 // indirect
github.com/aws/aws-sdk-go v1.55.6 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.55 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 // indirect
github.com/aws/smithy-go v1.22.2 // indirect
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 // indirect
@@ -107,7 +104,7 @@ require (
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.20.5 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.61.0 // indirect
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/prometheus/sigv4 v0.1.1 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
@@ -116,15 +113,15 @@ require (
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/collector/pdata v1.23.0 // indirect
go.opentelemetry.io/collector/semconv v0.117.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.58.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
go.opentelemetry.io/otel v1.33.0 // indirect
go.opentelemetry.io/otel/metric v1.33.0 // indirect
go.opentelemetry.io/collector/pdata v1.24.0 // indirect
go.opentelemetry.io/collector/semconv v0.118.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.59.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect
go.opentelemetry.io/otel v1.34.0 // indirect
go.opentelemetry.io/otel/metric v1.34.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.33.0 // indirect
go.opentelemetry.io/otel/trace v1.33.0 // indirect
go.opentelemetry.io/otel/trace v1.34.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/goleak v1.3.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
@@ -133,14 +130,14 @@ require (
golang.org/x/sync v0.10.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/time v0.9.0 // indirect
google.golang.org/genproto v0.0.0-20250106144421-5f5ef82da422 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250106144421-5f5ef82da422 // indirect
google.golang.org/grpc v1.69.2 // indirect
google.golang.org/protobuf v1.36.2 // indirect
google.golang.org/genproto v0.0.0-20250127172529-29210b9bc287 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250127172529-29210b9bc287 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 // indirect
google.golang.org/grpc v1.70.0 // indirect
google.golang.org/protobuf v1.36.4 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apimachinery v0.32.0 // indirect
k8s.io/client-go v0.32.0 // indirect
k8s.io/apimachinery v0.32.1 // indirect
k8s.io/client-go v0.32.1 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
)

176
go.sum
View File

@@ -14,10 +14,10 @@ cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyX
cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1 h1:1mvYtZfWQAnwNah/C+Z+Jb9rQH95LPE2vlmMuWAHJk8=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1/go.mod h1:75I/mXtme1JyWFtz8GocPHVFyH421IBoZErnO16dd0k=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.1 h1:Bk5uOhSAenHyR5P61D/NzeQCv+4fEVV8mOkJ82NqpWw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.1/go.mod h1:QZ4pw3or1WPmRBxf0cHd1tknzrT54WPBOQoGutCPvSU=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4=
@@ -26,8 +26,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 h1:mlmW46Q0B79I+Aj4azKC6xDMFN9a9SyZWESlGWYXbFs=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0/go.mod h1:PXe2h+LKcWTX9afWdZoHyODqR4fBa5boUM/8uJfZ0Jo=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ=
@@ -53,57 +53,57 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v1.32.8 h1:cZV+NUS/eGxKXMtmyhtYPJ7Z4YLoI/V8bkTdRZfYhGo=
github.com/aws/aws-sdk-go-v2 v1.32.8/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
github.com/aws/aws-sdk-go-v2/config v1.28.10 h1:fKODZHfqQu06pCzR69KJ3GuttraRJkhlC8g80RZ0Dfg=
github.com/aws/aws-sdk-go-v2/config v1.28.10/go.mod h1:PvdxRYZ5Um9QMq9PQ0zHHNdtKK+he2NHtFCUFMXWXeg=
github.com/aws/aws-sdk-go-v2/credentials v1.17.51 h1:F/9Sm6Y6k4LqDesZDPJCLxQGXNNHd/ZtJiWd0lCZKRk=
github.com/aws/aws-sdk-go-v2/credentials v1.17.51/go.mod h1:TKbzCHm43AoPyA+iLGGcruXd4AFhF8tOmLex2R9jWNQ=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 h1:IBAoD/1d8A8/1aA8g4MBVtTRHhXRiNAgwdbo/xRM2DI=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23/go.mod h1:vfENuCM7dofkgKpYzuzf1VT1UKkA/YL3qanfBn7HCaA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48 h1:XnXVe2zRyPf0+fAW5L05esmngvBpC6DQZK7oZB/z/Co=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48/go.mod h1:S3wey90OrS4f7kYxH6PT175YyEcHTORY07++HurMaRM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 h1:jSJjSBzw8VDIbWv+mmvBSP8ezsztMYJGH+eKqi9AmNs=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27/go.mod h1:/DAhLbFRgwhmvJdOfSm+WwikZrCuUJiA4WgJG0fTNSw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 h1:l+X4K77Dui85pIj5foXDhPlnqcNRG2QUyvca300lXh8=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27/go.mod h1:KvZXSFEXm6x84yE8qffKvT3x8J5clWnVFXphpohhzJ8=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 h1:AmB5QxnD+fBFrg9LcqzkgF/CaYvMyU/BTlejG4t1S7Q=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27/go.mod h1:Sai7P3xTiyv9ZUYO3IFxMnmiIP759/67iQbU4kdmkyU=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 h1:iwYS40JnrBeA9e9aI5S6KKN4EB2zR4iUVYN0nwVivz4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8/go.mod h1:Fm9Mi+ApqmFiknZtGpohVcBGvpTu542VC4XO9YudRi0=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 h1:cWno7lefSH6Pp+mSznagKCgfDGeZRin66UvYUqAkyeA=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8/go.mod h1:tPD+VjU3ABTBoEJ3nctu5Nyg4P4yjqSH5bJGGkY4+XE=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 h1:/Mn7gTedG86nbpjT4QEKsN1D/fThiYe1qvq7WsBGNHg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8/go.mod h1:Ae3va9LPmvjj231ukHB6UeT8nS7wTPfC3tMZSZMwNYg=
github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2 h1:a7aQ3RW+ug4IbhoQp29NZdc7vqrzKZZfWZSaQAXOZvQ=
github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2/go.mod h1:xMekrnhmJ5aqmyxtmALs7mlvXw5xRh+eYjOjvrIIFJ4=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 h1:YqtxripbjWb2QLyzRK9pByfEDvgg95gpC2AyDq4hFE8=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.9/go.mod h1:lV8iQpg6OLOfBnqbGMBKYjilBlf633qwHnBEiMSPoHY=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 h1:6dBT1Lz8fK11m22R+AqfRsFn8320K0T5DTGxxOQBSMw=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8/go.mod h1:/kiBvRQXBc6xeJTYzhSdGvJ5vm1tjaDEjH+MSeRJnlY=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 h1:VwhTrsTuVn52an4mXx29PqRzs2Dvu921NpGk7y43tAM=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.6/go.mod h1:+8h7PZb3yY5ftmVLD7ocEoE98hdc8PoKS0H3wfx1dlc=
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk=
github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v1.34.0 h1:9iyL+cjifckRGEVpRKZP3eIxVlL06Qk1Tk13vreaVQU=
github.com/aws/aws-sdk-go-v2 v1.34.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg=
github.com/aws/aws-sdk-go-v2/config v1.29.2 h1:JuIxOEPcSKpMB0J+khMjznG9LIhIBdmqNiEcPclnwqc=
github.com/aws/aws-sdk-go-v2/config v1.29.2/go.mod h1:HktTHregOZwNSM/e7WTfVSu9RCX+3eOv+6ij27PtaYs=
github.com/aws/aws-sdk-go-v2/credentials v1.17.55 h1:CDhKnDEaGkLA5ZszV/qw5uwN5M8rbv9Cl0JRN+PRsaM=
github.com/aws/aws-sdk-go-v2/credentials v1.17.55/go.mod h1:kPD/vj+RB5MREDUky376+zdnjZpR+WgdBBvwrmnlmKE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 h1:kU7tmXNaJ07LsyN3BUgGqAmVmQtq0w6duVIHAKfp0/w=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25/go.mod h1:OiC8+OiqrURb1wrwmr/UbOVLFSWEGxjinj5C299VQdo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54 h1:6BWOAho3Cgdy4cmNJ4HWY8VZgqODEU7Gw78XXireNZI=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54/go.mod h1:n+t/oyYErOV3jf/GxNTVlizSM9RMV1yH7jvcIvld3Do=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 h1:Ej0Rf3GMv50Qh4G4852j2djtoDb7AzQ7MuQeFHa3D70=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29/go.mod h1:oeNTC7PwJNoM5AznVr23wxhLnuJv0ZDe5v7w0wqIs9M=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 h1:6e8a71X+9GfghragVevC5bZqvATtc3mAMgxpSNbgzF0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29/go.mod h1:c4jkZiQ+BWpNqq7VtrxjwISrLrt/VvPq3XiopkUIolI=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 h1:g9OUETuxA8i/Www5Cby0R3WSTe7ppFTZXHVLNskNS4w=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29/go.mod h1:CQk+koLR1QeY1+vm7lqNfFii07DEderKq6T3F1L2pyc=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 h1:EP1ITDgYVPM2dL1bBBntJ7AW5yTjuWGz9XO+CZwpALU=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3/go.mod h1:5lWNWeAgWenJ/BZ/CP9k9DjLbC0pjnM045WjXRPPi14=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 h1:hN4yJBGswmFTOVYqmbz1GBs9ZMtQe8SrYxPwrkrlRv8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10/go.mod h1:TsxON4fEZXyrKY+D+3d2gSTyJkGORexIYab9PTf56DA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 h1:fXoWC2gi7tdJYNTPnnlSGzEVwewUchOi8xVq/dkg8Qs=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10/go.mod h1:cvzBApD5dVazHU8C2rbBQzzzsKc8m5+wNJ9mCRZLKPc=
github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1 h1:9LawY3cDJ3HE+v2GMd5SOkNLDwgN4K7TsCjyVBYu/L4=
github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1/go.mod h1:hHnELVnIHltd8EOF3YzahVX6F6y2C6dNqpRj1IMkS5I=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 h1:kznaW4f81mNMlREkU9w3jUuJvU5g/KsqDV43ab7Rp6s=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.12/go.mod h1:bZy9r8e0/s0P7BSDHgMLXK2KvdyRRBIQ2blKlvLt0IU=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 h1:mUwIpAvILeKFnRx4h1dEgGEFGuV8KJ3pEScZWVFYuZA=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11/go.mod h1:JDJtD+b8HNVv71axz8+S5492KM8wTzHRFpMKQbPlYxw=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 h1:g9d+TOsu3ac7SgmY2dUf1qMgu/uJVTlQ4VCbH6hRxSw=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.10/go.mod h1:WZfNmntu92HO44MVZAubQaz3qCuIdeOdog2sADfU6hU=
github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bmatcuk/doublestar/v4 v4.7.1 h1:fdDeAqgT47acgwd9bd9HxJRDmc9UAmPpc+2m0CXv75Q=
github.com/bmatcuk/doublestar/v4 v4.7.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cheggaaa/pb/v3 v3.1.5 h1:QuuUzeM2WsAqG2gMqtzaWithDJv0i+i6UlnwSCI4QLk=
github.com/cheggaaa/pb/v3 v3.1.5/go.mod h1:CrxkeghYTXi1lQBEI7jSn+3svI3cuc19haAj6jM60XI=
github.com/cheggaaa/pb/v3 v3.1.6 h1:h0x+vd7EiUohAJ29DJtJy+SNAc55t/elW3jCD086EXk=
github.com/cheggaaa/pb/v3 v3.1.6/go.mod h1:urxmfVtaxT+9aWk92DbsvXFZtNSWQSO5TRAp+MJ3l1s=
github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 h1:boJj011Hh+874zpIySeApCX4GeOjPl9qhRF3QuIZq+Q=
github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
@@ -298,16 +298,16 @@ github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+
github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ=
github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s=
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/prometheus/prometheus v0.301.0 h1:0z8dgegmILivNomCd79RKvVkIols8vBGPKmcIBc7OyY=
github.com/prometheus/prometheus v0.301.0/go.mod h1:BJLjWCKNfRfjp7Q48DrAjARnCi7GhfUVvUFEAWTssZM=
github.com/prometheus/sigv4 v0.1.1 h1:UJxjOqVcXctZlwDjpUpZ2OiMWJdFijgSofwLzO1Xk0Q=
github.com/prometheus/sigv4 v0.1.1/go.mod h1:RAmWVKqx0bwi0Qm4lrKMXFM0nhpesBcenfCtz9qRyH8=
github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -359,26 +359,26 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/collector/pdata v1.23.0 h1:tEk0dkfB8RdSukoOMfEa8duB938gfZowdfRkrJxGDrw=
go.opentelemetry.io/collector/pdata v1.23.0/go.mod h1:I2jggpBMiO8A+7TXhzNpcJZkJtvi1cU0iVNIi+6bc+o=
go.opentelemetry.io/collector/semconv v0.117.0 h1:SavOvSbHPVD/QdAnXlI/cMca+yxCNyXStY1mQzerHs4=
go.opentelemetry.io/collector/semconv v0.117.0/go.mod h1:N6XE8Q0JKgBN2fAhkUQtqK9LT7rEGR6+Wu/Rtbal1iI=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 h1:PS8wXpbyaDJQ2VDHHncMe9Vct0Zn1fEjpsjrLxGJoSc=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0/go.mod h1:HDBUsEjOuRC0EzKZ1bSaRGZWUBAzo+MhAcUUORSr4D0=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.58.0 h1:xwH3QJv6zL4u+gkPUu59NeT1Gyw9nScWT8FQpKLUJJI=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.58.0/go.mod h1:uosvgpqTcTXtcPQORTbEkZNDQTCDOgTz1fe6aLSyqrQ=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q=
go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw=
go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I=
go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ=
go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M=
go.opentelemetry.io/collector/pdata v1.24.0 h1:D6j92eAzmAbQgivNBUnt8r9juOl8ugb+ihYynoFZIEg=
go.opentelemetry.io/collector/pdata v1.24.0/go.mod h1:cf3/W9E/uIvPS4MR26SnMFJhraUCattzzM6qusuONuc=
go.opentelemetry.io/collector/semconv v0.118.0 h1:V4vlMIK7TIaemrrn2VawvQPwruIKpj7Xgw9P5+BL56w=
go.opentelemetry.io/collector/semconv v0.118.0/go.mod h1:N6XE8Q0JKgBN2fAhkUQtqK9LT7rEGR6+Wu/Rtbal1iI=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.59.0 h1:iQZYNQ7WwIcYXzOPR46FQv9O0dS1PW16RjvR0TjDOe8=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.59.0/go.mod h1:54CaSNqYEXvpzDh8KPjiMVoWm60t5R0dZRt0leEPgAs=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I=
go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ=
go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE=
go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM=
go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM=
go.opentelemetry.io/otel/sdk/metric v1.33.0 h1:Gs5VK9/WUJhNXZgn8MR6ITatvAmKeIuCtNbsP3JkNqU=
go.opentelemetry.io/otel/sdk/metric v1.33.0/go.mod h1:dL5ykHZmm1B1nVRk9dDjChwDmt81MjVp3gLkQRwKf/Q=
go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s=
go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck=
go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k=
go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -437,18 +437,18 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.216.0 h1:xnEHy+xWFrtYInWPy8OdGFsyIfWJjtVnO39g7pz2BFY=
google.golang.org/api v0.216.0/go.mod h1:K9wzQMvWi47Z9IU7OgdOofvZuw75Ge3PPITImZR/UyI=
google.golang.org/genproto v0.0.0-20250106144421-5f5ef82da422 h1:6GUHKGv2huWOHKmDXLMNE94q3fBDlEHI+oTRIZSebK0=
google.golang.org/genproto v0.0.0-20250106144421-5f5ef82da422/go.mod h1:1NPAxoesyw/SgLPqaUp9u1f9PWCLAk/jVmhx7gJZStg=
google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 h1:GVIKPyP/kLIyVOgOnTwFOrvQaQUzOzGMCxgFUOEmm24=
google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422/go.mod h1:b6h1vNKhxaSoEI+5jc3PJUCustfli/mRab7295pY7rw=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250106144421-5f5ef82da422 h1:3UsHvIr4Wc2aW4brOaSCmcxh9ksica6fHEr8P1XhkYw=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250106144421-5f5ef82da422/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4=
google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU=
google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
google.golang.org/api v0.218.0 h1:x6JCjEWeZ9PFCRe9z0FBrNwj7pB7DOAqT35N+IPnAUA=
google.golang.org/api v0.218.0/go.mod h1:5VGHBAkxrA/8EFjLVEYmMUJ8/8+gWWQ3s4cFH0FxG2M=
google.golang.org/genproto v0.0.0-20250127172529-29210b9bc287 h1:WoUI1G0DQ648FKvSl756SKxHQR/bI+y4HyyIQfxMWI8=
google.golang.org/genproto v0.0.0-20250127172529-29210b9bc287/go.mod h1:wkQ2Aj/xvshAUDtO/JHvu9y+AaN9cqs28QuSVSHtZSY=
google.golang.org/genproto/googleapis/api v0.0.0-20250127172529-29210b9bc287 h1:A2ni10G3UlplFrWdCDJTl7D7mJ7GSRm37S+PDimaKRw=
google.golang.org/genproto/googleapis/api v0.0.0-20250127172529-29210b9bc287/go.mod h1:iYONQfRdizDB8JJBybql13nArx91jcUk7zCXEsOofM4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 h1:J1H9f+LEdWAfHcez/4cvaVBox7cOYT+IU6rgqj5x++8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk=
google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ=
google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw=
google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM=
google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -464,12 +464,12 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/api v0.32.0 h1:OL9JpbvAU5ny9ga2fb24X8H6xQlVp+aJMFlgtQjR9CE=
k8s.io/api v0.32.0/go.mod h1:4LEwHZEf6Q/cG96F3dqR965sYOfmPM7rq81BLgsE0p0=
k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg=
k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8=
k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8=
k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc=
k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k=
k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs=
k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU=
k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y=

View File

@@ -8,6 +8,8 @@ import (
"strings"
"github.com/VictoriaMetrics/metrics"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
// AvailableCPUs returns the number of available CPU cores for the app.
@@ -44,6 +46,9 @@ func updateGOMAXPROCSToCPUQuota(cpuQuota float64) {
if gomaxprocs <= 0 {
gomaxprocs = 1
}
if cpuQuota > float64(gomaxprocs) {
logger.Warnf("rounding CPU quota %.1f to %d CPUs for performance reasons - see https://docs.victoriametrics.com/bestpractices/#kubernetes", cpuQuota, gomaxprocs)
}
numCPU := runtime.NumCPU()
if gomaxprocs > numCPU {

View File

@@ -1,19 +0,0 @@
package envutil
import (
"os"
"strconv"
)
// GetenvBool retrieves the value of the environment variable named by the key,
// attempts to convert the value to bool type and returns the result. In order
// for conversion to succeed, the value must be any value supported by
// strconv.ParseBool() function, otherwise the function will return false.
func GetenvBool(key string) bool {
s := os.Getenv(key)
b, err := strconv.ParseBool(s)
if err != nil {
return false
}
return b
}

View File

@@ -1,37 +0,0 @@
package envutil
import (
"os"
"testing"
)
func TestGetenvBool(t *testing.T) {
f := func(value string, want bool) {
t.Helper()
key := "VM_LIB_ENVUTIL_TEST_GETENV_BOOL"
os.Setenv(key, value)
defer os.Unsetenv(key)
if got := GetenvBool(key); got != want {
t.Errorf("GetenvBool(%s=%s) unexpected return value: got %t, want %t", key, value, got, want)
}
}
f("TRUE", true)
f("True", true)
f("true", true)
f("T", true)
f("t", true)
f("1", true)
f("FALSE", false)
f("False", false)
f("false", false)
f("F", false)
f("f", false)
f("0", false)
f("", false)
f("unsupported", false)
f("tRuE", false)
}

View File

@@ -9,7 +9,7 @@ import (
"sync"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fsutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
"github.com/VictoriaMetrics/metrics"
@@ -19,8 +19,6 @@ var disableFadvise = flag.Bool("filestream.disableFadvise", false, "Whether to d
"The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. "+
"In some rare cases it is better to disable the syscall if it uses too much CPU")
var disableFSyncForTesting = envutil.GetenvBool("DISABLE_FSYNC_FOR_TESTING")
const dontNeedBlockSize = 16 * 1024 * 1024
// ReadCloser is a standard interface for filestream Reader.
@@ -244,9 +242,9 @@ func (w *Writer) MustClose() {
putBufioWriter(w.bw)
w.bw = nil
if !disableFSyncForTesting {
if !fsutil.IsFsyncDisabled() {
if err := w.f.Sync(); err != nil {
logger.Panicf("FATAL: cannot sync file %q: %d", w.f.Name(), err)
logger.Panicf("FATAL: cannot sync file %q: %s", w.f.Name(), err)
}
}
if err := w.st.close(); err != nil {

View File

@@ -11,14 +11,11 @@ import (
"sync/atomic"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
var disableFSyncForTesting = envutil.GetenvBool("DISABLE_FSYNC_FOR_TESTING")
var tmpFileNum atomic.Uint64
// MustSyncPath syncs contents of the given path.

View File

@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fsutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"golang.org/x/sys/unix"
)
@@ -30,7 +31,7 @@ func mustSyncPath(path string) {
if err != nil {
logger.Panicf("FATAL: cannot open file for fsync: %s", err)
}
if !disableFSyncForTesting {
if !fsutil.IsFsyncDisabled() {
if err := d.Sync(); err != nil {
_ = d.Close()
logger.Panicf("FATAL: cannot flush %q to storage: %s", path, err)

View File

@@ -6,6 +6,7 @@ import (
"fmt"
"os"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fsutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"golang.org/x/sys/unix"
)
@@ -23,7 +24,7 @@ func mustSyncPath(path string) {
if err != nil {
logger.Panicf("FATAL: cannot open file for fsync: %s", err)
}
if !disableFSyncForTesting {
if !fsutil.IsFsyncDisabled() {
if err := d.Sync(); err != nil {
_ = d.Close()
logger.Panicf("FATAL: cannot flush %q to storage: %s", path, err)

View File

@@ -98,7 +98,7 @@ func mUnmap(data []byte) error {
mmapByAddrLock.Lock()
h, ok := mmapByAddr[addr]
if !ok {
logger.Fatalf("BUG: unmapping for non exist addr: %d", addr)
logger.Panicf("BUG: unmapping for non exist addr: %d", addr)
}
delete(mmapByAddr, addr)
mmapByAddrLock.Unlock()

32
lib/fs/fsutil/fsutil.go Normal file
View File

@@ -0,0 +1,32 @@
package fsutil
import (
"os"
"strconv"
"testing"
)
// IsFsyncDisabled returns true if fsync must be disabled
//
// The fsync is disabled in tests, since it significantly slows down tests which work with files.
// The fsync can be enabled in tests by setting DISABLE_FSYNC_FOR_TESTING environment variable to false.
//
// The fsync is enabled for ordinary programs. It can be disabled by setting DISABLE_FSYNC_FOR_TESTING
// environment variable to true.
func IsFsyncDisabled() bool {
return isFsyncDisabled
}
var isFsyncDisabled = isFsyncDisabledInternal()
func isFsyncDisabledInternal() bool {
s := os.Getenv("DISABLE_FSYNC_FOR_TESTING")
if s == "" {
return testing.Testing()
}
b, err := strconv.ParseBool(s)
if err != nil {
return false
}
return b
}

View File

@@ -0,0 +1,39 @@
package fsutil
import (
"os"
"testing"
)
func TestIsFsyncDisabledInternal(t *testing.T) {
f := func(envVarValue string, resultExpected bool) {
t.Helper()
os.Setenv("DISABLE_FSYNC_FOR_TESTING", envVarValue)
defer os.Unsetenv("DISABLE_FSYNC_FOR_TESTING")
result := isFsyncDisabledInternal()
if result != resultExpected {
t.Errorf("unexpected value for DISABLE_FSYNC_FOR_TESTING=%q; got %v; want %v", envVarValue, result, resultExpected)
}
}
// fsync must be unconditionally disabled in tests
f("", true)
f("TRUE", true)
f("True", true)
f("true", true)
f("T", true)
f("t", true)
f("1", true)
f("FALSE", false)
f("False", false)
f("false", false)
f("F", false)
f("f", false)
f("0", false)
f("unsupported", false)
f("tRuE", false)
}

View File

@@ -3,6 +3,7 @@ package logstorage
import (
"fmt"
"io"
"math"
"strings"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -19,37 +20,17 @@ func mustWriteColumnNames(w *writerWithStats, columnNames []string) {
func mustReadColumnNames(r filestream.ReadCloser) ([]string, map[string]uint64) {
src, err := io.ReadAll(r)
if err != nil {
logger.Panicf("FATAL: %s: cannot read colum names: %s", r.Path(), err)
logger.Panicf("FATAL: %s: cannot read column names: %s", r.Path(), err)
}
columnNames, err := unmarshalColumnNames(src)
columnNames, columnNameIDs, err := unmarshalColumnNames(src)
if err != nil {
logger.Panicf("FATAL: %s: %s", r.Path(), err)
}
columnNameIDs, err := getColumnNameIDs(columnNames)
if err != nil {
logger.Panicf("BUG: %s: %s; columnNames=%v", r.Path(), err, columnNameIDs)
}
return columnNames, columnNameIDs
}
func getColumnNameIDs(columnNames []string) (map[string]uint64, error) {
m := make(map[uint64]string, len(columnNames))
columnNameIDs := make(map[string]uint64, len(columnNames))
for i, name := range columnNames {
id := uint64(i)
if prevName, ok := m[id]; ok {
return nil, fmt.Errorf("duplicate column name id=%d for columns %q and %q", id, prevName, name)
}
m[id] = name
columnNameIDs[name] = id
}
return columnNameIDs, nil
}
func marshalColumnNames(dst []byte, columnNames []string) []byte {
data := encoding.MarshalVarUint64(nil, uint64(len(columnNames)))
data = marshalStrings(data, columnNames)
@@ -59,45 +40,50 @@ func marshalColumnNames(dst []byte, columnNames []string) []byte {
return dst
}
func unmarshalColumnNames(src []byte) ([]string, error) {
func unmarshalColumnNames(src []byte) ([]string, map[string]uint64, error) {
data, err := encoding.DecompressZSTD(nil, src)
if err != nil {
return nil, fmt.Errorf("cannot decompress column names from len(src)=%d: %w", len(src), err)
return nil, nil, fmt.Errorf("cannot decompress column names from len(src)=%d: %w", len(src), err)
}
src = data
n, nBytes := encoding.UnmarshalVarUint64(src)
if nBytes <= 0 {
return nil, fmt.Errorf("cannot parse the number of column names for len(src)=%d", len(src))
return nil, nil, fmt.Errorf("cannot parse the number of column names for len(src)=%d", len(src))
}
src = src[nBytes:]
if n > math.MaxInt {
return nil, nil, fmt.Errorf("too many distinct column names: %d; musn't exceed %d", n, math.MaxInt)
}
m := make(map[string]uint64, n)
dataBuf := make([]byte, len(src))
copy(dataBuf, src)
columnNameIDs := make(map[string]uint64, n)
columnNames := make([]string, n)
for id := uint64(0); id < n; id++ {
name, nBytes := encoding.UnmarshalBytes(dataBuf)
name, nBytes := encoding.UnmarshalBytes(src)
if nBytes <= 0 {
return nil, fmt.Errorf("cannot parse colum name number %d out of %d", id, n)
return nil, nil, fmt.Errorf("cannot parse column name number %d out of %d", id, n)
}
dataBuf = dataBuf[nBytes:]
src = src[nBytes:]
nameStr := bytesutil.ToUnsafeString(name)
// It should be good idea to intern column names, since usually the number of unique column names is quite small,
// even for wide events (e.g. less than a few thousands). So, if the average length of the column name
// exceeds 8 bytes (this is a typical case for Kubernetes with long column names), then interning saves some RAM.
nameStr := bytesutil.InternBytes(name)
if idPrev, ok := m[nameStr]; ok {
return nil, fmt.Errorf("duplicate ids for column name %q: %d and %d", name, idPrev, id)
if idPrev, ok := columnNameIDs[nameStr]; ok {
return nil, nil, fmt.Errorf("duplicate ids for column name %q: %d and %d", name, idPrev, id)
}
m[nameStr] = id
columnNameIDs[nameStr] = id
columnNames[id] = nameStr
}
if len(dataBuf) > 0 {
return nil, fmt.Errorf("unexpected non-empty tail left after unmarshaling column name ids; len(tail)=%d", len(dataBuf))
if len(src) > 0 {
return nil, nil, fmt.Errorf("unexpected non-empty tail left after unmarshaling column name ids; len(tail)=%d", len(src))
}
return columnNames, nil
return columnNames, columnNameIDs, nil
}
type columnNameIDGenerator struct {

View File

@@ -10,12 +10,23 @@ func TestMarshalUnmarshalColumnNames(t *testing.T) {
t.Helper()
data := marshalColumnNames(nil, columnNames)
result, err := unmarshalColumnNames(data)
resultColumnNames, resultColumnNameIDs, err := unmarshalColumnNames(data)
if err != nil {
t.Fatalf("unexpected error when unmarshaling columnNames: %s", err)
}
if !reflect.DeepEqual(columnNames, result) {
t.Fatalf("unexpected umarshaled columnNames\ngot\n%v\nwant\n%v", result, columnNames)
// Check columnNames
if !reflect.DeepEqual(resultColumnNames, columnNames) {
t.Fatalf("unexpected umarshaled columnNames\ngot\n%v\nwant\n%v", resultColumnNames, columnNames)
}
// Check columnNameIDs
expectedColumnNameIDs := make(map[string]uint64)
for i, n := range columnNames {
expectedColumnNameIDs[n] = uint64(i)
}
if !reflect.DeepEqual(resultColumnNameIDs, expectedColumnNameIDs) {
t.Fatalf("unexpected columnNameIDs\ngot\n%v\nwant\n%v", resultColumnNameIDs, expectedColumnNameIDs)
}
}

View File

@@ -29,10 +29,10 @@ const maxRowsPerBlock = 8 * 1024 * 1024
// in excess memory usage during data ingestion and significant slowdown during query execution.
const maxColumnsPerBlock = 2_000
// MaxFieldNameSize is the maximum size in bytes for field name.
// maxFieldNameSize is the maximum size in bytes for field name.
//
// Longer field names are truncated during data ingestion to MaxFieldNameSize length.
const MaxFieldNameSize = 128
// Log entries with longer field names are rejected during data ingestion.
const maxFieldNameSize = 128
// maxConstColumnValueSize is the maximum size in bytes for const column value.
//

View File

@@ -65,15 +65,6 @@ func (sf *sortedFields) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}
// RowFormatter implementes fmt.Stringer for []Field aka a single log row
type RowFormatter []Field
// String returns user-readable representation for rf
func (rf *RowFormatter) String() string {
result := MarshalFieldsToJSON(nil, *rf)
return string(result)
}
// Reset resets lr with all its settings.
//
// Call ResetKeepSettings() for resetting lr without resetting its settings.
@@ -142,22 +133,33 @@ func (lr *LogRows) NeedFlush() bool {
// It is OK to modify the args after returning from the function,
// since lr copies all the args to internal data.
//
// Field names longer than MaxFieldNameSize are automatically truncated to MaxFieldNameSize length.
//
// Log entries with too big number of fields are ignored.
// Loo long log entries are ignored.
// Log entries are dropped with the warning message in the following cases:
// - if there are too many log fields
// - if there are too long log field names
// - if the total length of log entries is too long
func (lr *LogRows) MustAdd(tenantID TenantID, timestamp int64, fields, streamFields []Field) {
// Verify that the log entry doesn't exceed limits.
if len(fields) > maxColumnsPerBlock {
fieldNames := make([]string, len(fields))
for i, f := range fields {
fieldNames[i] = f.Name
}
logger.Infof("ignoring log entry with too big number of fields, which exceeds %d; fieldNames=%q", maxColumnsPerBlock, fieldNames)
line := MarshalFieldsToJSON(nil, fields)
logger.Warnf("ignoring log entry with too big number of fields %d, since it exceeds the limit %d; "+
"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain ; log entry: %s", len(fields), maxColumnsPerBlock, line)
return
}
for i := range fields {
fieldName := fields[i].Name
if len(fieldName) > maxFieldNameSize {
line := MarshalFieldsToJSON(nil, fields)
logger.Warnf("ignoring log entry with too long field name %q, since its length (%d) exceeds the limit %d bytes; "+
"see https://docs.victoriametrics.com/victorialogs/faq/#what-is-the-maximum-supported-field-name-length ; log entry: %s",
fieldName, len(fieldName), maxFieldNameSize, line)
return
}
}
rowLen := uncompressedRowSizeBytes(fields)
if rowLen > maxUncompressedBlockSize {
logger.Infof("ignoring too long log entry with the estimated size %d bytes, since it exceeds the limit %d", rowLen, maxUncompressedBlockSize)
line := MarshalFieldsToJSON(nil, fields)
logger.Warnf("ignoring too long log entry with the estimated length of %d bytes, since it exceeds the limit %d bytes; "+
"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have ; log entry: %s", rowLen, maxUncompressedBlockSize, line)
return
}
@@ -247,9 +249,6 @@ func (lr *LogRows) addFieldsInternal(fields []Field, ignoreFields map[string]str
dstField := &fb[len(fb)-1]
fieldName := f.Name
if len(fieldName) > MaxFieldNameSize {
fieldName = fieldName[:MaxFieldNameSize]
}
if fieldName == "_msg" {
fieldName = ""
hasMsgField = true
@@ -266,20 +265,21 @@ func (lr *LogRows) addFieldsInternal(fields []Field, ignoreFields map[string]str
func (lr *LogRows) GetRowString(idx int) string {
tf := TimeFormatter(lr.timestamps[idx])
streamTags := getStreamTagsString(lr.streamTagsCanonicals[idx])
var rf RowFormatter
rf = append(rf[:0], lr.rows[idx]...)
rf = append(rf, Field{
var fields []Field
fields = append(fields[:0], lr.rows[idx]...)
fields = append(fields, Field{
Name: "_time",
Value: tf.String(),
})
rf = append(rf, Field{
fields = append(fields, Field{
Name: "_stream",
Value: streamTags,
})
sort.Slice(rf, func(i, j int) bool {
return rf[i].Name < rf[j].Name
sort.Slice(fields, func(i, j int) bool {
return fields[i].Name < fields[j].Name
})
return rf.String()
line := MarshalFieldsToJSON(nil, fields)
return string(line)
}
// GetLogRows returns LogRows from the pool for the given streamFields.

View File

@@ -259,6 +259,9 @@ type queryOptions struct {
//
// By default the number of concurrent workers equals to the number of available CPU cores.
concurrency uint
// if ignoreGlobalTimeFilter is set, then Query.AddTimeFilter doesn't add the time filter to the query and to all its subqueries.
ignoreGlobalTimeFilter *bool
}
func (opts *queryOptions) String() string {
@@ -269,6 +272,9 @@ func (opts *queryOptions) String() string {
if opts.concurrency > 0 {
a = append(a, fmt.Sprintf("concurrency=%d", opts.concurrency))
}
if opts.ignoreGlobalTimeFilter != nil {
a = append(a, fmt.Sprintf("ignore_global_time_filter=%v", *opts.ignoreGlobalTimeFilter))
}
if len(a) == 0 {
return ""
}
@@ -486,6 +492,16 @@ func (q *Query) AddTimeFilter(start, end int64) {
stringRepr: fmt.Sprintf("[%s, %s]", startStr, endStr),
}
q.visitSubqueries(func(q *Query) {
q.addTimeFilterNoSubqueries(ft)
})
}
func (q *Query) addTimeFilterNoSubqueries(ft *filterTime) {
if q.opts.ignoreGlobalTimeFilter != nil && *q.opts.ignoreGlobalTimeFilter {
return
}
fa, ok := q.f.(*filterAnd)
if ok {
filters := make([]filter, len(fa.filters)+1)
@@ -506,6 +522,12 @@ func (q *Query) AddExtraFilters(extraFilters *Filter) {
}
filters := []filter{extraFilters.f}
q.visitSubqueries(func(q *Query) {
q.addExtraFiltersNoSubqueries(filters)
})
}
func (q *Query) addExtraFiltersNoSubqueries(filters []filter) {
fa, ok := q.f.(*filterAnd)
if ok {
fa.filters = append(filters, fa.filters...)
@@ -527,6 +549,12 @@ func (q *Query) AddPipeLimit(n uint64) {
// optimize applies various optimations to q.
func (q *Query) optimize() {
q.visitSubqueries(func(q *Query) {
q.optimizeNoSubqueries()
})
}
func (q *Query) optimizeNoSubqueries() {
q.pipes = optimizeSortOffsetPipes(q.pipes)
q.pipes = optimizeSortLimitPipes(q.pipes)
q.pipes = optimizeUniqLimitPipes(q.pipes)
@@ -562,6 +590,39 @@ func (q *Query) optimize() {
q.f = mergeFiltersStream(q.f)
}
func (q *Query) visitSubqueries(visitFunc func(q *Query)) {
// call f for the query itself.
visitFunc(q)
// Visit subqueries in all the filters at q.
visitSubqueriesInFilter(q.f, visitFunc)
// Visit subqueries in all the pipes at q.
for _, p := range q.pipes {
p.visitSubqueries(visitFunc)
}
}
func visitSubqueriesInFilter(f filter, visitFunc func(q *Query)) {
if f == nil {
return
}
callback := func(f filter) bool {
switch t := f.(type) {
case *filterIn:
if t.q != nil {
t.q.visitSubqueries(visitFunc)
}
case *filterStreamID:
if t.q != nil {
t.q.visitSubqueries(visitFunc)
}
}
return false
}
_ = visitFilter(f, callback)
}
func mergeFiltersStream(f filter) filter {
fa, ok := f.(*filterAnd)
if !ok {
@@ -808,7 +869,7 @@ func flattenFiltersAnd(f filter) filter {
}
f, err := copyFilter(f, visitFunc, copyFunc)
if err != nil {
logger.Fatalf("BUG: unexpected error: %s", err)
logger.Panicf("BUG: unexpected error: %s", err)
}
return f
}
@@ -844,7 +905,7 @@ func flattenFiltersOr(f filter) filter {
}
f, err := copyFilter(f, visitFunc, copyFunc)
if err != nil {
logger.Fatalf("BUG: unexpected error: %s", err)
logger.Panicf("BUG: unexpected error: %s", err)
}
return f
}
@@ -861,7 +922,7 @@ func removeStarFilters(f filter) filter {
}
f, err := copyFilter(f, visitFunc, copyFunc)
if err != nil {
logger.Fatalf("BUG: unexpected error: %s", err)
logger.Panicf("BUG: unexpected error: %s", err)
}
// Drop filterNoop inside filterAnd
@@ -897,7 +958,7 @@ func removeStarFilters(f filter) filter {
}
f, err = copyFilter(f, visitFunc, copyFunc)
if err != nil {
logger.Fatalf("BUG: unexpected error: %s", err)
logger.Panicf("BUG: unexpected error: %s", err)
}
return f
@@ -1206,6 +1267,12 @@ func parseQueryOptions(lex *lexer) (*queryOptions, error) {
n = 1024
}
opts.concurrency = uint(n)
case "ignore_global_time_filter":
ignoreGlobalTimeFilter, err := strconv.ParseBool(v)
if err != nil {
return nil, fmt.Errorf("cannot parse 'ignore_global_time_filter=%q' option as boolean: %w", v, err)
}
opts.ignoreGlobalTimeFilter = &ignoreGlobalTimeFilter
default:
return nil, fmt.Errorf("unexpected option %q with value %q", k, v)
}

Some files were not shown because too many files have changed in this diff Show More