mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-06-04 09:31:57 +03:00
Compare commits
173 Commits
v1.136.0
...
issue-1065
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
534c57b79a | ||
|
|
e47abd6385 | ||
|
|
c04a5a597d | ||
|
|
e695d5f425 | ||
|
|
2bb03f6e34 | ||
|
|
92f03344eb | ||
|
|
e3360b87ff | ||
|
|
4c98b912fa | ||
|
|
225e2e870b | ||
|
|
2b078301c1 | ||
|
|
14090c5a07 | ||
|
|
66d47f23e4 | ||
|
|
eacdb80ed7 | ||
|
|
504cf31dab | ||
|
|
34d190b32a | ||
|
|
44fa216bb5 | ||
|
|
4589442345 | ||
|
|
78ad4b974c | ||
|
|
d12524749f | ||
|
|
1a5235a18f | ||
|
|
27847dbbb8 | ||
|
|
33fab3a2d6 | ||
|
|
695b21ecfc | ||
|
|
4ba488f806 | ||
|
|
1e046d35a8 | ||
|
|
8c9b202c94 | ||
|
|
060423141d | ||
|
|
0ee16ff2e5 | ||
|
|
b22853b97f | ||
|
|
b578fe9817 | ||
|
|
e9b7adc0e5 | ||
|
|
82eab5c5b7 | ||
|
|
5b4ab4456e | ||
|
|
d3ccc8d7a7 | ||
|
|
eb34bdd8d9 | ||
|
|
3139fa1c9b | ||
|
|
8f4cdb8a42 | ||
|
|
f236801fa4 | ||
|
|
2c48133ad8 | ||
|
|
1cb634858e | ||
|
|
4b45f909b5 | ||
|
|
4ae495bd1d | ||
|
|
925b0ecdc9 | ||
|
|
1348b0e424 | ||
|
|
83656e544d | ||
|
|
38a76eca7b | ||
|
|
dea915c10d | ||
|
|
b3f57c113b | ||
|
|
686c9a21ff | ||
|
|
8f215137e7 | ||
|
|
ed5dc35876 | ||
|
|
13ab8cfb78 | ||
|
|
f8a101e45e | ||
|
|
a1a35fd870 | ||
|
|
0d5df2722d | ||
|
|
db3353c6e1 | ||
|
|
cfbc5ae31d | ||
|
|
fdb3c96fc1 | ||
|
|
486d923351 | ||
|
|
f8552bdc96 | ||
|
|
893c981c57 | ||
|
|
3d7ff783b6 | ||
|
|
78543b7f87 | ||
|
|
f54d22562a | ||
|
|
b672e05dce | ||
|
|
847871b916 | ||
|
|
2aecca1163 | ||
|
|
d1efb2dd37 | ||
|
|
6882c72075 | ||
|
|
60eb543dba | ||
|
|
7db42b0659 | ||
|
|
8d924f0631 | ||
|
|
791679253d | ||
|
|
a745bb797a | ||
|
|
3607c53b7c | ||
|
|
7969647553 | ||
|
|
5f887b66c5 | ||
|
|
d3e2946791 | ||
|
|
603dc03c7d | ||
|
|
1cc471a6c1 | ||
|
|
d40adb1e58 | ||
|
|
8056806d5f | ||
|
|
3d67942a65 | ||
|
|
23bdd14cee | ||
|
|
18a2955553 | ||
|
|
570a9ef627 | ||
|
|
40e27fc2c8 | ||
|
|
befbf9afca | ||
|
|
65d0a8e129 | ||
|
|
c2841ca36c | ||
|
|
cd2026e430 | ||
|
|
216821aa1c | ||
|
|
ef507d372b | ||
|
|
e383b62f59 | ||
|
|
8f34284dd2 | ||
|
|
8f4eca39f7 | ||
|
|
d467faf739 | ||
|
|
673b2ca7db | ||
|
|
40ccf0c333 | ||
|
|
fe341a4204 | ||
|
|
83ebf00659 | ||
|
|
5e602726f5 | ||
|
|
a6200cc83d | ||
|
|
a5811d3c3b | ||
|
|
5962b47c31 | ||
|
|
9a4edc738a | ||
|
|
30d01e9cae | ||
|
|
6b46f3920c | ||
|
|
97b11146ee | ||
|
|
2ef74bd6ea | ||
|
|
845161e377 | ||
|
|
f176a6624a | ||
|
|
4d06e34b66 | ||
|
|
6d8ddcb9ed | ||
|
|
dd4167709a | ||
|
|
71e253e1f0 | ||
|
|
9e155ffd9e | ||
|
|
2e9e40dc75 | ||
|
|
10d4294f9b | ||
|
|
5e77771668 | ||
|
|
dda5545078 | ||
|
|
087efbc451 | ||
|
|
68e64536b1 | ||
|
|
6e3ce4d55c | ||
|
|
8d1b88f985 | ||
|
|
3d3c057d52 | ||
|
|
94622fef29 | ||
|
|
804d77ffc5 | ||
|
|
79b18e9742 | ||
|
|
3404a47a6d | ||
|
|
0b8205ef46 | ||
|
|
53514febdc | ||
|
|
8531d86da0 | ||
|
|
a47d32e129 | ||
|
|
df96f4d3ab | ||
|
|
84dc5453ad | ||
|
|
8093d98c0e | ||
|
|
809f9471df | ||
|
|
f9d6d2e428 | ||
|
|
32eac31416 | ||
|
|
4d4c1ff72e | ||
|
|
645ce2b6b3 | ||
|
|
89600bd229 | ||
|
|
9b3a60efee | ||
|
|
a8c5934d1b | ||
|
|
43544fdb63 | ||
|
|
7a4df5755a | ||
|
|
83bcbc43d1 | ||
|
|
79921cf434 | ||
|
|
40402fdac3 | ||
|
|
05943abc11 | ||
|
|
e66e71c87e | ||
|
|
7f682c4c76 | ||
|
|
4947cd7f14 | ||
|
|
5ea7314912 | ||
|
|
655f0e9c1d | ||
|
|
2ffd25a120 | ||
|
|
175fcf6676 | ||
|
|
c05516afbe | ||
|
|
6b12684e56 | ||
|
|
8f7c94f512 | ||
|
|
4a6259a9b2 | ||
|
|
d5b9d3e641 | ||
|
|
6863de2c0e | ||
|
|
51a3e4e27a | ||
|
|
d7046d6e19 | ||
|
|
7e6c03e9c6 | ||
|
|
5267f35104 | ||
|
|
172ff84299 | ||
|
|
a3f955dd84 | ||
|
|
19e7d986fe | ||
|
|
db2ad6f900 | ||
|
|
db1f3f4ab8 |
23
.github/copilot-instructions.md
vendored
23
.github/copilot-instructions.md
vendored
@@ -1,23 +0,0 @@
|
||||
# Project Overview
|
||||
|
||||
VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
|
||||
|
||||
## Folder Structure
|
||||
|
||||
- `/app`: Contains the compilable binaries.
|
||||
- `/lib`: Contains the golang reusable libraries
|
||||
- `/docs/victoriametrics`: Contains documentation for the project.
|
||||
- `/apptest/tests`: Contains integration tests.
|
||||
|
||||
## Libraries and Frameworks
|
||||
|
||||
- Backend: Golang, no framework. Use third-party libraries sparingly.
|
||||
- Frontend: React.
|
||||
|
||||
## Code review guidelines
|
||||
|
||||
Ensure the feature or bugfix includes a changelog entry in /docs/victoriametrics/changelog/CHANGELOG.md.
|
||||
Verify the entry is under the ## tip section and matches the structure and style of existing entries.
|
||||
Chore-only changes may be omitted from the changelog.
|
||||
|
||||
|
||||
2
.github/workflows/docs.yaml
vendored
2
.github/workflows/docs.yaml
vendored
@@ -28,7 +28,7 @@ jobs:
|
||||
path: __vm-docs
|
||||
|
||||
- name: Import GPG key
|
||||
uses: crazy-max/ghaction-import-gpg@v6
|
||||
uses: crazy-max/ghaction-import-gpg@v7
|
||||
id: import-gpg
|
||||
with:
|
||||
gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
|
||||
|
||||
4
.github/workflows/test.yml
vendored
4
.github/workflows/test.yml
vendored
@@ -86,7 +86,7 @@ jobs:
|
||||
- run: go version
|
||||
|
||||
- name: Run tests
|
||||
run: GOGC=10 make ${{ matrix.scenario}}
|
||||
run: make ${{ matrix.scenario}}
|
||||
|
||||
- name: Publish coverage
|
||||
uses: codecov/codecov-action@v5
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
apptest:
|
||||
name: apptest
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: apptest
|
||||
|
||||
steps:
|
||||
- name: Code checkout
|
||||
|
||||
25
SECURITY.md
25
SECURITY.md
@@ -12,6 +12,31 @@ The following versions of VictoriaMetrics receive regular security fixes:
|
||||
|
||||
See [this page](https://victoriametrics.com/security/) for more details.
|
||||
|
||||
## Software Bill of Materials (SBOM)
|
||||
|
||||
Every VictoriaMetrics container{{% available_from "#" %}} image published to
|
||||
[Docker Hub](https://hub.docker.com/u/victoriametrics)
|
||||
and [Quay.io](https://quay.io/organization/victoriametrics)
|
||||
includes an [SPDX](https://spdx.dev/) SBOM attestation
|
||||
generated automatically by BuildKit during
|
||||
`docker buildx build`.
|
||||
|
||||
To inspect the SBOM for an image:
|
||||
|
||||
```sh
|
||||
docker buildx imagetools inspect \
|
||||
docker.io/victoriametrics/victoria-metrics:latest \
|
||||
--format "{{ json .SBOM }}"
|
||||
```
|
||||
|
||||
To scan an image using its SBOM attestation with
|
||||
[Trivy](https://github.com/aquasecurity/trivy):
|
||||
|
||||
```sh
|
||||
trivy image --sbom-sources oci \
|
||||
docker.io/victoriametrics/victoria-metrics:latest
|
||||
```
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please report any security issues to <security@victoriametrics.com>
|
||||
|
||||
@@ -33,13 +33,13 @@ func PopulateTimeTpl(b []byte, tGlobal time.Time) []byte {
|
||||
}
|
||||
switch strings.TrimSpace(parts[0]) {
|
||||
case `TIME_S`:
|
||||
return []byte(fmt.Sprintf("%d", t.Unix()))
|
||||
return fmt.Appendf(nil, "%d", t.Unix())
|
||||
case `TIME_MSZ`:
|
||||
return []byte(fmt.Sprintf("%d", t.Unix()*1e3))
|
||||
return fmt.Appendf(nil, "%d", t.Unix()*1e3)
|
||||
case `TIME_MS`:
|
||||
return []byte(fmt.Sprintf("%d", timeToMillis(t)))
|
||||
return fmt.Appendf(nil, "%d", timeToMillis(t))
|
||||
case `TIME_NS`:
|
||||
return []byte(fmt.Sprintf("%d", t.UnixNano()))
|
||||
return fmt.Appendf(nil, "%d", t.UnixNano())
|
||||
default:
|
||||
log.Fatalf("unknown time pattern %s in %s", parts[0], repl)
|
||||
}
|
||||
|
||||
@@ -49,6 +49,11 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
|
||||
Name: "__name__",
|
||||
Value: m.Name,
|
||||
})
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557
|
||||
labels = append(labels, prompb.Label{
|
||||
Name: "host",
|
||||
Value: sketch.Host,
|
||||
})
|
||||
for _, label := range m.Labels {
|
||||
labels = append(labels, prompb.Label{
|
||||
Name: label.Name,
|
||||
@@ -57,9 +62,6 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
|
||||
}
|
||||
for _, tag := range sketch.Tags {
|
||||
name, value := datadogutil.SplitTag(tag)
|
||||
if name == "host" {
|
||||
name = "exported_host"
|
||||
}
|
||||
labels = append(labels, prompb.Label{
|
||||
Name: name,
|
||||
Value: value,
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestCalculateRetryDuration(t *testing.T) {
|
||||
f := func(retryAfterDuration, retryDuration time.Duration, n int, expectMinDuration time.Duration) {
|
||||
t.Helper()
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
for range n {
|
||||
retryDuration = getRetryDuration(retryAfterDuration, retryDuration, time.Minute)
|
||||
}
|
||||
|
||||
|
||||
@@ -51,9 +51,9 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
|
||||
|
||||
func newTestWriteRequest(seriesCount, labelsCount int) *prompb.WriteRequest {
|
||||
var wr prompb.WriteRequest
|
||||
for i := 0; i < seriesCount; i++ {
|
||||
for i := range seriesCount {
|
||||
var labels []prompb.Label
|
||||
for j := 0; j < labelsCount; j++ {
|
||||
for j := range labelsCount {
|
||||
labels = append(labels, prompb.Label{
|
||||
Name: fmt.Sprintf("label_%d_%d", i, j),
|
||||
Value: fmt.Sprintf("value_%d_%d", i, j),
|
||||
|
||||
@@ -38,7 +38,7 @@ var (
|
||||
labelsGlobal []prompb.Label
|
||||
|
||||
remoteWriteRelabelConfigData atomic.Pointer[[]byte]
|
||||
remoteWriteURLRelabelConfigData atomic.Pointer[[]interface{}]
|
||||
remoteWriteURLRelabelConfigData atomic.Pointer[[]any]
|
||||
|
||||
relabelConfigReloads *metrics.Counter
|
||||
relabelConfigReloadErrors *metrics.Counter
|
||||
@@ -90,8 +90,8 @@ func WriteURLRelabelConfigData(w io.Writer) {
|
||||
return
|
||||
}
|
||||
type urlRelabelCfg struct {
|
||||
Url string `yaml:"url"`
|
||||
RelabelConfig interface{} `yaml:"relabel_config"`
|
||||
Url string `yaml:"url"`
|
||||
RelabelConfig any `yaml:"relabel_config"`
|
||||
}
|
||||
var cs []urlRelabelCfg
|
||||
for i, url := range *remoteWriteURLs {
|
||||
@@ -144,7 +144,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
|
||||
len(*relabelConfigPaths), (len(*remoteWriteURLs)))
|
||||
}
|
||||
|
||||
var urlRelabelCfgs []interface{}
|
||||
var urlRelabelCfgs []any
|
||||
rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
|
||||
for i, path := range *relabelConfigPaths {
|
||||
if len(path) == 0 {
|
||||
@@ -157,7 +157,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
|
||||
}
|
||||
rcs.perURL[i] = prc
|
||||
|
||||
var parsedCfg interface{}
|
||||
var parsedCfg any
|
||||
_ = yaml.Unmarshal(rawCfg, &parsedCfg)
|
||||
urlRelabelCfgs = append(urlRelabelCfgs, parsedCfg)
|
||||
}
|
||||
|
||||
@@ -28,12 +28,12 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
|
||||
itemsCount := 1_000 * bucketsCount
|
||||
m := make([]int, bucketsCount)
|
||||
var labels []prompb.Label
|
||||
for i := 0; i < itemsCount; i++ {
|
||||
for i := range itemsCount {
|
||||
labels = append(labels[:0], prompb.Label{
|
||||
Name: "__name__",
|
||||
Value: fmt.Sprintf("some_name_%d", i),
|
||||
})
|
||||
for j := 0; j < 10; j++ {
|
||||
for j := range 10 {
|
||||
labels = append(labels, prompb.Label{
|
||||
Name: fmt.Sprintf("label_%d", j),
|
||||
Value: fmt.Sprintf("value_%d_%d", i, j),
|
||||
@@ -248,7 +248,7 @@ func TestShardAmountRemoteWriteCtx(t *testing.T) {
|
||||
seriesCount := 100000
|
||||
// build 1000000 series
|
||||
tssBlock := make([]prompb.TimeSeries, 0, seriesCount)
|
||||
for i := 0; i < seriesCount; i++ {
|
||||
for i := range seriesCount {
|
||||
tssBlock = append(tssBlock, prompb.TimeSeries{
|
||||
Labels: []prompb.Label{
|
||||
{
|
||||
@@ -269,7 +269,7 @@ func TestShardAmountRemoteWriteCtx(t *testing.T) {
|
||||
// build active time series set
|
||||
nodes := make([]string, 0, remoteWriteCount)
|
||||
activeTimeSeriesByNodes := make([]map[string]struct{}, remoteWriteCount)
|
||||
for i := 0; i < remoteWriteCount; i++ {
|
||||
for i := range remoteWriteCount {
|
||||
nodes = append(nodes, fmt.Sprintf("node%d", i))
|
||||
activeTimeSeriesByNodes[i] = make(map[string]struct{})
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ func TestParseInputValue_Success(t *testing.T) {
|
||||
if len(outputExpected) != len(output) {
|
||||
t.Fatalf("unexpected output length; got %d; want %d", len(outputExpected), len(output))
|
||||
}
|
||||
for i := 0; i < len(outputExpected); i++ {
|
||||
for i := range outputExpected {
|
||||
if outputExpected[i].Omitted != output[i].Omitted {
|
||||
t.Fatalf("unexpected Omitted field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
@@ -348,9 +350,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
|
||||
for k := range alertEvalTimesMap {
|
||||
alertEvalTimes = append(alertEvalTimes, k)
|
||||
}
|
||||
sort.Slice(alertEvalTimes, func(i, j int) bool {
|
||||
return alertEvalTimes[i] < alertEvalTimes[j]
|
||||
})
|
||||
slices.Sort(alertEvalTimes)
|
||||
|
||||
// sort group eval order according to the given "group_eval_order".
|
||||
sort.Slice(testGroups, func(i, j int) bool {
|
||||
@@ -361,12 +361,8 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
|
||||
var groups []*rule.Group
|
||||
for _, group := range testGroups {
|
||||
mergedExternalLabels := make(map[string]string)
|
||||
for k, v := range tg.ExternalLabels {
|
||||
mergedExternalLabels[k] = v
|
||||
}
|
||||
for k, v := range externalLabels {
|
||||
mergedExternalLabels[k] = v
|
||||
}
|
||||
maps.Copy(mergedExternalLabels, tg.ExternalLabels)
|
||||
maps.Copy(mergedExternalLabels, externalLabels)
|
||||
ng := rule.NewGroup(group, q, time.Minute, mergedExternalLabels)
|
||||
ng.Init()
|
||||
groups = append(groups, ng)
|
||||
|
||||
@@ -81,12 +81,9 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
|
||||
if g.Interval.Duration() < 0 {
|
||||
return fmt.Errorf("interval shouldn't be lower than 0")
|
||||
}
|
||||
if g.EvalOffset.Duration() < 0 {
|
||||
return fmt.Errorf("eval_offset shouldn't be lower than 0")
|
||||
}
|
||||
// if `eval_offset` is set, interval won't use global evaluationInterval flag and must bigger than offset.
|
||||
if g.EvalOffset.Duration() > g.Interval.Duration() {
|
||||
return fmt.Errorf("eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
|
||||
// if `eval_offset` is set, the group interval must be specified explicitly(instead of inherited from global evaluationInterval flag) and must bigger than offset.
|
||||
if g.EvalOffset.Duration().Abs() > g.Interval.Duration() {
|
||||
return fmt.Errorf("the abs value of eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
|
||||
}
|
||||
if g.EvalOffset != nil && g.EvalDelay != nil {
|
||||
return fmt.Errorf("eval_offset cannot be used with eval_delay")
|
||||
|
||||
@@ -176,11 +176,17 @@ func TestGroupValidate_Failure(t *testing.T) {
|
||||
}, false, "interval shouldn't be lower than 0")
|
||||
|
||||
f(&Group{
|
||||
Name: "wrong eval_offset",
|
||||
Name: "too big eval_offset",
|
||||
Interval: promutil.NewDuration(time.Minute),
|
||||
EvalOffset: promutil.NewDuration(2 * time.Minute),
|
||||
}, false, "eval_offset should be smaller than interval")
|
||||
|
||||
f(&Group{
|
||||
Name: "too big negative eval_offset",
|
||||
Interval: promutil.NewDuration(time.Minute),
|
||||
EvalOffset: promutil.NewDuration(-2 * time.Minute),
|
||||
}, false, "eval_offset should be smaller than interval")
|
||||
|
||||
limit := -1
|
||||
f(&Group{
|
||||
Name: "wrong limit",
|
||||
|
||||
@@ -2,6 +2,7 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaLogs/lib/logstorage"
|
||||
@@ -80,12 +81,8 @@ func (t *Type) ValidateExpr(expr string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain labels from LogsQL expr: %q, err: %w", expr, err)
|
||||
}
|
||||
for i := range labels {
|
||||
// VictoriaLogs inserts `_time` field as a label in result when query with `stats by (_time:step)`,
|
||||
// making the result meaningless and may lead to cardinality issues.
|
||||
if labels[i] == "_time" {
|
||||
return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
|
||||
}
|
||||
if slices.Contains(labels, "_time") {
|
||||
return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknown datasource type=%q", t.Name)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -91,9 +92,7 @@ func (c *Client) Clone() *Client {
|
||||
ns.extraHeaders = make([]keyValue, len(c.extraHeaders))
|
||||
copy(ns.extraHeaders, c.extraHeaders)
|
||||
}
|
||||
for k, v := range c.extraParams {
|
||||
ns.extraParams[k] = v
|
||||
}
|
||||
maps.Copy(ns.extraParams, c.extraParams)
|
||||
|
||||
return ns
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ type promResponse struct {
|
||||
// Stats supported by VictoriaMetrics since v1.90
|
||||
Stats struct {
|
||||
SeriesFetched *string `json:"seriesFetched,omitempty"`
|
||||
} `json:"stats,omitempty"`
|
||||
} `json:"stats"`
|
||||
// IsPartial supported by VictoriaMetrics
|
||||
IsPartial *bool `json:"isPartial,omitempty"`
|
||||
}
|
||||
|
||||
@@ -134,7 +134,7 @@ func (ls Labels) String() string {
|
||||
func LabelCompare(a, b Labels) int {
|
||||
l := min(len(b), len(a))
|
||||
|
||||
for i := 0; i < l; i++ {
|
||||
for i := range l {
|
||||
if a[i].Name != b[i].Name {
|
||||
if a[i].Name < b[i].Name {
|
||||
return -1
|
||||
|
||||
@@ -13,7 +13,7 @@ func BenchmarkPromInstantUnmarshal(b *testing.B) {
|
||||
|
||||
// BenchmarkParsePrometheusResponse/Instant_std+fastjson-10 1760 668959 ns/op 280147 B/op 5781 allocs/op
|
||||
b.Run("Instant std+fastjson", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
for range b.N {
|
||||
var pi promInstant
|
||||
err = pi.Unmarshal(data)
|
||||
if err != nil {
|
||||
|
||||
@@ -56,7 +56,7 @@ absolute path to all .tpl files in root.
|
||||
-rule.templates="dir/**/*.tpl". Includes all the .tpl files in "dir" subfolders recursively.
|
||||
`)
|
||||
|
||||
configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
|
||||
configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule', '-rule.templates' and '-notifier.config' files. "+
|
||||
"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
|
||||
|
||||
httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "Address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
|
||||
|
||||
@@ -69,7 +69,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
|
||||
for n := range workers {
|
||||
wg.Go(func() {
|
||||
r := rand.New(rand.NewSource(int64(n)))
|
||||
for i := 0; i < iterations; i++ {
|
||||
for range iterations {
|
||||
rnd := r.Intn(len(paths))
|
||||
cfg, err := config.Parse([]string{paths[rnd]}, notifier.ValidateTemplates, true)
|
||||
if err != nil { // update can fail and this is expected
|
||||
|
||||
@@ -216,7 +216,7 @@ consul_sd_configs:
|
||||
for n := range workers {
|
||||
wg.Go(func() {
|
||||
r := rand.New(rand.NewSource(int64(n)))
|
||||
for i := 0; i < iterations; i++ {
|
||||
for range iterations {
|
||||
rnd := r.Intn(len(paths))
|
||||
_ = cw.reload(paths[rnd]) // update can fail and this is expected
|
||||
_ = cw.notifiers()
|
||||
|
||||
@@ -113,7 +113,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
|
||||
input: make(chan prompb.TimeSeries, cfg.MaxQueueSize),
|
||||
}
|
||||
|
||||
for i := 0; i < cc; i++ {
|
||||
for range cc {
|
||||
c.run(ctx)
|
||||
}
|
||||
return c, nil
|
||||
@@ -186,6 +186,11 @@ func (c *Client) run(ctx context.Context) {
|
||||
return
|
||||
case <-ticker.C:
|
||||
c.flush(ctx, wr)
|
||||
// drain the potential stale tick to avoid small or empty flushes after a slow flush.
|
||||
select {
|
||||
case <-ticker.C:
|
||||
default:
|
||||
}
|
||||
case ts, ok := <-c.input:
|
||||
if !ok {
|
||||
continue
|
||||
@@ -238,8 +243,10 @@ func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
|
||||
defer func() {
|
||||
sendDuration.Add(time.Since(timeStart).Seconds())
|
||||
}()
|
||||
|
||||
attempts := 0
|
||||
L:
|
||||
for attempts := 0; ; attempts++ {
|
||||
for {
|
||||
err := c.send(ctx, b)
|
||||
if err != nil && (errors.Is(err, io.EOF) || netutil.IsTrivialNetworkError(err)) {
|
||||
// Something in the middle between client and destination might be closing
|
||||
@@ -281,6 +288,7 @@ L:
|
||||
time.Sleep(retryInterval)
|
||||
retryInterval *= 2
|
||||
|
||||
attempts++
|
||||
}
|
||||
|
||||
rwErrors.Inc()
|
||||
|
||||
@@ -44,7 +44,7 @@ func TestClient_Push(t *testing.T) {
|
||||
|
||||
r := rand.New(rand.NewSource(1))
|
||||
const rowsN = int(1e4)
|
||||
for i := 0; i < rowsN; i++ {
|
||||
for range rowsN {
|
||||
s := prompb.TimeSeries{
|
||||
Samples: []prompb.Sample{{
|
||||
Value: r.Float64(),
|
||||
@@ -102,7 +102,7 @@ func TestClient_run_maxBatchSizeDuringShutdown(t *testing.T) {
|
||||
}
|
||||
|
||||
// push time series to the client.
|
||||
for i := 0; i < pushCnt; i++ {
|
||||
for range pushCnt {
|
||||
if err = rwClient.Push(prompb.TimeSeries{}); err != nil {
|
||||
t.Fatalf("cannot time series to the client: %s", err)
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestDebugClient_Push(t *testing.T) {
|
||||
|
||||
const rowsN = 100
|
||||
var sent int
|
||||
for i := 0; i < rowsN; i++ {
|
||||
for i := range rowsN {
|
||||
s := prompb.TimeSeries{
|
||||
Samples: []prompb.Sample{{
|
||||
Value: float64(i),
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"maps"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -30,8 +31,8 @@ var (
|
||||
"0 means no limit.")
|
||||
ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
|
||||
"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overridden per rule via update_entries_limit param.")
|
||||
resendDelay = flag.Duration("rule.resendDelay", 0, "MiniMum amount of time to wait before resending an alert to notifier.")
|
||||
maxResolveDuration = flag.Duration("rule.maxResolveDuration", 0, "Limits the maxiMum duration for automatic alert expiration, "+
|
||||
resendDelay = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier.")
|
||||
maxResolveDuration = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
|
||||
"which by default is 4 times evaluationInterval of the parent group")
|
||||
evalDelay = flag.Duration("rule.evalDelay", 30*time.Second, "Adjustment of the 'time' parameter for rule evaluation requests to compensate intentional data delay from the datasource. "+
|
||||
"Normally, should be equal to '-search.latencyOffset' (cmd-line flag configured for VictoriaMetrics single-node or vmselect). "+
|
||||
@@ -97,9 +98,7 @@ type groupMetrics struct {
|
||||
// set2 has priority over set1.
|
||||
func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[string]string {
|
||||
r := map[string]string{}
|
||||
for k, v := range set1 {
|
||||
r[k] = v
|
||||
}
|
||||
maps.Copy(r, set1)
|
||||
for k, v := range set2 {
|
||||
if prevV, ok := r[k]; ok {
|
||||
logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
|
||||
@@ -485,8 +484,15 @@ func (g *Group) UpdateWith(newGroup *Group) {
|
||||
// delayBeforeStart calculates delay based on Group ID, so all groups will start at different moments of time.
|
||||
func (g *Group) delayBeforeStart(ts time.Time, maxDelay time.Duration) time.Duration {
|
||||
if g.EvalOffset != nil {
|
||||
offset := *g.EvalOffset
|
||||
// adjust the offset for negative evalOffset, the rule is:
|
||||
// `eval_offset: -x` is equivalent to `eval_offset: y` for `interval: x+y`.
|
||||
// For example, `eval_offset: -6m` is equivalent to `eval_offset: 4m` for `interval: 10m`.
|
||||
if offset < 0 {
|
||||
offset += g.Interval
|
||||
}
|
||||
// if offset is specified, ignore the maxDelay and return a duration aligned with offset
|
||||
currentOffsetPoint := ts.Truncate(g.Interval).Add(*g.EvalOffset)
|
||||
currentOffsetPoint := ts.Truncate(g.Interval).Add(offset)
|
||||
if currentOffsetPoint.Before(ts) {
|
||||
// wait until the next offset point
|
||||
return currentOffsetPoint.Add(g.Interval).Sub(ts)
|
||||
@@ -495,11 +501,8 @@ func (g *Group) delayBeforeStart(ts time.Time, maxDelay time.Duration) time.Dura
|
||||
}
|
||||
|
||||
// otherwise, return a random duration between [0..min(interval, maxDelay)] based on group ID
|
||||
interval := g.Interval
|
||||
if interval > maxDelay {
|
||||
// artificially limit interval, so groups with big intervals could start sooner.
|
||||
interval = maxDelay
|
||||
}
|
||||
// artificially limit interval, so groups with big intervals could start sooner.
|
||||
interval := min(g.Interval, maxDelay)
|
||||
var randSleep time.Duration
|
||||
randSleep = time.Duration(float64(interval) * (float64(g.GetID()) / (1 << 64)))
|
||||
sleepOffset := time.Duration(ts.UnixNano() % interval.Nanoseconds())
|
||||
|
||||
@@ -405,7 +405,8 @@ func TestGroupStart(t *testing.T) {
|
||||
|
||||
var cur uint64
|
||||
prev := g.metrics.iterationTotal.Get()
|
||||
for i := 0; ; i++ {
|
||||
i := 0
|
||||
for {
|
||||
if i > 40 {
|
||||
t.Fatalf("group wasn't able to perform %d evaluations during %d eval intervals", n, i)
|
||||
}
|
||||
@@ -414,6 +415,7 @@ func TestGroupStart(t *testing.T) {
|
||||
return
|
||||
}
|
||||
time.Sleep(interval)
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -604,6 +606,15 @@ func TestGroupStartDelay(t *testing.T) {
|
||||
f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
|
||||
f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
|
||||
|
||||
// test group with negative offset -2min, which is equivalent to 3min offset for 5min interval
|
||||
offset = -2 * time.Minute
|
||||
g.EvalOffset = &offset
|
||||
|
||||
f("2023-01-01T00:00:15.000+00:00", "2023-01-01T00:03:00.000+00:00")
|
||||
f("2023-01-01T00:01:00.000+00:00", "2023-01-01T00:03:00.000+00:00")
|
||||
f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
|
||||
f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
|
||||
|
||||
maxDelay = time.Minute * 1
|
||||
g.EvalOffset = nil
|
||||
|
||||
|
||||
@@ -121,7 +121,7 @@ func (s *ruleState) add(e StateEntry) {
|
||||
func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int) (int, error) {
|
||||
var err error
|
||||
var tss []prompb.TimeSeries
|
||||
for i := 0; i < replayRuleRetryAttempts; i++ {
|
||||
for i := range replayRuleRetryAttempts {
|
||||
tss, err = r.execRange(context.Background(), start, end)
|
||||
if err == nil {
|
||||
break
|
||||
|
||||
@@ -40,7 +40,7 @@ func TestRule_state(t *testing.T) {
|
||||
}
|
||||
|
||||
var last time.Time
|
||||
for i := 0; i < stateEntriesN*2; i++ {
|
||||
for range stateEntriesN * 2 {
|
||||
last = time.Now()
|
||||
r.state.add(StateEntry{At: last})
|
||||
}
|
||||
@@ -68,7 +68,7 @@ func TestRule_stateConcurrent(_ *testing.T) {
|
||||
var wg sync.WaitGroup
|
||||
for range workers {
|
||||
wg.Go(func() {
|
||||
for i := 0; i < iterations; i++ {
|
||||
for range iterations {
|
||||
r.state.add(StateEntry{At: time.Now()})
|
||||
r.state.getAll()
|
||||
r.state.getLast()
|
||||
|
||||
@@ -42,7 +42,7 @@ func TestErrGroupConcurrent(_ *testing.T) {
|
||||
|
||||
const writersN = 4
|
||||
payload := make(chan error, writersN)
|
||||
for i := 0; i < writersN; i++ {
|
||||
for range writersN {
|
||||
go func() {
|
||||
for err := range payload {
|
||||
eg.Add(err)
|
||||
@@ -51,7 +51,7 @@ func TestErrGroupConcurrent(_ *testing.T) {
|
||||
}
|
||||
|
||||
const iterations = 500
|
||||
for i := 0; i < iterations; i++ {
|
||||
for i := range iterations {
|
||||
payload <- fmt.Errorf("error %d", i)
|
||||
if i%10 == 0 {
|
||||
_ = eg.Err()
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -28,6 +29,7 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
|
||||
@@ -65,10 +67,11 @@ type AuthConfig struct {
|
||||
type UserInfo struct {
|
||||
Name string `yaml:"name,omitempty"`
|
||||
|
||||
BearerToken string `yaml:"bearer_token,omitempty"`
|
||||
AuthToken string `yaml:"auth_token,omitempty"`
|
||||
Username string `yaml:"username,omitempty"`
|
||||
Password string `yaml:"password,omitempty"`
|
||||
BearerToken string `yaml:"bearer_token,omitempty"`
|
||||
JWT *JWTConfig `yaml:"jwt,omitempty"`
|
||||
AuthToken string `yaml:"auth_token,omitempty"`
|
||||
Username string `yaml:"username,omitempty"`
|
||||
Password string `yaml:"password,omitempty"`
|
||||
|
||||
URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
|
||||
DiscoverBackendIPs *bool `yaml:"discover_backend_ips,omitempty"`
|
||||
@@ -89,6 +92,8 @@ type UserInfo struct {
|
||||
|
||||
MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
|
||||
|
||||
AccessLog *AccessLog `yaml:"access_log,omitempty"`
|
||||
|
||||
concurrencyLimitCh chan struct{}
|
||||
concurrencyLimitReached *metrics.Counter
|
||||
|
||||
@@ -101,11 +106,40 @@ type UserInfo struct {
|
||||
requestsDuration *metrics.Summary
|
||||
}
|
||||
|
||||
// AccessLog represents configuration for access log settings.
|
||||
type AccessLog struct {
|
||||
Filters *AccessLogFilters `yaml:"filters"`
|
||||
}
|
||||
|
||||
// AccessLogFilters represents list of filters for access logs printing
|
||||
type AccessLogFilters struct {
|
||||
// SkipStatusCodes is a list of HTTP status codes for which access logs will be skipped
|
||||
SkipStatusCodes []int `yaml:"skip_status_codes"`
|
||||
}
|
||||
|
||||
func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int, duration time.Duration) {
|
||||
if ui.AccessLog == nil {
|
||||
return
|
||||
}
|
||||
filters := ui.AccessLog.Filters
|
||||
if filters != nil && len(filters.SkipStatusCodes) > 0 {
|
||||
if slices.Contains(filters.SkipStatusCodes, statusCode) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
|
||||
requestURI := httpserver.GetRequestURI(r)
|
||||
logger.Infof("access_log request_host=%q request_uri=%q status_code=%d remote_addr=%s user_agent=%q referer=%q duration_ms=%d username=%q",
|
||||
r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), duration.Milliseconds(), userName)
|
||||
}
|
||||
|
||||
// HeadersConf represents config for request and response headers.
|
||||
type HeadersConf struct {
|
||||
RequestHeaders []*Header `yaml:"headers,omitempty"`
|
||||
ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
|
||||
KeepOriginalHost *bool `yaml:"keep_original_host,omitempty"`
|
||||
RequestHeaders []*Header `yaml:"headers,omitempty"`
|
||||
ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
|
||||
KeepOriginalHost *bool `yaml:"keep_original_host,omitempty"`
|
||||
hasAnyPlaceHolders bool
|
||||
}
|
||||
|
||||
func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
|
||||
@@ -113,7 +147,7 @@ func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
|
||||
case ui.concurrencyLimitCh <- struct{}{}:
|
||||
return nil
|
||||
default:
|
||||
// The number of concurrently executed requests for the given user equals the limt.
|
||||
// The number of concurrently executed requests for the given user equals the limit.
|
||||
// Wait until some of the currently executed requests are finished, so the current request could be executed.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
|
||||
select {
|
||||
@@ -348,6 +382,7 @@ func (bus *backendURLs) add(u *url.URL) {
|
||||
url: u,
|
||||
healthCheckContext: bus.healthChecksContext,
|
||||
healthCheckWG: &bus.healthChecksWG,
|
||||
hasPlaceHolders: hasAnyPlaceholders(u),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -365,6 +400,8 @@ type backendURL struct {
|
||||
concurrentRequests atomic.Int32
|
||||
|
||||
url *url.URL
|
||||
|
||||
hasPlaceHolders bool
|
||||
}
|
||||
|
||||
func (bu *backendURL) isBroken() bool {
|
||||
@@ -588,7 +625,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
|
||||
|
||||
// Slow path - select other backend urls.
|
||||
n := atomicCounter.Add(1) - 1
|
||||
for i := uint32(0); i < uint32(len(bus)); i++ {
|
||||
for i := range uint32(len(bus)) {
|
||||
idx := (n + i) % uint32(len(bus))
|
||||
bu := bus[idx]
|
||||
if bu.isBroken() {
|
||||
@@ -598,7 +635,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
|
||||
// The Load() in front of CompareAndSwap() avoids CAS overhead for items with values bigger than 0.
|
||||
if bu.concurrentRequests.Load() == 0 && bu.concurrentRequests.CompareAndSwap(0, 1) {
|
||||
atomicCounter.CompareAndSwap(n+1, idx+1)
|
||||
// There is no need in the call bu.get(), because we already incremented bu.concrrentRequests above.
|
||||
// There is no need in the call bu.get(), because we already incremented bu.concurrentRequests above.
|
||||
return bu
|
||||
}
|
||||
}
|
||||
@@ -799,6 +836,9 @@ var (
|
||||
// authUsers contains the currently loaded auth users
|
||||
authUsers atomic.Pointer[map[string]*UserInfo]
|
||||
|
||||
// jwt authentication cache
|
||||
jwtAuthCache atomic.Pointer[jwtCache]
|
||||
|
||||
authConfigWG sync.WaitGroup
|
||||
stopCh chan struct{}
|
||||
)
|
||||
@@ -838,6 +878,16 @@ func reloadAuthConfigData(data []byte) (bool, error) {
|
||||
return false, fmt.Errorf("failed to parse auth config: %w", err)
|
||||
}
|
||||
|
||||
jui, oidcDP, err := parseJWTUsers(ac)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to parse JWT users from auth config: %w", err)
|
||||
}
|
||||
oidcDP.startDiscovery()
|
||||
jwtc := &jwtCache{
|
||||
users: jui,
|
||||
oidcDP: oidcDP,
|
||||
}
|
||||
|
||||
m, err := parseAuthConfigUsers(ac)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to parse users from auth config: %w", err)
|
||||
@@ -854,9 +904,15 @@ func reloadAuthConfigData(data []byte) (bool, error) {
|
||||
}
|
||||
metrics.RegisterSet(ac.ms)
|
||||
|
||||
jwtcPrev := jwtAuthCache.Load()
|
||||
if jwtcPrev != nil {
|
||||
jwtcPrev.oidcDP.stopDiscovery()
|
||||
}
|
||||
|
||||
authConfig.Store(ac)
|
||||
authConfigData.Store(&data)
|
||||
authUsers.Store(&m)
|
||||
jwtAuthCache.Store(jwtc)
|
||||
|
||||
return true, nil
|
||||
}
|
||||
@@ -881,12 +937,18 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
|
||||
if ui.BearerToken != "" {
|
||||
return nil, fmt.Errorf("field bearer_token can't be specified for unauthorized_user section")
|
||||
}
|
||||
if ui.JWT != nil {
|
||||
return nil, fmt.Errorf("field jwt can't be specified for unauthorized_user section")
|
||||
}
|
||||
if ui.AuthToken != "" {
|
||||
return nil, fmt.Errorf("field auth_token can't be specified for unauthorized_user section")
|
||||
}
|
||||
if ui.Name != "" {
|
||||
return nil, fmt.Errorf("field name can't be specified for unauthorized_user section")
|
||||
}
|
||||
if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ui.initURLs(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -927,16 +989,27 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
|
||||
}
|
||||
for i := range uis {
|
||||
ui := &uis[i]
|
||||
// users with jwt tokens are parsed by parseJWTUsers function.
|
||||
// the function also checks that users with jwt tokens do not have auth tokens, bearer tokens, usernames and passwords.
|
||||
if ui.JWT != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
ats, err := getAuthTokens(ui.AuthToken, ui.BearerToken, ui.Username, ui.Password)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, at := range ats {
|
||||
if uiOld := byAuthToken[at]; uiOld != nil {
|
||||
return nil, fmt.Errorf("duplicate auth token=%q found for username=%q, name=%q; the previous one is set for username=%q, name=%q",
|
||||
at, ui.Username, ui.Name, uiOld.Username, uiOld.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ui.initURLs(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1036,6 +1109,7 @@ func (ui *UserInfo) initURLs() error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, e := range ui.URLMaps {
|
||||
if len(e.SrcPaths) == 0 && len(e.SrcHosts) == 0 && len(e.SrcQueryArgs) == 0 && len(e.SrcHeaders) == 0 {
|
||||
return fmt.Errorf("missing `src_paths`, `src_hosts`, `src_query_args` and `src_headers` in `url_map`")
|
||||
@@ -1095,6 +1169,9 @@ func (ui *UserInfo) name() string {
|
||||
h := xxhash.Sum64([]byte(ui.AuthToken))
|
||||
return fmt.Sprintf("auth_token:hash:%016X", h)
|
||||
}
|
||||
if ui.JWT != nil {
|
||||
return `jwt`
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
|
||||
@@ -4,8 +4,11 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
@@ -276,6 +279,50 @@ users:
|
||||
url_prefix: http://foo.bar
|
||||
metric_labels:
|
||||
not-prometheus-compatible: value
|
||||
`)
|
||||
// placeholder in url_prefix
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
password: bar
|
||||
url_prefix: 'http://ahost/{{a_placeholder}}/foobar'
|
||||
`)
|
||||
// placeholder in a header
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
password: bar
|
||||
headers:
|
||||
- 'X-Foo: {{a_placeholder}}'
|
||||
url_prefix: 'http://ahost'
|
||||
`)
|
||||
// placeholder in url_prefix
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
password: bar
|
||||
url_prefix: 'http://ahost/{{a_placeholder}}/foobar'
|
||||
`)
|
||||
// placeholder in a header in url_map
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
password: bar
|
||||
url_map:
|
||||
- src_paths: ["/select/.*"]
|
||||
headers:
|
||||
- 'X-Foo: {{a_placeholder}}'
|
||||
url_prefix: 'http://ahost'
|
||||
`)
|
||||
|
||||
// placeholder in a header in url_map
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
password: bar
|
||||
url_map:
|
||||
- src_paths: ["/select/.*"]
|
||||
url_prefix: 'http://ahost/{{a_placeholder}}/foobar'
|
||||
`)
|
||||
}
|
||||
|
||||
@@ -378,7 +425,7 @@ users:
|
||||
RetryStatusCodes: []int{500, 501},
|
||||
LoadBalancingPolicy: "first_available",
|
||||
MergeQueryArgs: []string{"foo", "bar"},
|
||||
DropSrcPathPrefixParts: intp(1),
|
||||
DropSrcPathPrefixParts: new(1),
|
||||
DiscoverBackendIPs: &discoverBackendIPsTrue,
|
||||
},
|
||||
}, nil)
|
||||
@@ -621,6 +668,47 @@ unauthorized_user:
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// skip user info with jwt, it is parsed by parseJWTUsers
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
password: bar
|
||||
url_prefix: http://aaa:343/bbb
|
||||
- jwt: {skip_verify: true}
|
||||
url_prefix: http://aaa:343/bbb
|
||||
`, map[string]*UserInfo{
|
||||
getHTTPAuthBasicToken("foo", "bar"): {
|
||||
Username: "foo",
|
||||
Password: "bar",
|
||||
URLPrefix: mustParseURL("http://aaa:343/bbb"),
|
||||
},
|
||||
}, nil)
|
||||
|
||||
// Multiple users with access logs enabled
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
url_prefix: http://foo
|
||||
access_log: {}
|
||||
- username: bar
|
||||
url_prefix: https://bar/x/
|
||||
access_log:
|
||||
filters:
|
||||
skip_status_codes: [404]
|
||||
`, map[string]*UserInfo{
|
||||
getHTTPAuthBasicToken("foo", ""): {
|
||||
Username: "foo",
|
||||
URLPrefix: mustParseURL("http://foo"),
|
||||
AccessLog: &AccessLog{},
|
||||
},
|
||||
getHTTPAuthBasicToken("bar", ""): {
|
||||
Username: "bar",
|
||||
URLPrefix: mustParseURL("https://bar/x/"),
|
||||
AccessLog: &AccessLog{Filters: &AccessLogFilters{SkipStatusCodes: []int{404}}},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
}
|
||||
|
||||
func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
|
||||
@@ -831,7 +919,7 @@ func TestBrokenBackend(t *testing.T) {
|
||||
bus[1].setBroken()
|
||||
|
||||
// broken backend should never return while there are healthy backends
|
||||
for i := 0; i < 1e3; i++ {
|
||||
for range int(1e3) {
|
||||
b := up.getBackendURL()
|
||||
if b.isBroken() {
|
||||
t.Fatalf("unexpected broken backend %q", b.url)
|
||||
@@ -908,6 +996,41 @@ func TestDiscoverBackendIPsWithIPV6(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestLogRequest(t *testing.T) {
|
||||
ui := &UserInfo{AccessLog: &AccessLog{}}
|
||||
|
||||
testOutput := &bytes.Buffer{}
|
||||
logger.SetOutputForTests(testOutput)
|
||||
defer logger.ResetOutputForTest()
|
||||
|
||||
req, err := http.NewRequest("GET", "http://localhost:8080/select/0/prometheus", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
f := func(user string, status int, duration time.Duration, expectedLog string) {
|
||||
t.Helper()
|
||||
|
||||
testOutput.Reset()
|
||||
ui.logRequest(req, user, status, duration)
|
||||
|
||||
got := testOutput.String()
|
||||
if expectedLog == "" && got != "" {
|
||||
t.Fatalf("expected empty log, got %q", got)
|
||||
}
|
||||
if !strings.Contains(got, expectedLog) {
|
||||
t.Fatalf("output \n%q \nshould contain \n%q", testOutput.String(), expectedLog)
|
||||
}
|
||||
}
|
||||
|
||||
f("foo", 200, 10*time.Millisecond, `access_log request_host="localhost:8080" request_uri="" status_code=200 remote_addr="" user_agent="" referer="" duration_ms=10 username="foo"`)
|
||||
f("foo", 404, time.Second, `access_log request_host="localhost:8080" request_uri="" status_code=404 remote_addr="" user_agent="" referer="" duration_ms=1000 username="foo"`)
|
||||
|
||||
ui.AccessLog.Filters = &AccessLogFilters{SkipStatusCodes: []int{200}}
|
||||
f("foo", 200, 10*time.Millisecond, ``)
|
||||
f("foo", 404, 10*time.Millisecond, `access_log request_host="localhost:8080" request_uri="" status_code=404 remote_addr="" user_agent="" referer="" duration_ms=10 username="foo"`)
|
||||
}
|
||||
|
||||
func getRegexs(paths []string) []*Regex {
|
||||
var sps []*Regex
|
||||
for _, path := range paths {
|
||||
@@ -963,10 +1086,6 @@ func mustParseURLs(us []string) *URLPrefix {
|
||||
return up
|
||||
}
|
||||
|
||||
func intp(n int) *int {
|
||||
return &n
|
||||
}
|
||||
|
||||
func mustNewRegex(s string) *Regex {
|
||||
var re Regex
|
||||
if err := yaml.Unmarshal([]byte(s), &re); err != nil {
|
||||
|
||||
@@ -116,6 +116,20 @@ users:
|
||||
- "http://default1:8888/unsupported_url_handler"
|
||||
- "http://default2:8888/unsupported_url_handler"
|
||||
|
||||
# A JWT token based routing:
|
||||
# - Requests with JWT token that has the following structure:
|
||||
# {"team": "ops", "security": {"read_access": "1"}, "vm_access": {"metrics_account_id": 1000,"metrics_project_id":5}}
|
||||
# is routed to vmselect nodes and request url placeholder replaced with metrics tenant identificators
|
||||
- name: jwt-opts-team
|
||||
jwt:
|
||||
match_claims:
|
||||
team: ops
|
||||
security.read_access: "1"
|
||||
skip_verify: true
|
||||
url_prefix:
|
||||
- "http://vmselect1:8481/select/{{.MetricsTenant}}/prometheus"
|
||||
- "http://vmselect2:8481/select/{{.MetricsTenant}}/prometheus"
|
||||
|
||||
# Requests without Authorization header are proxied according to `unauthorized_user` section.
|
||||
# Requests are proxied in round-robin fashion between `url_prefix` backends.
|
||||
# The deny_partial_response query arg is added to all the proxied requests.
|
||||
@@ -125,3 +139,8 @@ unauthorized_user:
|
||||
- http://vmselect-az1/?deny_partial_response=1
|
||||
- http://vmselect-az2/?deny_partial_response=1
|
||||
retry_status_codes: [503, 500]
|
||||
# log access for requests routed to this user
|
||||
access_log:
|
||||
filters:
|
||||
# except requests with Status Codes below
|
||||
skip_status_codes: [200, 202]
|
||||
|
||||
486
app/vmauth/jwt.go
Normal file
486
app/vmauth/jwt.go
Normal file
@@ -0,0 +1,486 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/jwt"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
metricsTenantPlaceholder = `{{.MetricsTenant}}`
|
||||
metricsExtraLabelsPlaceholder = `{{.MetricsExtraLabels}}`
|
||||
metricsExtraFiltersPlaceholder = `{{.MetricsExtraFilters}}`
|
||||
|
||||
logsAccountIDPlaceholder = `{{.LogsAccountID}}`
|
||||
logsProjectIDPlaceholder = `{{.LogsProjectID}}`
|
||||
logsExtraFiltersPlaceholder = `{{.LogsExtraFilters}}`
|
||||
logsExtraStreamFiltersPlaceholder = `{{.LogsExtraStreamFilters}}`
|
||||
|
||||
placeholderPrefix = `{{`
|
||||
)
|
||||
|
||||
var allPlaceholders = []string{
|
||||
metricsTenantPlaceholder,
|
||||
metricsExtraLabelsPlaceholder,
|
||||
metricsExtraFiltersPlaceholder,
|
||||
logsAccountIDPlaceholder,
|
||||
logsProjectIDPlaceholder,
|
||||
logsExtraFiltersPlaceholder,
|
||||
logsExtraStreamFiltersPlaceholder,
|
||||
}
|
||||
|
||||
var urlPathPlaceHolders = []string{
|
||||
metricsTenantPlaceholder,
|
||||
logsAccountIDPlaceholder,
|
||||
logsProjectIDPlaceholder,
|
||||
}
|
||||
|
||||
type jwtCache struct {
|
||||
// users contain UserInfo`s from AuthConfig with JWTConfig set
|
||||
users []*UserInfo
|
||||
|
||||
oidcDP *oidcDiscovererPool
|
||||
}
|
||||
|
||||
type JWTConfig struct {
|
||||
PublicKeys []string `yaml:"public_keys,omitempty"`
|
||||
PublicKeyFiles []string `yaml:"public_key_files,omitempty"`
|
||||
SkipVerify bool `yaml:"skip_verify,omitempty"`
|
||||
OIDC *oidcConfig `yaml:"oidc,omitempty"`
|
||||
MatchClaims map[string]string `yaml:"match_claims,omitempty"`
|
||||
parsedMatchClaims []*jwt.Claim
|
||||
|
||||
// verifierPool is used to verify JWT tokens.
|
||||
// It is initialized from PublicKeys and/or PublicKeyFiles.
|
||||
// In this case, it is initialized once at config reload and never updated until next reload
|
||||
// In case of OIDC, it is initialized on config reload and periodically updated by discovery process.
|
||||
verifierPool atomic.Pointer[jwt.VerifierPool]
|
||||
}
|
||||
|
||||
func parseJWTUsers(ac *AuthConfig) ([]*UserInfo, *oidcDiscovererPool, error) {
|
||||
jui := make([]*UserInfo, 0, len(ac.Users))
|
||||
oidcDP := &oidcDiscovererPool{}
|
||||
|
||||
uniqClaims := make(map[string]*UserInfo)
|
||||
var sortedClaims []string
|
||||
for idx, ui := range ac.Users {
|
||||
jwtToken := ui.JWT
|
||||
if jwtToken == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if ui.AuthToken != "" || ui.BearerToken != "" || ui.Username != "" || ui.Password != "" {
|
||||
return nil, nil, fmt.Errorf("auth_token, bearer_token, username and password cannot be specified if jwt is set")
|
||||
}
|
||||
if len(jwtToken.PublicKeys) == 0 && len(jwtToken.PublicKeyFiles) == 0 && !jwtToken.SkipVerify && jwtToken.OIDC == nil {
|
||||
return nil, nil, fmt.Errorf("jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true")
|
||||
}
|
||||
var claimsString string
|
||||
sortedClaims = sortedClaims[:0]
|
||||
parsedClaims := make([]*jwt.Claim, 0, len(jwtToken.MatchClaims))
|
||||
for ck, cv := range jwtToken.MatchClaims {
|
||||
sortedClaims = append(sortedClaims, fmt.Sprintf("%s=%s", ck, cv))
|
||||
pc, err := jwt.NewClaim(ck, cv)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("incorrect match claim, key=%q, value regex=%q: %w", ck, cv, err)
|
||||
}
|
||||
parsedClaims = append(parsedClaims, pc)
|
||||
}
|
||||
ui.JWT.parsedMatchClaims = parsedClaims
|
||||
sort.Strings(sortedClaims)
|
||||
claimsString = strings.Join(sortedClaims, ",")
|
||||
|
||||
if oldUI, ok := uniqClaims[claimsString]; ok {
|
||||
return nil, nil, fmt.Errorf("duplicate match claims=%q found for name=%q at idx=%d; the previous one is set for name=%q", claimsString, ui.Name, idx, oldUI.Name)
|
||||
}
|
||||
uniqClaims[claimsString] = &ui
|
||||
if len(jwtToken.PublicKeys) > 0 || len(jwtToken.PublicKeyFiles) > 0 {
|
||||
keys := make([]any, 0, len(jwtToken.PublicKeys)+len(jwtToken.PublicKeyFiles))
|
||||
|
||||
for i := range jwtToken.PublicKeys {
|
||||
k, err := jwt.ParseKey([]byte(jwtToken.PublicKeys[i]))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
|
||||
for _, filePath := range jwtToken.PublicKeyFiles {
|
||||
keyData, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot read public key from file %q: %w", filePath, err)
|
||||
}
|
||||
k, err := jwt.ParseKey(keyData)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot parse public key from file %q: %w", filePath, err)
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
|
||||
vp, err := jwt.NewVerifierPool(keys)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
jwtToken.verifierPool.Store(vp)
|
||||
}
|
||||
if jwtToken.OIDC != nil {
|
||||
if len(jwtToken.PublicKeys) > 0 || len(jwtToken.PublicKeyFiles) > 0 || jwtToken.SkipVerify {
|
||||
return nil, nil, fmt.Errorf("jwt with oidc cannot contain public keys or have skip_verify=true")
|
||||
}
|
||||
|
||||
if jwtToken.OIDC.Issuer == "" {
|
||||
return nil, nil, fmt.Errorf("oidc issuer cannot be empty")
|
||||
}
|
||||
isserURL, err := url.Parse(jwtToken.OIDC.Issuer)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("oidc issuer %q must be a valid URL", jwtToken.OIDC.Issuer)
|
||||
}
|
||||
if isserURL.Scheme != "https" && isserURL.Scheme != "http" {
|
||||
return nil, nil, fmt.Errorf("oidc issuer %q must have http or https scheme", jwtToken.OIDC.Issuer)
|
||||
}
|
||||
|
||||
oidcDP.createOrAdd(ui.JWT.OIDC.Issuer, &ui.JWT.verifierPool)
|
||||
}
|
||||
|
||||
if err := parseJWTPlaceholdersForUserInfo(&ui, true); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if err := ui.initURLs(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
metricLabels, err := ui.getMetricLabels()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot parse metric_labels: %w", err)
|
||||
}
|
||||
ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
|
||||
ui.requestErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_errors_total` + metricLabels)
|
||||
ui.backendRequests = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_requests_total` + metricLabels)
|
||||
ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
|
||||
ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
|
||||
mcr := ui.getMaxConcurrentRequests()
|
||||
ui.concurrencyLimitCh = make(chan struct{}, mcr)
|
||||
ui.concurrencyLimitReached = ac.ms.GetOrCreateCounter(`vmauth_user_concurrent_requests_limit_reached_total` + metricLabels)
|
||||
_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_capacity`+metricLabels, func() float64 {
|
||||
return float64(cap(ui.concurrencyLimitCh))
|
||||
})
|
||||
_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
|
||||
return float64(len(ui.concurrencyLimitCh))
|
||||
})
|
||||
|
||||
rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot initialize HTTP RoundTripper: %w", err)
|
||||
}
|
||||
ui.rt = rt
|
||||
|
||||
jui = append(jui, &ui)
|
||||
}
|
||||
|
||||
// sort by amount of matching claims
|
||||
// it allows to more specific claim win in case of clash
|
||||
sort.SliceStable(jui, func(i, j int) bool {
|
||||
return len(jui[i].JWT.MatchClaims) > len(jui[j].JWT.MatchClaims)
|
||||
})
|
||||
|
||||
return jui, oidcDP, nil
|
||||
}
|
||||
|
||||
var tokenPool sync.Pool
|
||||
|
||||
func getToken() *jwt.Token {
|
||||
tkn := tokenPool.Get()
|
||||
if tkn == nil {
|
||||
return &jwt.Token{}
|
||||
}
|
||||
return tkn.(*jwt.Token)
|
||||
}
|
||||
|
||||
func putToken(tkn *jwt.Token) {
|
||||
tkn.Reset()
|
||||
tokenPool.Put(tkn)
|
||||
}
|
||||
|
||||
func getJWTUserInfo(ats []string) (*UserInfo, *jwt.Token) {
|
||||
js := *jwtAuthCache.Load()
|
||||
if len(js.users) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
tkn := getToken()
|
||||
|
||||
for _, at := range ats {
|
||||
if strings.Count(at, ".") != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
at, _ = strings.CutPrefix(at, `http_auth:`)
|
||||
tkn.Reset()
|
||||
if err := tkn.Parse(at, true); err != nil {
|
||||
if *logInvalidAuthTokens {
|
||||
logger.Infof("cannot parse jwt token: %s", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if tkn.IsExpired(time.Now()) {
|
||||
if *logInvalidAuthTokens {
|
||||
// TODO: add more context:
|
||||
// token claims with issuer
|
||||
logger.Infof("jwt token is expired")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if ui := getUserInfoByJWTToken(tkn, js.users); ui != nil {
|
||||
return ui, tkn
|
||||
}
|
||||
}
|
||||
|
||||
putToken(tkn)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func getUserInfoByJWTToken(tkn *jwt.Token, users []*UserInfo) *UserInfo {
|
||||
for _, ui := range users {
|
||||
if !tkn.MatchClaims(ui.JWT.parsedMatchClaims) {
|
||||
continue
|
||||
}
|
||||
|
||||
if ui.JWT.SkipVerify {
|
||||
return ui
|
||||
}
|
||||
|
||||
if ui.JWT.OIDC != nil {
|
||||
// OIDC requires iss claim.
|
||||
// It must match the discovery issuer URL set in OIDC config.
|
||||
// https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata
|
||||
if tkn.Issuer() == "" {
|
||||
if *logInvalidAuthTokens {
|
||||
logger.Infof("jwt token must have issuer filed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if tkn.Issuer() != ui.JWT.OIDC.Issuer {
|
||||
if *logInvalidAuthTokens {
|
||||
logger.Infof("jwt token issuer: %q does not match oidc issuer: %q", tkn.Issuer(), ui.JWT.OIDC.Issuer)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
vp := ui.JWT.verifierPool.Load()
|
||||
if vp == nil {
|
||||
if *logInvalidAuthTokens {
|
||||
logger.Infof("jwt verifier not initialed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := vp.Verify(tkn); err != nil {
|
||||
if *logInvalidAuthTokens {
|
||||
logger.Infof("cannot verify jwt token: %s", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return ui
|
||||
}
|
||||
|
||||
if *logInvalidAuthTokens {
|
||||
logger.Infof("no user match jwt token")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func replaceJWTPlaceholders(bu *backendURL, hc HeadersConf, vma *jwt.VMAccessClaim) (*url.URL, HeadersConf) {
|
||||
if !bu.hasPlaceHolders && !hc.hasAnyPlaceHolders {
|
||||
return bu.url, hc
|
||||
}
|
||||
targetURL := bu.url
|
||||
data := jwtClaimsData(vma)
|
||||
if bu.hasPlaceHolders {
|
||||
// template url params and request path
|
||||
// make a copy of url
|
||||
uCopy := *bu.url
|
||||
for _, uph := range urlPathPlaceHolders {
|
||||
replacement := data[uph]
|
||||
uCopy.Path = strings.ReplaceAll(uCopy.Path, uph, replacement[0])
|
||||
}
|
||||
query := uCopy.Query()
|
||||
var foundAnyQueryPlaceholder bool
|
||||
var templatedValues []string
|
||||
for param, values := range query {
|
||||
templatedValues = templatedValues[:0]
|
||||
// filter in-place values with placeholders
|
||||
// and accumulate replacements
|
||||
// it will change the order of param values
|
||||
// but it's not guaranteed
|
||||
// and will be changed in any way with multiple arg templates
|
||||
var cnt int
|
||||
for _, value := range values {
|
||||
if dv, ok := data[value]; ok {
|
||||
foundAnyQueryPlaceholder = true
|
||||
templatedValues = append(templatedValues, dv...)
|
||||
continue
|
||||
}
|
||||
values[cnt] = value
|
||||
cnt++
|
||||
}
|
||||
values = values[:cnt]
|
||||
values = append(values, templatedValues...)
|
||||
query[param] = values
|
||||
}
|
||||
if foundAnyQueryPlaceholder {
|
||||
uCopy.RawQuery = query.Encode()
|
||||
}
|
||||
targetURL = &uCopy
|
||||
}
|
||||
if hc.hasAnyPlaceHolders {
|
||||
// make a copy of headers and update only values with placeholder
|
||||
rhs := make([]*Header, 0, len(hc.RequestHeaders))
|
||||
for _, rh := range hc.RequestHeaders {
|
||||
if dv, ok := data[rh.Value]; ok {
|
||||
rh := &Header{
|
||||
Name: rh.Name,
|
||||
Value: strings.Join(dv, ","),
|
||||
}
|
||||
rhs = append(rhs, rh)
|
||||
continue
|
||||
}
|
||||
rhs = append(rhs, rh)
|
||||
}
|
||||
hc.RequestHeaders = rhs
|
||||
}
|
||||
|
||||
return targetURL, hc
|
||||
}
|
||||
|
||||
func jwtClaimsData(vma *jwt.VMAccessClaim) map[string][]string {
|
||||
data := map[string][]string{
|
||||
// TODO: optimize at parsing stage
|
||||
metricsTenantPlaceholder: {fmt.Sprintf("%d:%d", vma.MetricsAccountID, vma.MetricsProjectID)},
|
||||
metricsExtraLabelsPlaceholder: vma.MetricsExtraLabels,
|
||||
metricsExtraFiltersPlaceholder: vma.MetricsExtraFilters,
|
||||
|
||||
// TODO: optimize at parsing stage
|
||||
logsAccountIDPlaceholder: {fmt.Sprintf("%d", vma.LogsAccountID)},
|
||||
logsProjectIDPlaceholder: {fmt.Sprintf("%d", vma.LogsProjectID)},
|
||||
logsExtraFiltersPlaceholder: vma.LogsExtraFilters,
|
||||
logsExtraStreamFiltersPlaceholder: vma.LogsExtraStreamFilters,
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func parseJWTPlaceholdersForUserInfo(ui *UserInfo, isAllowed bool) error {
|
||||
if ui.URLPrefix != nil {
|
||||
if err := validateJWTPlaceholdersForURL(ui.URLPrefix, isAllowed); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := parsePlaceholdersForHC(&ui.HeadersConf, isAllowed); err != nil {
|
||||
return err
|
||||
}
|
||||
if ui.DefaultURL != nil {
|
||||
if err := validateJWTPlaceholdersForURL(ui.DefaultURL, isAllowed); err != nil {
|
||||
return fmt.Errorf("invalid `default_url` placeholders: %w", err)
|
||||
}
|
||||
}
|
||||
for i := range ui.URLMaps {
|
||||
e := &ui.URLMaps[i]
|
||||
if e.URLPrefix != nil {
|
||||
if err := validateJWTPlaceholdersForURL(e.URLPrefix, isAllowed); err != nil {
|
||||
return fmt.Errorf("invalid `url_map` `url_prefix` placeholders: %w", err)
|
||||
}
|
||||
}
|
||||
if err := parsePlaceholdersForHC(&e.HeadersConf, isAllowed); err != nil {
|
||||
return fmt.Errorf("invalid `url_map` headers placeholders: %w", err)
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateJWTPlaceholdersForURL(up *URLPrefix, isAllowed bool) error {
|
||||
for _, bu := range up.busOriginal {
|
||||
ok := strings.Contains(bu.Path, placeholderPrefix)
|
||||
if ok && !isAllowed {
|
||||
return fmt.Errorf("placeholder: %q is only allowed at JWT token context", bu.Path)
|
||||
}
|
||||
if ok {
|
||||
p := bu.Path
|
||||
for _, ph := range allPlaceholders {
|
||||
p = strings.ReplaceAll(p, ph, ``)
|
||||
}
|
||||
if strings.Contains(p, placeholderPrefix) {
|
||||
return fmt.Errorf("invalid placeholder found in URL request path: %q, supported values are: %s", bu.Path, strings.Join(allPlaceholders, ", "))
|
||||
|
||||
}
|
||||
}
|
||||
for param, values := range bu.Query() {
|
||||
for _, value := range values {
|
||||
ok := strings.Contains(value, placeholderPrefix)
|
||||
if ok && !isAllowed {
|
||||
return fmt.Errorf("query param: %q with placeholder: %q is only allowed at JWT token context", param, value)
|
||||
}
|
||||
if ok {
|
||||
// possible placeholder
|
||||
if !slices.Contains(allPlaceholders, value) {
|
||||
return fmt.Errorf("query param: %q has unsupported placeholder string: %q, supported values are: %s", param, value, strings.Join(allPlaceholders, ", "))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parsePlaceholdersForHC(hc *HeadersConf, isAllowed bool) error {
|
||||
for _, rhs := range hc.RequestHeaders {
|
||||
ok := strings.Contains(rhs.Value, placeholderPrefix)
|
||||
if ok && !isAllowed {
|
||||
return fmt.Errorf("request header: %q placeholder: %q is only supported at JWT context", rhs.Name, rhs.Value)
|
||||
}
|
||||
if ok {
|
||||
if !slices.Contains(allPlaceholders, rhs.Value) {
|
||||
return fmt.Errorf("request header: %q has unsupported placeholder: %q, supported values are: %s", rhs.Name, rhs.Value, strings.Join(allPlaceholders, ", "))
|
||||
}
|
||||
hc.hasAnyPlaceHolders = true
|
||||
}
|
||||
}
|
||||
for _, rhs := range hc.ResponseHeaders {
|
||||
if strings.Contains(rhs.Value, placeholderPrefix) {
|
||||
return fmt.Errorf("response header placeholders are not supported; found placeholder prefix at header: %q with value: %q", rhs.Name, rhs.Value)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasAnyPlaceholders(u *url.URL) bool {
|
||||
if strings.Contains(u.Path, placeholderPrefix) {
|
||||
return true
|
||||
}
|
||||
if len(u.Query()) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, values := range u.Query() {
|
||||
for _, value := range values {
|
||||
if strings.HasPrefix(value, placeholderPrefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return false
|
||||
}
|
||||
503
app/vmauth/jwt_test.go
Normal file
503
app/vmauth/jwt_test.go
Normal file
@@ -0,0 +1,503 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestJWTParseAuthConfigFailure(t *testing.T) {
|
||||
validRSAPublicKey := `-----BEGIN PUBLIC KEY-----
|
||||
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAiX7oPWKOWRQsGFEWvwZO
|
||||
mL2PYsdYUsu9nr0qtPCjxQHUJgLfT3rdKlvKpPFYv7ZmKnqTncg36Wz9uiYmWJ7e
|
||||
IB5Z+fko8kVIMzarCqVvpAJDzYF/pUii68xvuYoK3L9TIOAeyCXv+prwnr2IH+Mw
|
||||
9AONzWbRrYoO74XyTE9vMU5qmI/L1VPk+PR8lqPOSptLvzsfoaIk2ED4yK2nRB+6
|
||||
st+k4nccPqbErqHc8aiXnXfugfnr6b+NPFYUzKsDqkymGOokVijrI8B3jNw6c6Do
|
||||
zphk+D3wgLsXYHfMcZbXIMqffqm/aB8Qg88OpFOkQ3rd2p6R9+hacnZkfkn3Phiw
|
||||
yQIDAQAB
|
||||
-----END PUBLIC KEY-----
|
||||
`
|
||||
// ECDSA with the P-521 curve
|
||||
validECDSAPublicKey := `-----BEGIN PUBLIC KEY-----
|
||||
MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQAU9RmtkCRuYTKCyvLlDn5DtBZOHSe
|
||||
QTa5j9q/oQVpCKqcXVFrH5dgh0GL+P/ZhkeuowPzCZqntGf0+7wPt9OxSJcADVJm
|
||||
dv92m540MXss8zdHf5qtE0gsu2Ved0R7Z8a8QwGZ/1mYZ+kFGGbdQTlSvRqDySTq
|
||||
XOtclIk1uhc03oL9nOQ=
|
||||
-----END PUBLIC KEY-----
|
||||
`
|
||||
|
||||
f := func(s string, expErr string) {
|
||||
t.Helper()
|
||||
ac, err := parseAuthConfig([]byte(s))
|
||||
if err != nil {
|
||||
if expErr != err.Error() {
|
||||
t.Fatalf("unexpected error; got\n%q\nwant\n%q", err.Error(), expErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
users, oidcDP, err := parseJWTUsers(ac)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error; got %v", users)
|
||||
}
|
||||
if expErr != err.Error() {
|
||||
t.Fatalf("unexpected error; got\n%q\nwant \n%q", err.Error(), expErr)
|
||||
}
|
||||
if oidcDP != nil {
|
||||
t.Fatalf("expecting nil oidcDP; got %v", oidcDP)
|
||||
}
|
||||
}
|
||||
|
||||
// unauthorized_user cannot be used with jwt
|
||||
f(`
|
||||
unauthorized_user:
|
||||
jwt: {skip_verify: true}
|
||||
url_prefix: http://foo.bar
|
||||
`, `field jwt can't be specified for unauthorized_user section`)
|
||||
|
||||
// username and jwt in a single config
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
jwt: {skip_verify: true}
|
||||
url_prefix: http://foo.bar
|
||||
`, `auth_token, bearer_token, username and password cannot be specified if jwt is set`)
|
||||
// bearer_token and jwt in a single config
|
||||
f(`
|
||||
users:
|
||||
- bearer_token: foo
|
||||
jwt: {skip_verify: true}
|
||||
url_prefix: http://foo.bar
|
||||
`, `auth_token, bearer_token, username and password cannot be specified if jwt is set`)
|
||||
// bearer_token and jwt in a single config
|
||||
f(`
|
||||
users:
|
||||
- auth_token: "Foo token"
|
||||
jwt: {skip_verify: true}
|
||||
url_prefix: http://foo.bar
|
||||
`, `auth_token, bearer_token, username and password cannot be specified if jwt is set`)
|
||||
|
||||
// jwt public_keys or skip_verify must be set, part 1
|
||||
f(`
|
||||
users:
|
||||
- jwt: {}
|
||||
url_prefix: http://foo.bar
|
||||
`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
|
||||
|
||||
// jwt public_keys or skip_verify must be set, part 2
|
||||
f(`
|
||||
users:
|
||||
- jwt: {public_keys: null}
|
||||
url_prefix: http://foo.bar
|
||||
`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
|
||||
|
||||
// jwt public_keys or skip_verify must be set, part 3
|
||||
f(`
|
||||
users:
|
||||
- jwt: {public_keys: []}
|
||||
url_prefix: http://foo.bar
|
||||
`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
|
||||
|
||||
// jwt public_keys, public_key_files or skip_verify must be set
|
||||
f(`
|
||||
users:
|
||||
- jwt: {public_key_files: []}
|
||||
url_prefix: http://foo.bar
|
||||
`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
|
||||
|
||||
// invalid public key, part 1
|
||||
f(`
|
||||
users:
|
||||
- jwt: {public_keys: [""]}
|
||||
url_prefix: http://foo.bar
|
||||
`, `failed to parse key "": failed to decode PEM block containing public key`)
|
||||
|
||||
// invalid public key, part 2
|
||||
f(`
|
||||
users:
|
||||
- jwt: {public_keys: ["invalid"]}
|
||||
url_prefix: http://foo.bar
|
||||
`, `failed to parse key "invalid": failed to decode PEM block containing public key`)
|
||||
|
||||
// invalid public key, part 2
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
- %q
|
||||
- "invalid"
|
||||
url_prefix: http://foo.bar
|
||||
`, validRSAPublicKey, validECDSAPublicKey), `failed to parse key "invalid": failed to decode PEM block containing public key`)
|
||||
|
||||
// several jwt users
|
||||
// invalid public key, part 2
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, validRSAPublicKey, validECDSAPublicKey), `duplicate match claims="" found for name="" at idx=1; the previous one is set for name=""`)
|
||||
|
||||
// public key file doesn't exist
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
public_key_files:
|
||||
- /path/to/nonexistent/file.pem
|
||||
url_prefix: http://foo.bar
|
||||
`, "cannot read public key from file \"/path/to/nonexistent/file.pem\": open /path/to/nonexistent/file.pem: no such file or directory")
|
||||
|
||||
// public key file invalid
|
||||
// auth with key from file
|
||||
publicKeyFile := filepath.Join(t.TempDir(), "a_public_key.pem")
|
||||
if err := os.WriteFile(publicKeyFile, []byte(`invalidPEM`), 0o644); err != nil {
|
||||
t.Fatalf("failed to write public key file: %s", err)
|
||||
}
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
public_key_files:
|
||||
- `+publicKeyFile+`
|
||||
url_prefix: http://foo.bar
|
||||
`, "cannot parse public key from file \""+publicKeyFile+"\": failed to parse key \"invalidPEM\": failed to decode PEM block containing public key")
|
||||
|
||||
// unsupported placeholder in a header
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
skip_verify: true
|
||||
url_prefix: http://foo.bar/{{.UnsupportedPlaceholder}}/foo`,
|
||||
"invalid placeholder found in URL request path: \"/{{.UnsupportedPlaceholder}}/foo\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
|
||||
)
|
||||
// unsupported placeholder in a header
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
skip_verify: true
|
||||
headers:
|
||||
- "AccountID: {{.UnsupportedPlaceholder}}"
|
||||
url_prefix: http://foo.bar
|
||||
`,
|
||||
"request header: \"AccountID\" has unsupported placeholder: \"{{.UnsupportedPlaceholder}}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
|
||||
)
|
||||
|
||||
// spaces in templating not allowed
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
skip_verify: true
|
||||
headers:
|
||||
- "AccountID: {{ .LogsAccountID }}"
|
||||
url_prefix: http://foo.bar
|
||||
`,
|
||||
"request header: \"AccountID\" has unsupported placeholder: \"{{ .LogsAccountID }}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
|
||||
)
|
||||
|
||||
// oidc is not an object
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
oidc: "not an object"
|
||||
url_prefix: http://foo.bar
|
||||
`,
|
||||
"cannot unmarshal AuthConfig data: yaml: unmarshal errors:\n line 4: cannot unmarshal !!str `not an ...` into main.oidcConfig",
|
||||
)
|
||||
|
||||
// oidc issuer empty
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
oidc: {}
|
||||
url_prefix: http://foo.bar
|
||||
`,
|
||||
"oidc issuer cannot be empty",
|
||||
)
|
||||
|
||||
// oidc issuer invalid urls
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
oidc:
|
||||
issuer: "::invalid-url"
|
||||
url_prefix: http://foo.bar
|
||||
`,
|
||||
"oidc issuer \"::invalid-url\" must be a valid URL",
|
||||
)
|
||||
|
||||
// oidc issuer invalid urls
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
oidc:
|
||||
issuer: "invalid-url"
|
||||
url_prefix: http://foo.bar
|
||||
`,
|
||||
"oidc issuer \"invalid-url\" must have http or https scheme",
|
||||
)
|
||||
|
||||
// oidc and public_keys are not allowed
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
oidc:
|
||||
issuer: https://example.com
|
||||
url_prefix: http://foo.bar
|
||||
`, validRSAPublicKey),
|
||||
"jwt with oidc cannot contain public keys or have skip_verify=true",
|
||||
)
|
||||
|
||||
// oidc and skip_verify are not allowed
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
skip_verify: true
|
||||
oidc:
|
||||
issuer: https://example.com
|
||||
url_prefix: http://foo.bar
|
||||
`,
|
||||
"jwt with oidc cannot contain public keys or have skip_verify=true",
|
||||
)
|
||||
// duplicate claims
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
skip_verify: true
|
||||
match_claims:
|
||||
team: ops
|
||||
name: user-1
|
||||
url_prefix: http://foo.bar
|
||||
- jwt:
|
||||
skip_verify: true
|
||||
match_claims:
|
||||
team: ops
|
||||
name: user-2
|
||||
url_prefix: http://foo.bar`,
|
||||
"duplicate match claims=\"team=ops\" found for name=\"user-2\" at idx=1; the previous one is set for name=\"user-1\"",
|
||||
)
|
||||
}
|
||||
|
||||
func TestJWTParseAuthConfigSuccess(t *testing.T) {
|
||||
validRSAPublicKey := `-----BEGIN PUBLIC KEY-----
|
||||
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAiX7oPWKOWRQsGFEWvwZO
|
||||
mL2PYsdYUsu9nr0qtPCjxQHUJgLfT3rdKlvKpPFYv7ZmKnqTncg36Wz9uiYmWJ7e
|
||||
IB5Z+fko8kVIMzarCqVvpAJDzYF/pUii68xvuYoK3L9TIOAeyCXv+prwnr2IH+Mw
|
||||
9AONzWbRrYoO74XyTE9vMU5qmI/L1VPk+PR8lqPOSptLvzsfoaIk2ED4yK2nRB+6
|
||||
st+k4nccPqbErqHc8aiXnXfugfnr6b+NPFYUzKsDqkymGOokVijrI8B3jNw6c6Do
|
||||
zphk+D3wgLsXYHfMcZbXIMqffqm/aB8Qg88OpFOkQ3rd2p6R9+hacnZkfkn3Phiw
|
||||
yQIDAQAB
|
||||
-----END PUBLIC KEY-----
|
||||
`
|
||||
// ECDSA with the P-521 curve
|
||||
validECDSAPublicKey := `-----BEGIN PUBLIC KEY-----
|
||||
MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQAU9RmtkCRuYTKCyvLlDn5DtBZOHSe
|
||||
QTa5j9q/oQVpCKqcXVFrH5dgh0GL+P/ZhkeuowPzCZqntGf0+7wPt9OxSJcADVJm
|
||||
dv92m540MXss8zdHf5qtE0gsu2Ved0R7Z8a8QwGZ/1mYZ+kFGGbdQTlSvRqDySTq
|
||||
XOtclIk1uhc03oL9nOQ=
|
||||
-----END PUBLIC KEY-----
|
||||
`
|
||||
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
ac, err := parseAuthConfig([]byte(s))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
jui, oidcDP, err := parseJWTUsers(ac)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
oidcDP.startDiscovery()
|
||||
defer oidcDP.stopDiscovery()
|
||||
|
||||
for _, ui := range jui {
|
||||
if ui.JWT == nil {
|
||||
t.Fatalf("unexpected nil JWTConfig")
|
||||
}
|
||||
|
||||
if ui.JWT.SkipVerify {
|
||||
if ui.JWT.verifierPool.Load() != nil {
|
||||
t.Fatalf("unexpected non-nil verifier pool for skip_verify=true")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if ui.JWT.verifierPool.Load() == nil {
|
||||
t.Fatalf("unexpected nil verifier pool for non-empty public keys")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, validRSAPublicKey))
|
||||
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, validECDSAPublicKey))
|
||||
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, validRSAPublicKey, validECDSAPublicKey))
|
||||
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
skip_verify: true
|
||||
url_prefix: http://foo.bar
|
||||
`)
|
||||
|
||||
// combined with other auth methods
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
password: bar
|
||||
url_prefix: http://foo.bar
|
||||
|
||||
- jwt:
|
||||
skip_verify: true
|
||||
url_prefix: http://foo.bar
|
||||
|
||||
- bearer_token: foo
|
||||
url_prefix: http://foo.bar
|
||||
`)
|
||||
|
||||
rsaKeyFile := filepath.Join(t.TempDir(), "rsa_public_key.pem")
|
||||
if err := os.WriteFile(rsaKeyFile, []byte(validRSAPublicKey), 0o644); err != nil {
|
||||
t.Fatalf("failed to write RSA key file: %s", err)
|
||||
}
|
||||
ecdsaKeyFile := filepath.Join(t.TempDir(), "ecdsa_public_key.pem")
|
||||
if err := os.WriteFile(ecdsaKeyFile, []byte(validECDSAPublicKey), 0o644); err != nil {
|
||||
t.Fatalf("failed to write ECDSA key file: %s", err)
|
||||
}
|
||||
|
||||
// Test single public key file
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_key_files:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, rsaKeyFile))
|
||||
|
||||
// Test multiple public key files
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_key_files:
|
||||
- %q
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, rsaKeyFile, ecdsaKeyFile))
|
||||
|
||||
// Test combined inline keys and files
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
public_key_files:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, validECDSAPublicKey, rsaKeyFile))
|
||||
|
||||
// oidc stub server
|
||||
var ipSrv *httptest.Server
|
||||
ipSrv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/.well-known/openid-configuration" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"issuer": ipSrv.URL,
|
||||
"jwks_uri": fmt.Sprintf("%s/jwks", ipSrv.URL),
|
||||
})
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/jwks" {
|
||||
// resp generated by https://jwkset.com/generate
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`
|
||||
{
|
||||
"keys": [
|
||||
{
|
||||
"kty": "RSA",
|
||||
"kid": "f13eee91-f566-4829-80fa-fca847c21f0e",
|
||||
"d": "Ua1llEFz3LZ05CrK5a2JxKMUEWJGXhBPPF20hHQjzxd1w0IEJK_mhPZQG8dNtBROBNIi1FC9l6QRw-RTnVIVat5Xy4yDFNKXXL3ZLXejOHY8SXrNEIDqQ-cSwIpK9cK7Umib0PcPeEeeAED5mqDH75D8_YssWFF18kLbNB5Z9pZmn6Fshiht7l2Sh4GN-KcReOW6eiQQwckDte3OGmZCRbtEriLWJt5TUGUvfZVIlcclqNMycNB6jGa9E1pO5Up7Ki3ZbI_-6XmRgZPtqnR9oLJ1zn3fj3hYpCXo-zcqLuOu3qxcslsq5igsfBzgGtfIJHY9LfWmHUsaDEa5cAX1gQ",
|
||||
"n": "xbLXXBTNREk70UCMiqZ53_mTzYh89W-UaPU61GZ-RZ5lYcLgyWOb5mdyRbvJpcgfZpsOeGAUWbk3GkQ4vqn8kUMnnWhUum2Qk9kGubOJGLW6yaURd00j3E-ilQ5xO2R_Hzz8bAojxV8GKdGTQ-iTf8z8nsSHH8kR2SERbNJCFFtwtFU7vyFWyoH4Lmvu2UpICTHFCR9RqwQVjyoKB1JjJ6Dh1L4zPTlsvQEnqoeFQHPYr0QcQSMYXdfPvlt_FiLOAOE89fX_9T2r9WbFAoda3uTRE5_aal0jxUU2cFyeVSIgauNtF07fp422XFb4XPkWQWrdNx0KX53laSIYQ9HOpw",
|
||||
"e": "AQAB",
|
||||
"p": "2JT57AD-Q2lamgjgyn0wL7DgYZ3OoCTTrDm5_NHg6h13uDvyIlXSukuUeWm4tzPSDedpstbS7dgXkLw5eQXBHwPYtByTcEZS8Z37CBnhMOOhfo_U1aNIPPanJACvWBgz47-TxHsxW1YhztZqghRoicBZPSSBAj49MgANJ4jF0zc",
|
||||
"q": "6a4MkeSXJI-ZzQ-bgP8hwJqpLFr0AiNGQcjZMH4Nn4CPGdnGiqqe6flhfLimgbNhbb67B0-8fLIji8zGhGKDL_JSIpAAdmfs2vzeEsY2hScrqVbd1VbfRcRh0J6lsn7obxkbvQthp9sX2DQbeDcEeaFEvd9gDKQSATYEqWo7eBE",
|
||||
"dp": "haL2yu6Z9RJuuxi7S3YPY33qFZF_y0St71j3L854zzw7gMxMTW9TRWwZQwk-1pv9AmNFzvnK0MNDVyUs-UXZsb932TrApshdqYRnPsppLvdl0GgDVYcYrbUr0IUzrFHSwraVAOlavRbaaXvX4EejcUvkRFvf1nh83fs2Iqy8E-U",
|
||||
"dq": "Cnf5qC-Ndd3ZDg688LJ9WJuVKJ-Kfu4Fn7zXvgxnn9Wqk4XmFyA9rk21yFidXQIkQz5gMpun3g48-W5bFmMzbVp1w4af_q35NnZNnJm0p5Jxqkxx87TIm9-IYkg5NB3rW87MJ1PzNAnkr5LmCCSu1qQa6Eaxjt9qzxMUcmKH94E",
|
||||
"qi": "saAeU11iaKHmye3cwCAYkegcyWbXV3xIXEVJtS9Af_yM19UhspwY2VhuwRaajcwYZwtvR9_ITmX9M-ea7uLdd7aDYO1fujC8NGbopeC4Hkr7yb5vTly3pfKf4h-3LwGGUucJUetdz1lmMIYiyuG4_gSf1yIEtPDLKzXiedgEMdI"
|
||||
}
|
||||
]
|
||||
}
|
||||
`))
|
||||
return
|
||||
}
|
||||
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer ipSrv.Close()
|
||||
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
oidc:
|
||||
issuer: ` + ipSrv.URL + `
|
||||
url_prefix: http://foo.bar
|
||||
`)
|
||||
// multiple match claims
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
match_claims:
|
||||
role: ro
|
||||
team: dev
|
||||
public_keys:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
- jwt:
|
||||
match_claims:
|
||||
role: admin
|
||||
team: dev
|
||||
public_key_files:
|
||||
- %q
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
- jwt:
|
||||
match_claims:
|
||||
role: viewer
|
||||
team: dev
|
||||
department: ceo
|
||||
skip_verify: true
|
||||
url_prefix: http://foo.bar
|
||||
|
||||
|
||||
`, validRSAPublicKey, rsaKeyFile, ecdsaKeyFile))
|
||||
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/jwt"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
|
||||
@@ -47,7 +48,7 @@ var (
|
||||
responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
|
||||
|
||||
requestBufferSize = flagutil.NewBytes("requestBufferSize", 32*1024, "The size of the buffer for reading the request body before proxying the request to backends. "+
|
||||
"This allows reducing the comsumption of backend resources when processing requests from clients connected via slow networks. "+
|
||||
"This allows reducing the consumption of backend resources when processing requests from clients connected via slow networks. "+
|
||||
"Set to 0 to disable request buffering. See https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering")
|
||||
maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size to buffer in memory for potential retries at other backends. "+
|
||||
"Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request body buffering and retries. "+
|
||||
@@ -173,7 +174,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
// Process requests for unauthorized users
|
||||
ui := authConfig.Load().UnauthorizedUser
|
||||
if ui != nil {
|
||||
processUserRequest(w, r, ui)
|
||||
processUserRequest(w, r, ui, nil)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -181,29 +182,36 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
ui := getUserInfoByAuthTokens(ats)
|
||||
if ui == nil {
|
||||
uu := authConfig.Load().UnauthorizedUser
|
||||
if uu != nil {
|
||||
processUserRequest(w, r, uu)
|
||||
return true
|
||||
}
|
||||
|
||||
invalidAuthTokenRequests.Inc()
|
||||
if *logInvalidAuthTokens {
|
||||
err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
|
||||
err = &httpserver.ErrorWithStatusCode{
|
||||
Err: err,
|
||||
StatusCode: http.StatusUnauthorized,
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
} else {
|
||||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||||
if ui := getUserInfoByAuthTokens(ats); ui != nil {
|
||||
processUserRequest(w, r, ui, nil)
|
||||
return true
|
||||
}
|
||||
if ui, tkn := getJWTUserInfo(ats); ui != nil {
|
||||
if tkn == nil {
|
||||
logger.Panicf("BUG: unexpected nil jwt token for user %q", ui.name())
|
||||
}
|
||||
defer putToken(tkn)
|
||||
processUserRequest(w, r, ui, tkn)
|
||||
return true
|
||||
}
|
||||
|
||||
processUserRequest(w, r, ui)
|
||||
uu := authConfig.Load().UnauthorizedUser
|
||||
if uu != nil {
|
||||
processUserRequest(w, r, uu, nil)
|
||||
return true
|
||||
}
|
||||
|
||||
invalidAuthTokenRequests.Inc()
|
||||
if *logInvalidAuthTokens {
|
||||
err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
|
||||
err = &httpserver.ErrorWithStatusCode{
|
||||
Err: err,
|
||||
StatusCode: http.StatusUnauthorized,
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
} else {
|
||||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -218,7 +226,37 @@ func getUserInfoByAuthTokens(ats []string) *UserInfo {
|
||||
return nil
|
||||
}
|
||||
|
||||
func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
// responseWriterWithStatus is a wrapper around http.ResponseWriter that captures the status code written to the response.
|
||||
type responseWriterWithStatus struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
}
|
||||
|
||||
// WriteHeader records the status so it can be easily retrieved later
|
||||
func (rws *responseWriterWithStatus) WriteHeader(status int) {
|
||||
rws.status = status
|
||||
rws.ResponseWriter.WriteHeader(status)
|
||||
}
|
||||
|
||||
// Flush implements net/http.Flusher interface
|
||||
//
|
||||
// This is needed for the copyStreamToClient()
|
||||
func (rws *responseWriterWithStatus) Flush() {
|
||||
flusher, ok := rws.ResponseWriter.(http.Flusher)
|
||||
if !ok {
|
||||
logger.Panicf("BUG: it is expected http.ResponseWriter (%T) supports http.Flusher interface", rws.ResponseWriter)
|
||||
}
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
// Unwrap returns the original ResponseWriter wrapped by rws.
|
||||
//
|
||||
// This is needed for the net/http.ResponseController - see https://pkg.go.dev/net/http#NewResponseController
|
||||
func (rws *responseWriterWithStatus) Unwrap() http.ResponseWriter {
|
||||
return rws.ResponseWriter
|
||||
}
|
||||
|
||||
func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token) {
|
||||
startTime := time.Now()
|
||||
defer ui.requestsDuration.UpdateDuration(startTime)
|
||||
|
||||
@@ -227,6 +265,20 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), *maxQueueDuration)
|
||||
defer cancel()
|
||||
|
||||
userName := ui.name()
|
||||
if userName == "" {
|
||||
userName = "unauthorized"
|
||||
}
|
||||
|
||||
if ui.AccessLog != nil {
|
||||
w = &responseWriterWithStatus{ResponseWriter: w}
|
||||
defer func() {
|
||||
rws := w.(*responseWriterWithStatus)
|
||||
duration := time.Since(startTime)
|
||||
ui.logRequest(r, userName, rws.status, duration)
|
||||
}()
|
||||
}
|
||||
|
||||
// Acquire global concurrency limit.
|
||||
if err := beginConcurrencyLimit(ctx); err != nil {
|
||||
handleConcurrencyLimitError(w, r, err)
|
||||
@@ -245,10 +297,6 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
}
|
||||
|
||||
// Read the initial chunk for the request body.
|
||||
userName := ui.name()
|
||||
if userName == "" {
|
||||
userName = "unauthorized"
|
||||
}
|
||||
bb, err := bufferRequestBody(ctx, r.Body, userName)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
@@ -269,7 +317,7 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
defer ui.endConcurrencyLimit()
|
||||
|
||||
// Process the request.
|
||||
processRequest(w, r, ui)
|
||||
processRequest(w, r, ui, tkn)
|
||||
}
|
||||
|
||||
func beginConcurrencyLimit(ctx context.Context) error {
|
||||
@@ -342,7 +390,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
|
||||
return bb, nil
|
||||
}
|
||||
|
||||
func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token) {
|
||||
u := normalizeURL(r.URL)
|
||||
up, hc := ui.getURLPrefixAndHeaders(u, r.Host, r.Header)
|
||||
isDefault := false
|
||||
@@ -368,22 +416,27 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
}
|
||||
|
||||
maxAttempts := up.getBackendsCount()
|
||||
for i := 0; i < maxAttempts; i++ {
|
||||
for range maxAttempts {
|
||||
bu := up.getBackendURL()
|
||||
if bu == nil {
|
||||
break
|
||||
}
|
||||
targetURL := bu.url
|
||||
if tkn != nil {
|
||||
// for security reasons allow templating only for configured url values and headers
|
||||
targetURL, hc = replaceJWTPlaceholders(bu, hc, tkn.VMAccess())
|
||||
}
|
||||
if isDefault {
|
||||
// Don't change path and add request_path query param for default route.
|
||||
targetURLCopy := *targetURL
|
||||
query := targetURL.Query()
|
||||
query.Set("request_path", u.String())
|
||||
targetURL.RawQuery = query.Encode()
|
||||
targetURLCopy.RawQuery = query.Encode()
|
||||
targetURL = &targetURLCopy
|
||||
} else {
|
||||
// Update path for regular routes.
|
||||
targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts, up.mergeQueryArgs)
|
||||
}
|
||||
|
||||
wasLocalRetry := false
|
||||
again:
|
||||
ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui, bu)
|
||||
@@ -401,7 +454,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
ui.backendErrors.Inc()
|
||||
}
|
||||
err := &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
|
||||
Err: fmt.Errorf("all the %d backends for the user %q are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend", up.getBackendsCount(), ui.name()),
|
||||
StatusCode: http.StatusBadGateway,
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
194
app/vmauth/main_timing_test.go
Normal file
194
app/vmauth/main_timing_test.go
Normal file
@@ -0,0 +1,194 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func BenchmarkJWTRequestHandler(b *testing.B) {
|
||||
// Generate RSA key pair for testing
|
||||
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
b.Fatalf("cannot generate RSA key: %s", err)
|
||||
}
|
||||
|
||||
// Generate public key PEM
|
||||
publicKeyBytes, err := x509.MarshalPKIXPublicKey(&privateKey.PublicKey)
|
||||
if err != nil {
|
||||
b.Fatalf("cannot marshal public key: %s", err)
|
||||
}
|
||||
publicKeyPEM := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "PUBLIC KEY",
|
||||
Bytes: publicKeyBytes,
|
||||
})
|
||||
|
||||
genToken := func(t *testing.B, body map[string]any, valid bool) string {
|
||||
t.Helper()
|
||||
|
||||
headerJSON, err := json.Marshal(map[string]any{
|
||||
"alg": "RS256",
|
||||
"typ": "JWT",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("cannot marshal header: %s", err)
|
||||
}
|
||||
headerB64 := base64.RawURLEncoding.EncodeToString(headerJSON)
|
||||
|
||||
bodyJSON, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot marshal body: %s", err)
|
||||
}
|
||||
bodyB64 := base64.RawURLEncoding.EncodeToString(bodyJSON)
|
||||
|
||||
payload := headerB64 + "." + bodyB64
|
||||
|
||||
var signatureB64 string
|
||||
if valid {
|
||||
// Create real RSA signature
|
||||
hash := crypto.SHA256
|
||||
h := hash.New()
|
||||
h.Write([]byte(payload))
|
||||
digest := h.Sum(nil)
|
||||
|
||||
signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, hash, digest)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot sign token: %s", err)
|
||||
}
|
||||
signatureB64 = base64.RawURLEncoding.EncodeToString(signature)
|
||||
} else {
|
||||
signatureB64 = base64.RawURLEncoding.EncodeToString([]byte("invalid_signature"))
|
||||
}
|
||||
|
||||
return payload + "." + signatureB64
|
||||
}
|
||||
|
||||
f := func(name string, cfgStr string, r *http.Request, statusCodeExpected int) {
|
||||
b.Helper()
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := w.Write([]byte("path: " + r.URL.Path + "\n")); err != nil {
|
||||
panic(fmt.Errorf("cannot write response: %w", err))
|
||||
}
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
cfgStr = strings.ReplaceAll(cfgStr, "{BACKEND}", ts.URL)
|
||||
|
||||
cfgOrigP := authConfigData.Load()
|
||||
if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
|
||||
b.Fatalf("cannot load config data: %s", err)
|
||||
}
|
||||
defer func() {
|
||||
cfgOrig := []byte("unauthorized_user:\n url_prefix: http://foo/bar")
|
||||
if cfgOrigP != nil {
|
||||
cfgOrig = *cfgOrigP
|
||||
}
|
||||
_, err := reloadAuthConfigData(cfgOrig)
|
||||
if err != nil {
|
||||
b.Fatalf("cannot load the original config: %s", err)
|
||||
}
|
||||
}()
|
||||
|
||||
b.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
w := &fakeResponseWriter{}
|
||||
for pb.Next() {
|
||||
w.reset()
|
||||
if !requestHandlerWithInternalRoutes(w, r) {
|
||||
b.Fatalf("unexpected false is returned from requestHandler")
|
||||
}
|
||||
if w.statusCode != statusCodeExpected {
|
||||
b.Fatalf("unexpected response code (-%d;+%d)", statusCodeExpected, w.statusCode)
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
simpleCfgStr := fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
url_prefix: {BACKEND}/foo`, string(publicKeyPEM))
|
||||
noVMAccessClaimToken := genToken(b, nil, true)
|
||||
expiredToken := genToken(b, map[string]any{
|
||||
"exp": 10,
|
||||
"vm_access": map[string]any{},
|
||||
}, true)
|
||||
|
||||
fullToken := genToken(b, map[string]any{
|
||||
"exp": time.Now().Add(10 * time.Minute).Unix(),
|
||||
"scope": "email id",
|
||||
"vm_access": map[string]any{
|
||||
"extra_labels": map[string]string{
|
||||
"label": "value1",
|
||||
"label2": "value3",
|
||||
},
|
||||
"extra_filters": []string{"stream_filter1", "stream_filter2"},
|
||||
"metrics_account_id": 123,
|
||||
"metrics_project_id": 234,
|
||||
"metrics_extra_labels": []string{
|
||||
"label1=value1",
|
||||
"label2=value2",
|
||||
},
|
||||
"metrics_extra_filters": []string{
|
||||
`{label3="value3"}`,
|
||||
`{label4="value4"}`,
|
||||
},
|
||||
"logs_account_id": 345,
|
||||
"logs_project_id": 456,
|
||||
"logs_extra_filters": []string{
|
||||
`{"namespace":"my-app","env":"prod"}`,
|
||||
},
|
||||
"logs_extra_stream_filters": []string{
|
||||
`{"team":"dev"}`,
|
||||
},
|
||||
},
|
||||
}, true)
|
||||
|
||||
// tenant headers are overwritten if set as placeholders
|
||||
// extra_filters extra_stream_filters from vm_access claim merged with statically defined
|
||||
request := httptest.NewRequest(`GET`, "http://some-host.com/query", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+fullToken)
|
||||
f("full_template",
|
||||
fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
headers:
|
||||
- "AccountID: {{.LogsAccountID}}"
|
||||
- "ProjectID: {{.LogsProjectID}}"
|
||||
url_prefix: {BACKEND}/select/logsql/?extra_filters=aStaticFilter&extra_stream_filters=aStaticStreamFilter&extra_filters={{.LogsExtraFilters}}&extra_stream_filters={{.LogsExtraStreamFilters}}`, string(publicKeyPEM)),
|
||||
request,
|
||||
http.StatusOK,
|
||||
)
|
||||
|
||||
// token without vm_access claim
|
||||
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+noVMAccessClaimToken)
|
||||
f("token_without_claim", simpleCfgStr, request, http.StatusUnauthorized)
|
||||
|
||||
// expired token
|
||||
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+expiredToken)
|
||||
f("expired_token", simpleCfgStr, request, http.StatusUnauthorized)
|
||||
}
|
||||
195
app/vmauth/oidc.go
Normal file
195
app/vmauth/oidc.go
Normal file
@@ -0,0 +1,195 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/jwt"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
)
|
||||
|
||||
type oidcConfig struct {
|
||||
Issuer string `yaml:"issuer"`
|
||||
}
|
||||
|
||||
type oidcDiscovererPool struct {
|
||||
ds map[string]*oidcDiscoverer
|
||||
|
||||
context context.Context
|
||||
cancel func()
|
||||
wg *sync.WaitGroup
|
||||
}
|
||||
|
||||
func (dp *oidcDiscovererPool) createOrAdd(issuer string, vp *atomic.Pointer[jwt.VerifierPool]) {
|
||||
if dp.ds == nil {
|
||||
dp.ds = make(map[string]*oidcDiscoverer)
|
||||
dp.context, dp.cancel = context.WithCancel(context.Background())
|
||||
dp.wg = &sync.WaitGroup{}
|
||||
}
|
||||
|
||||
ds, found := dp.ds[issuer]
|
||||
if !found {
|
||||
ds = &oidcDiscoverer{
|
||||
issuer: issuer,
|
||||
}
|
||||
dp.ds[issuer] = ds
|
||||
}
|
||||
|
||||
ds.vps = append(ds.vps, vp)
|
||||
}
|
||||
|
||||
func (dp *oidcDiscovererPool) startDiscovery() {
|
||||
if len(dp.ds) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for _, d := range dp.ds {
|
||||
dp.wg.Go(func() {
|
||||
if err := d.refreshVerifierPools(dp.context); err != nil {
|
||||
logger.Errorf("failed to initialize OIDC verifier pool at start for issuer %q: %s", d.issuer, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
dp.wg.Wait()
|
||||
|
||||
for _, d := range dp.ds {
|
||||
dp.wg.Go(func() {
|
||||
d.run(dp.context)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (dp *oidcDiscovererPool) stopDiscovery() {
|
||||
if len(dp.ds) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
dp.cancel()
|
||||
dp.wg.Wait()
|
||||
}
|
||||
|
||||
type oidcDiscoverer struct {
|
||||
issuer string
|
||||
vps []*atomic.Pointer[jwt.VerifierPool]
|
||||
}
|
||||
|
||||
func (d *oidcDiscoverer) run(ctx context.Context) {
|
||||
t := time.NewTimer(timeutil.AddJitterToDuration(time.Second * 10))
|
||||
defer t.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
if err := d.refreshVerifierPools(ctx); errors.Is(err, context.Canceled) {
|
||||
return
|
||||
} else if err != nil {
|
||||
t.Reset(timeutil.AddJitterToDuration(time.Second * 10))
|
||||
logger.Errorf("failed to refresh OIDC verifier pool for issuer %q: %v", d.issuer, err)
|
||||
continue
|
||||
}
|
||||
// OIDC may return Cache-Control header with max-age directive.
|
||||
// It could be used as time range for next refresh.
|
||||
// https://openid.net/specs/openid-connect-core-1_0.html#RotateEncKeys
|
||||
t.Reset(timeutil.AddJitterToDuration(time.Minute * 5))
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *oidcDiscoverer) refreshVerifierPools(ctx context.Context) error {
|
||||
cfg, err := getOpenIDConfiguration(ctx, d.issuer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The issuer in the OIDC configuration must match the expected issuer.
|
||||
// https://openid.net/specs/openid-connect-core-1_0.html#RotateEncKeys
|
||||
if cfg.Issuer != d.issuer {
|
||||
return fmt.Errorf("openid configuration issuer %q does not match expected issuer %q", cfg.Issuer, d.issuer)
|
||||
}
|
||||
|
||||
verifierPool, err := fetchAndParseJWKs(ctx, cfg.JWKsURI)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, vp := range d.vps {
|
||||
vp.Store(verifierPool)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// See https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata for details.
|
||||
type openidConfig struct {
|
||||
Issuer string `json:"issuer"`
|
||||
JWKsURI string `json:"jwks_uri"`
|
||||
}
|
||||
|
||||
var oidcHTTPClient = &http.Client{
|
||||
Timeout: time.Second * 5,
|
||||
}
|
||||
|
||||
func fetchAndParseJWKs(ctx context.Context, jwksURI string) (*jwt.VerifierPool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, jwksURI, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request for fetching jwks keys from %q: %w", jwksURI, err)
|
||||
}
|
||||
|
||||
resp, err := oidcHTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch jwks keys from %q: %w", jwksURI, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("unexpected status code %d when fetching jwks keys from %q", resp.StatusCode, jwksURI)
|
||||
}
|
||||
|
||||
b, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body from %q: %w", jwksURI, err)
|
||||
}
|
||||
|
||||
vp, err := jwt.ParseJWKs(b)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse jwks keys from %q: %v", jwksURI, err)
|
||||
}
|
||||
|
||||
return vp, nil
|
||||
}
|
||||
|
||||
func getOpenIDConfiguration(ctx context.Context, issuer string) (openidConfig, error) {
|
||||
issuer, _ = strings.CutSuffix(issuer, "/")
|
||||
configURL := fmt.Sprintf("%s/.well-known/openid-configuration", issuer)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, configURL, nil)
|
||||
if err != nil {
|
||||
return openidConfig{}, fmt.Errorf("failed to create request for fetching openid config from %q: %w", configURL, err)
|
||||
}
|
||||
|
||||
resp, err := oidcHTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return openidConfig{}, fmt.Errorf("failed to fetch openid config from %q: %w", configURL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return openidConfig{}, fmt.Errorf("unexpected status code %d when fetching openid config from %q", resp.StatusCode, configURL)
|
||||
}
|
||||
|
||||
var cfg openidConfig
|
||||
if err := json.NewDecoder(resp.Body).Decode(&cfg); err != nil {
|
||||
return openidConfig{}, fmt.Errorf("failed to decode openid config from %q: %s", configURL, err)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
@@ -174,7 +174,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
|
||||
},
|
||||
RetryStatusCodes: []int{503, 501},
|
||||
LoadBalancingPolicy: "first_available",
|
||||
DropSrcPathPrefixParts: intp(2),
|
||||
DropSrcPathPrefixParts: new(2),
|
||||
}, "/a/b/c", "http://foo.bar/c", `bb: aaa`, `x: y`, []int{503, 501}, "first_available", 2)
|
||||
f(&UserInfo{
|
||||
URLPrefix: mustParseURL("http://foo.bar/federate"),
|
||||
@@ -219,13 +219,13 @@ func TestCreateTargetURLSuccess(t *testing.T) {
|
||||
},
|
||||
RetryStatusCodes: []int{503, 500, 501},
|
||||
LoadBalancingPolicy: "first_available",
|
||||
DropSrcPathPrefixParts: intp(1),
|
||||
DropSrcPathPrefixParts: new(1),
|
||||
},
|
||||
{
|
||||
SrcPaths: getRegexs([]string{"/api/v1/write"}),
|
||||
URLPrefix: mustParseURL("http://vminsert/0/prometheus"),
|
||||
RetryStatusCodes: []int{},
|
||||
DropSrcPathPrefixParts: intp(0),
|
||||
DropSrcPathPrefixParts: new(0),
|
||||
},
|
||||
{
|
||||
SrcPaths: getRegexs([]string{"/metrics"}),
|
||||
@@ -242,7 +242,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
|
||||
},
|
||||
},
|
||||
RetryStatusCodes: []int{502},
|
||||
DropSrcPathPrefixParts: intp(2),
|
||||
DropSrcPathPrefixParts: new(2),
|
||||
}
|
||||
f(ui, "http://host42/vmsingle/api/v1/query?query=up&db=foo", "http://vmselect/0/prometheus/api/v1/query?db=foo&query=up",
|
||||
"xx: aa\nyy: asdf", "qwe: rty", []int{503, 500, 501}, "first_available", 1)
|
||||
@@ -259,7 +259,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
|
||||
SrcPaths: getRegexs([]string{"/api/v1/write"}),
|
||||
URLPrefix: mustParseURL("http://vminsert/0/prometheus"),
|
||||
RetryStatusCodes: []int{},
|
||||
DropSrcPathPrefixParts: intp(0),
|
||||
DropSrcPathPrefixParts: new(0),
|
||||
},
|
||||
{
|
||||
SrcPaths: getRegexs([]string{"/metrics/a/b"}),
|
||||
@@ -275,7 +275,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
|
||||
},
|
||||
},
|
||||
RetryStatusCodes: []int{502},
|
||||
DropSrcPathPrefixParts: intp(2),
|
||||
DropSrcPathPrefixParts: new(2),
|
||||
}
|
||||
f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", []int{}, "least_loaded", 0)
|
||||
f(ui, "https://foo-host/metrics/a/b", "http://metrics-server/b", "", "", []int{502}, "least_loaded", 2)
|
||||
|
||||
@@ -47,7 +47,7 @@ func New(retries int, factor float64, minDuration time.Duration) (*Backoff, erro
|
||||
// Retry process retries until all attempts are completed
|
||||
func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
|
||||
var attempt uint64
|
||||
for i := 0; i < b.retries; i++ {
|
||||
for i := range b.retries {
|
||||
err := cb()
|
||||
if err == nil {
|
||||
return attempt, nil
|
||||
|
||||
@@ -76,11 +76,11 @@ func (ts *TimeSeries) write(w io.Writer) (int, error) {
|
||||
|
||||
pointsCount := len(timestampsBatch)
|
||||
cw.printf(`},"timestamps":[`)
|
||||
for i := 0; i < pointsCount-1; i++ {
|
||||
for i := range pointsCount - 1 {
|
||||
cw.printf(`%d,`, timestampsBatch[i])
|
||||
}
|
||||
cw.printf(`%d],"values":[`, timestampsBatch[pointsCount-1])
|
||||
for i := 0; i < pointsCount-1; i++ {
|
||||
for i := range pointsCount - 1 {
|
||||
cw.printf(`%v,`, valuesBatch[i])
|
||||
}
|
||||
cw.printf("%v]}\n", valuesBatch[pointsCount-1])
|
||||
|
||||
@@ -262,7 +262,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
|
||||
errCh := make(chan error, p.cc)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < p.cc; i++ {
|
||||
for range p.cc {
|
||||
wg.Go(func() {
|
||||
for f := range filterCh {
|
||||
if !p.disablePerMetricRequests {
|
||||
|
||||
@@ -55,7 +55,7 @@ var (
|
||||
deduplicator *streamaggr.Deduplicator
|
||||
)
|
||||
|
||||
// CheckStreamAggrConfig checks config pointed by -stramaggr.config
|
||||
// CheckStreamAggrConfig checks config pointed by -streamaggr.config
|
||||
func CheckStreamAggrConfig() error {
|
||||
if *streamAggrConfig == "" {
|
||||
return nil
|
||||
|
||||
@@ -45,15 +45,14 @@ func insertRows(sketches []*datadogsketches.Sketch, extraLabels []prompb.Label)
|
||||
ms := sketch.ToSummary()
|
||||
for _, m := range ms {
|
||||
ctx.Labels = ctx.Labels[:0]
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557
|
||||
ctx.AddLabel("host", sketch.Host) // newly added
|
||||
ctx.AddLabel("", m.Name)
|
||||
for _, label := range m.Labels {
|
||||
ctx.AddLabel(label.Name, label.Value)
|
||||
}
|
||||
for _, tag := range sketch.Tags {
|
||||
name, value := datadogutil.SplitTag(tag)
|
||||
if name == "host" {
|
||||
name = "exported_host"
|
||||
}
|
||||
ctx.AddLabel(name, value)
|
||||
}
|
||||
for j := range extraLabels {
|
||||
|
||||
@@ -77,7 +77,7 @@ func push(ctx *common.InsertCtx, tss []prompb.TimeSeries) {
|
||||
r := &ts.Samples[i]
|
||||
metricNameRaw, err = ctx.WriteDataPointExt(metricNameRaw, ctx.Labels, r.Timestamp, r.Value)
|
||||
if err != nil {
|
||||
logger.Errorf("cannot write promscape data to storage: %s", err)
|
||||
logger.Errorf("cannot write promscrape data to storage: %s", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,7 +142,7 @@ type aggrStatePercentile struct {
|
||||
|
||||
func newAggrStatePercentile(pointsLen int, n float64) aggrState {
|
||||
hs := make([]*histogram.Fast, pointsLen)
|
||||
for i := 0; i < pointsLen; i++ {
|
||||
for i := range pointsLen {
|
||||
hs[i] = histogram.NewFast()
|
||||
}
|
||||
return &aggrStatePercentile{
|
||||
|
||||
@@ -50,7 +50,7 @@ func (ec *evalConfig) newTimestamps(step int64) []int64 {
|
||||
pointsLen := ec.pointsLen(step)
|
||||
timestamps := make([]int64, pointsLen)
|
||||
ts := ec.startTime
|
||||
for i := 0; i < pointsLen; i++ {
|
||||
for i := range pointsLen {
|
||||
timestamps[i] = ts
|
||||
ts += step
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ func naturalLess(a, b string) bool {
|
||||
}
|
||||
|
||||
func getNonNumPrefix(s string) (prefix string, tail string) {
|
||||
for i := 0; i < len(s); i++ {
|
||||
for i := range len(s) {
|
||||
ch := s[i]
|
||||
if ch >= '0' && ch <= '9' {
|
||||
return s[:i], s[i:]
|
||||
|
||||
@@ -209,7 +209,7 @@ func parseInterval(s string) (int64, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
prefix := s
|
||||
var suffix string
|
||||
for i := 0; i < len(s); i++ {
|
||||
for i := range len(s) {
|
||||
ch := s[i]
|
||||
if ch != '-' && ch != '+' && ch != '.' && (ch < '0' || ch > '9') {
|
||||
prefix = s[:i]
|
||||
|
||||
@@ -1228,7 +1228,7 @@ func transformDelay(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, er
|
||||
stepsLocal = len(values)
|
||||
}
|
||||
copy(values[stepsLocal:], values[:len(values)-stepsLocal])
|
||||
for i := 0; i < stepsLocal; i++ {
|
||||
for i := range stepsLocal {
|
||||
values[i] = nan
|
||||
}
|
||||
}
|
||||
@@ -1740,7 +1740,7 @@ func transformGroup(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, er
|
||||
|
||||
func groupSeriesLists(ec *evalConfig, args []*graphiteql.ArgExpr, expr graphiteql.Expr) (nextSeriesFunc, error) {
|
||||
var nextSeriess []nextSeriesFunc
|
||||
for i := 0; i < len(args); i++ {
|
||||
for i := range args {
|
||||
nextSeries, err := evalSeriesList(ec, args, "seriesList", i)
|
||||
if err != nil {
|
||||
for _, f := range nextSeriess {
|
||||
@@ -3233,7 +3233,7 @@ func transformSeriesByTag(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFu
|
||||
return nil, fmt.Errorf("at least one tagExpression must be passed to seriesByTag")
|
||||
}
|
||||
var tagExpressions []string
|
||||
for i := 0; i < len(args); i++ {
|
||||
for i := range args {
|
||||
te, err := getString(args, "tagExpressions", i)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -3633,7 +3633,7 @@ var graphiteToGolangRe = regexp.MustCompile(`\\(\d+)`)
|
||||
|
||||
func getNodes(args []*graphiteql.ArgExpr) ([]graphiteql.Expr, error) {
|
||||
var nodes []graphiteql.Expr
|
||||
for i := 0; i < len(args); i++ {
|
||||
for i := range args {
|
||||
expr := args[i].Expr
|
||||
switch expr.(type) {
|
||||
case *graphiteql.NumberExpr, *graphiteql.StringExpr:
|
||||
@@ -4052,7 +4052,7 @@ func formatPathsFromSeriesExpressions(seriesExpressions []string, sortPaths bool
|
||||
|
||||
func newNaNSeries(ec *evalConfig, step int64) *series {
|
||||
values := make([]float64, ec.pointsLen(step))
|
||||
for i := 0; i < len(values); i++ {
|
||||
for i := range values {
|
||||
values[i] = nan
|
||||
}
|
||||
return &series{
|
||||
@@ -5244,7 +5244,7 @@ func transformLinearRegression(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSer
|
||||
|
||||
func linearRegressionForSeries(ec *evalConfig, fe *graphiteql.FuncExpr, ss, sourceSeries []*series) (nextSeriesFunc, error) {
|
||||
var resp []*series
|
||||
for i := 0; i < len(ss); i++ {
|
||||
for i := range ss {
|
||||
source := sourceSeries[i]
|
||||
s := ss[i]
|
||||
s.Tags["linearRegressions"] = fmt.Sprintf("%d, %d", ec.startTime/1e3, ec.endTime/1e3)
|
||||
@@ -5258,7 +5258,7 @@ func linearRegressionForSeries(ec *evalConfig, fe *graphiteql.FuncExpr, ss, sour
|
||||
continue
|
||||
}
|
||||
values := s.Values
|
||||
for j := 0; j < len(values); j++ {
|
||||
for j := range values {
|
||||
values[j] = offset + (float64(int(s.Timestamps[0])+j*int(s.step)))*factor
|
||||
}
|
||||
resp = append(resp, s)
|
||||
@@ -5370,7 +5370,7 @@ func holtWinterConfidenceBands(ec *evalConfig, fe *graphiteql.FuncExpr, args []*
|
||||
valuesLen := len(forecastValues)
|
||||
upperBand := make([]float64, 0, valuesLen)
|
||||
lowerBand := make([]float64, 0, valuesLen)
|
||||
for i := 0; i < valuesLen; i++ {
|
||||
for i := range valuesLen {
|
||||
forecastItem := forecastValues[i]
|
||||
deviationItem := deviationValues[i]
|
||||
if math.IsNaN(forecastItem) || math.IsNaN(deviationItem) {
|
||||
@@ -5464,7 +5464,7 @@ func transformHoltWintersAberration(ec *evalConfig, fe *graphiteql.FuncExpr) (ne
|
||||
return nil, fmt.Errorf("bug, len mismatch for series: %d and upperBand values: %d or lowerBand values: %d", len(values), len(upperBand), len(lowerBand))
|
||||
}
|
||||
aberration := make([]float64, 0, len(values))
|
||||
for i := 0; i < len(values); i++ {
|
||||
for i := range values {
|
||||
v := values[i]
|
||||
upperValue := upperBand[i]
|
||||
lowerValue := lowerBand[i]
|
||||
|
||||
@@ -280,7 +280,7 @@ func isMetricExprChar(ch byte) bool {
|
||||
}
|
||||
|
||||
func appendEscapedIdent(dst []byte, s string) []byte {
|
||||
for i := 0; i < len(s); i++ {
|
||||
for i := range len(s) {
|
||||
ch := s[i]
|
||||
if isIdentChar(ch) || isMetricExprChar(ch) {
|
||||
if i == 0 && !isFirstIdentChar(ch) {
|
||||
|
||||
@@ -321,19 +321,23 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
return true
|
||||
case "/tags/tagSeries":
|
||||
graphiteTagsTagSeriesRequests.Inc()
|
||||
if err := graphite.TagsTagSeriesHandler(startTime, w, r); err != nil {
|
||||
graphiteTagsTagSeriesErrors.Inc()
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
err := &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("graphite tag registration has been disabled and is planned to be removed in future. " +
|
||||
"See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544"),
|
||||
StatusCode: http.StatusNotImplemented,
|
||||
}
|
||||
graphiteTagsTagSeriesErrors.Inc()
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
case "/tags/tagMultiSeries":
|
||||
graphiteTagsTagMultiSeriesRequests.Inc()
|
||||
if err := graphite.TagsTagMultiSeriesHandler(startTime, w, r); err != nil {
|
||||
graphiteTagsTagMultiSeriesErrors.Inc()
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
err := &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("graphite tag registration has been disabled and is planned to be removed in future. " +
|
||||
"See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544"),
|
||||
StatusCode: http.StatusNotImplemented,
|
||||
}
|
||||
graphiteTagsTagMultiSeriesErrors.Inc()
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
case "/tags":
|
||||
graphiteTagsRequests.Inc()
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -491,10 +492,7 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
|
||||
}
|
||||
|
||||
// Prepare worker channels.
|
||||
workers := min(len(upws), gomaxprocs)
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
workers := max(min(len(upws), gomaxprocs), 1)
|
||||
itemsPerWorker := (len(upws) + workers - 1) / workers
|
||||
workChs := make([]chan *unpackWork, workers)
|
||||
for i := range workChs {
|
||||
@@ -832,12 +830,7 @@ func GraphiteTags(qt *querytracer.Tracer, filter string, limit int, deadline sea
|
||||
}
|
||||
|
||||
func hasString(a []string, s string) bool {
|
||||
for _, x := range a {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.Contains(a, s)
|
||||
}
|
||||
|
||||
// LabelValues returns label values matching the given labelName and sq until the given deadline.
|
||||
|
||||
@@ -10,14 +10,14 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
|
||||
b.Run(fmt.Sprintf("replicationFactor-%d", replicationFactor), func(b *testing.B) {
|
||||
const samplesPerBlock = 8192
|
||||
var blocks []*sortBlock
|
||||
for j := 0; j < 10; j++ {
|
||||
for j := range 10 {
|
||||
timestamps := make([]int64, samplesPerBlock)
|
||||
values := make([]float64, samplesPerBlock)
|
||||
for i := range timestamps {
|
||||
timestamps[i] = int64(j*samplesPerBlock + i)
|
||||
values[i] = float64(j*samplesPerBlock + i)
|
||||
}
|
||||
for i := 0; i < replicationFactor; i++ {
|
||||
for range replicationFactor {
|
||||
blocks = append(blocks, &sortBlock{
|
||||
Timestamps: timestamps,
|
||||
Values: values,
|
||||
@@ -30,7 +30,7 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
|
||||
b.Run("overlapped-blocks-bestcase", func(b *testing.B) {
|
||||
const samplesPerBlock = 8192
|
||||
var blocks []*sortBlock
|
||||
for j := 0; j < 10; j++ {
|
||||
for j := range 10 {
|
||||
timestamps := make([]int64, samplesPerBlock)
|
||||
values := make([]float64, samplesPerBlock)
|
||||
for i := range timestamps {
|
||||
@@ -45,7 +45,7 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
|
||||
for j := 1; j < len(blocks); j++ {
|
||||
prev := blocks[j-1].Timestamps
|
||||
curr := blocks[j].Timestamps
|
||||
for i := 0; i < samplesPerBlock/2; i++ {
|
||||
for i := range samplesPerBlock / 2 {
|
||||
prev[i+samplesPerBlock/2], curr[i] = curr[i], prev[i+samplesPerBlock/2]
|
||||
}
|
||||
}
|
||||
@@ -54,7 +54,7 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
|
||||
b.Run("overlapped-blocks-worstcase", func(b *testing.B) {
|
||||
const samplesPerBlock = 8192
|
||||
var blocks []*sortBlock
|
||||
for j := 0; j < 5; j++ {
|
||||
for j := range 5 {
|
||||
timestamps := make([]int64, samplesPerBlock)
|
||||
values := make([]float64, samplesPerBlock)
|
||||
for i := range timestamps {
|
||||
|
||||
@@ -6,11 +6,13 @@ import (
|
||||
"math"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/VictoriaMetrics/metricsql"
|
||||
@@ -527,6 +529,14 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s
|
||||
return err
|
||||
}
|
||||
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxLabelsAPISeries)
|
||||
|
||||
if strings.HasPrefix(labelName, "U__") {
|
||||
// This label seems to be Unicode-encoded according to the Prometheus spec.
|
||||
// See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
|
||||
// Spec: https://github.com/prometheus/proposals/blob/main/proposals/0028-utf8.md
|
||||
labelName = unescapePrometheusLabelName(labelName)
|
||||
}
|
||||
|
||||
labelValues, err := netstorage.LabelValues(qt, labelName, sq, limit, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain values for label %q: %w", labelName, err)
|
||||
@@ -1004,14 +1014,7 @@ func removeEmptyValuesAndTimeseries(tss []netstorage.Result) []netstorage.Result
|
||||
dst := tss[:0]
|
||||
for i := range tss {
|
||||
ts := &tss[i]
|
||||
hasNaNs := false
|
||||
for _, v := range ts.Values {
|
||||
if math.IsNaN(v) {
|
||||
hasNaNs = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasNaNs {
|
||||
if !slices.ContainsFunc(ts.Values, math.IsNaN) {
|
||||
// Fast path: nothing to remove.
|
||||
if len(ts.Values) > 0 {
|
||||
dst = append(dst, *ts)
|
||||
@@ -1336,3 +1339,70 @@ func calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequests, remainingMem
|
||||
func GetMaxUniqueTimeSeries() int {
|
||||
return maxUniqueTimeseriesValue
|
||||
}
|
||||
|
||||
// copied from https://github.com/prometheus/common/blob/adea6285c1c7447fcb7bfdeb6abfc6eff893e0a7/model/metric.go#L483
|
||||
// it's not possible to use direct import due to increased binary size
|
||||
func unescapePrometheusLabelName(name string) string {
|
||||
// lower function taken from strconv.atoi.
|
||||
lower := func(c byte) byte {
|
||||
return c | ('x' - 'X')
|
||||
}
|
||||
if len(name) == 0 {
|
||||
return name
|
||||
}
|
||||
escapedName, found := strings.CutPrefix(name, "U__")
|
||||
if !found {
|
||||
return name
|
||||
}
|
||||
|
||||
var unescaped strings.Builder
|
||||
TOP:
|
||||
for i := 0; i < len(escapedName); i++ {
|
||||
// All non-underscores are treated normally.
|
||||
if escapedName[i] != '_' {
|
||||
unescaped.WriteByte(escapedName[i])
|
||||
continue
|
||||
}
|
||||
i++
|
||||
if i >= len(escapedName) {
|
||||
return name
|
||||
}
|
||||
// A double underscore is a single underscore.
|
||||
if escapedName[i] == '_' {
|
||||
unescaped.WriteByte('_')
|
||||
continue
|
||||
}
|
||||
// We think we are in a UTF-8 code, process it.
|
||||
var utf8Val uint
|
||||
for j := 0; i < len(escapedName); j++ {
|
||||
// This is too many characters for a utf8 value based on the MaxRune
|
||||
// value of '\U0010FFFF'.
|
||||
if j >= 6 {
|
||||
return name
|
||||
}
|
||||
// Found a closing underscore, convert to a rune, check validity, and append.
|
||||
if escapedName[i] == '_' {
|
||||
utf8Rune := rune(utf8Val)
|
||||
if !utf8.ValidRune(utf8Rune) {
|
||||
return name
|
||||
}
|
||||
unescaped.WriteRune(utf8Rune)
|
||||
continue TOP
|
||||
}
|
||||
r := lower(escapedName[i])
|
||||
utf8Val *= 16
|
||||
switch {
|
||||
case r >= '0' && r <= '9':
|
||||
utf8Val += uint(r) - '0'
|
||||
case r >= 'a' && r <= 'f':
|
||||
utf8Val += uint(r) - 'a' + 10
|
||||
default:
|
||||
return name
|
||||
}
|
||||
i++
|
||||
}
|
||||
// Didn't find closing underscore, invalid.
|
||||
return name
|
||||
}
|
||||
return unescaped.String()
|
||||
}
|
||||
|
||||
@@ -742,7 +742,7 @@ func getRangeTopKTimeseries(tss []*timeseries, modifier *metricsql.ModifierExpr,
|
||||
|
||||
func reverseSeries(tss []*timeseries) {
|
||||
j := len(tss)
|
||||
for i := 0; i < len(tss)/2; i++ {
|
||||
for i := range len(tss) / 2 {
|
||||
j--
|
||||
tss[i], tss[j] = tss[j], tss[i]
|
||||
}
|
||||
@@ -983,7 +983,7 @@ func getPerPointIQRBounds(tss []*timeseries) ([]float64, []float64) {
|
||||
var qs []float64
|
||||
lower := make([]float64, pointsLen)
|
||||
upper := make([]float64, pointsLen)
|
||||
for i := 0; i < pointsLen; i++ {
|
||||
for i := range pointsLen {
|
||||
values = values[:0]
|
||||
for _, ts := range tss {
|
||||
v := ts.Values[i]
|
||||
|
||||
@@ -53,7 +53,7 @@ func TestIncrementalAggr(t *testing.T) {
|
||||
Values: valuesExpected,
|
||||
}}
|
||||
// run the test multiple times to make sure there are no side effects on concurrency
|
||||
for i := 0; i < 10; i++ {
|
||||
for i := range 10 {
|
||||
iafc := newIncrementalAggrFuncContext(ae, callbacks)
|
||||
tssSrcCopy := copyTimeseries(tssSrc)
|
||||
if err := testIncrementalParallelAggr(iafc, tssSrcCopy, tssExpected); err != nil {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -1165,6 +1166,61 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
|
||||
},
|
||||
}
|
||||
return evalExpr(qt, ec, be)
|
||||
// the cached rate result could be inaccurate in edge cases, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098
|
||||
case "rate":
|
||||
if iafc != nil {
|
||||
if !strings.EqualFold(iafc.ae.Name, "sum") {
|
||||
qt.Printf("do not apply instant rollup optimization for incremental aggregate %s()", iafc.ae.Name)
|
||||
return evalAt(qt, timestamp, window)
|
||||
}
|
||||
qt.Printf("optimized calculation for sum(rate(m[d])) as (sum(increase(m[d])) / d)")
|
||||
afe := expr.(*metricsql.AggrFuncExpr)
|
||||
fe := afe.Args[0].(*metricsql.FuncExpr)
|
||||
feIncrease := *fe
|
||||
feIncrease.Name = "increase"
|
||||
// copy RollupExpr to drop possible offset,
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
|
||||
newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
|
||||
newArg.Offset = nil
|
||||
feIncrease.Args = []metricsql.Expr{newArg}
|
||||
d := newArg.Window.Duration(ec.Step)
|
||||
if d == 0 {
|
||||
d = ec.Step
|
||||
}
|
||||
afeIncrease := *afe
|
||||
afeIncrease.Args = []metricsql.Expr{&feIncrease}
|
||||
be := &metricsql.BinaryOpExpr{
|
||||
Op: "/",
|
||||
KeepMetricNames: true,
|
||||
Left: &afeIncrease,
|
||||
Right: &metricsql.NumberExpr{
|
||||
N: float64(d) / 1000,
|
||||
},
|
||||
}
|
||||
return evalExpr(qt, ec, be)
|
||||
}
|
||||
qt.Printf("optimized calculation for instant rollup rate(m[d]) as (increase(m[d]) / d)")
|
||||
fe := expr.(*metricsql.FuncExpr)
|
||||
feIncrease := *fe
|
||||
feIncrease.Name = "increase"
|
||||
// copy RollupExpr to drop possible offset,
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
|
||||
newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
|
||||
newArg.Offset = nil
|
||||
feIncrease.Args = []metricsql.Expr{newArg}
|
||||
d := newArg.Window.Duration(ec.Step)
|
||||
if d == 0 {
|
||||
d = ec.Step
|
||||
}
|
||||
be := &metricsql.BinaryOpExpr{
|
||||
Op: "/",
|
||||
KeepMetricNames: fe.KeepMetricNames,
|
||||
Left: &feIncrease,
|
||||
Right: &metricsql.NumberExpr{
|
||||
N: float64(d) / 1000,
|
||||
},
|
||||
}
|
||||
return evalExpr(qt, ec, be)
|
||||
case "max_over_time":
|
||||
if iafc != nil {
|
||||
if !strings.EqualFold(iafc.ae.Name, "max") {
|
||||
@@ -1935,14 +1991,7 @@ func dropStaleNaNs(funcName string, values []float64, timestamps []int64) ([]flo
|
||||
return values, timestamps
|
||||
}
|
||||
// Remove Prometheus staleness marks, so non-default rollup functions don't hit NaN values.
|
||||
hasStaleSamples := false
|
||||
for _, v := range values {
|
||||
if decimal.IsStaleNaN(v) {
|
||||
hasStaleSamples = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasStaleSamples {
|
||||
if !slices.ContainsFunc(values, decimal.IsStaleNaN) {
|
||||
// Fast path: values have no Prometheus staleness marks.
|
||||
return values, timestamps
|
||||
}
|
||||
|
||||
@@ -313,7 +313,7 @@ func escapeDots(s string) string {
|
||||
return s
|
||||
}
|
||||
result := make([]byte, 0, len(s)+2*dotsCount)
|
||||
for i := 0; i < len(s); i++ {
|
||||
for i := range len(s) {
|
||||
if s[i] == '.' && (i == 0 || s[i-1] != '\\') && (i+1 == len(s) || i+1 < len(s) && s[i+1] != '*' && s[i+1] != '+' && s[i+1] != '{') {
|
||||
// Escape a dot if the following conditions are met:
|
||||
// - if it isn't escaped already, i.e. if there is no `\` char before the dot.
|
||||
|
||||
@@ -67,7 +67,7 @@ func TestExecSuccess(t *testing.T) {
|
||||
Deadline: searchutil.NewDeadline(time.Now(), time.Minute, ""),
|
||||
RoundDigits: 100,
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
for range 5 {
|
||||
result, err := Exec(nil, ec, q, false)
|
||||
if err != nil {
|
||||
t.Fatalf(`unexpected error when executing %q: %s`, q, err)
|
||||
@@ -4018,6 +4018,12 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_fraction(scalar)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_fraction(123, 456, time())`
|
||||
resultExpected := []netstorage.Result{}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_quantile(single-value-no-le)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_quantile(0.6, label_set(100, "foo", "bar"))`
|
||||
@@ -4030,6 +4036,12 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_fraction(single-value-no-le)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_fraction(123,456, label_set(100, "foo", "bar"))`
|
||||
resultExpected := []netstorage.Result{}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_quantile(single-value-invalid-le)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_quantile(0.6, label_set(100, "le", "foobar"))`
|
||||
@@ -4042,6 +4054,12 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_fraction(single-value-invalid-le)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_fraction(50, 60, label_set(100, "le", "foobar"))`
|
||||
resultExpected := []netstorage.Result{}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_quantile(single-value-inf-le)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_quantile(0.6, label_set(100, "le", "+Inf"))`
|
||||
@@ -4183,6 +4201,28 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_fraction(single-value-valid-le)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_fraction(0, 100, label_set(100, "le", "200"))`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0.5, 0.5, 0.5, 0.5, 0.5, 0.5},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_fraction(single-value-valid-le)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_fraction(200, 300, label_set(100, "le", "200"))`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0, 0, 0, 0, 0, 0},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_quantile(single-value-valid-le, boundsLabel)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `sort(histogram_quantile(0.6, label_set(100, "le", "200"), "foobar"))`
|
||||
@@ -4212,7 +4252,7 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{r1, r2, r3}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_quantile(single-value-valid-le, boundsLabel)`, func(t *testing.T) {
|
||||
t.Run(`histogram_share(single-value-valid-le, boundsLabel)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `sort(histogram_share(120, label_set(100, "le", "200"), "foobar"))`
|
||||
r1 := netstorage.Result{
|
||||
@@ -4311,7 +4351,37 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_share(single-value-valid-le-mid-le)`, func(t *testing.T) {
|
||||
t.Run(`histogram_fraction(single-value-valid-le-max-le)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_fraction(0,100, (
|
||||
label_set(100, "le", "100"),
|
||||
label_set(40, "le", "50"),
|
||||
label_set(0, "le", "10"),
|
||||
))`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{1, 1, 1, 1, 1, 1},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_fraction(single-value-valid-le-min-le)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_fraction(0,10, (
|
||||
label_set(100, "le", "100"),
|
||||
label_set(40, "le", "50"),
|
||||
label_set(0, "le", "10"),
|
||||
))`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0, 0, 0, 0, 0, 0},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_share(single-value-valid-le-mid-le-1)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_share(105, (
|
||||
label_set(100, "le", "200"),
|
||||
@@ -4325,6 +4395,34 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_share(single-value-valid-le-mid-le-2)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_share(55, (
|
||||
label_set(100, "le", "200"),
|
||||
label_set(0, "le", "55"),
|
||||
))`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0, 0, 0, 0, 0, 0},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_fraction(single-value-valid-le-mid-le)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_fraction(55,105, (
|
||||
label_set(100, "le", "200"),
|
||||
label_set(0, "le", "55"),
|
||||
))`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0.3448275862068966, 0.3448275862068966, 0.3448275862068966, 0.3448275862068966, 0.3448275862068966, 0.3448275862068966},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_quantile(single-value-valid-le-min-phi-no-zero-bucket)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_quantile(0, label_set(100, "le", "200"))`
|
||||
@@ -4358,6 +4456,17 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_fraction(scalar-phi)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_fraction(25, time() / 8, label_set(100, "le", "200"))`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0.5, 0.625, 0.75, 0.875, 0.875, 0.875},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_quantile(duplicate-le)`, func(t *testing.T) {
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3225
|
||||
t.Parallel()
|
||||
@@ -4439,6 +4548,36 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{r1, r2}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_fraction(valid)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `sort(histogram_fraction(0, 25,
|
||||
label_set(90, "foo", "bar", "le", "10")
|
||||
or label_set(100, "foo", "bar", "le", "30")
|
||||
or label_set(300, "foo", "bar", "le", "+Inf")
|
||||
or label_set(200, "tag", "xx", "le", "10")
|
||||
or label_set(300, "tag", "xx", "le", "30")
|
||||
))`
|
||||
r1 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0.325, 0.325, 0.325, 0.325, 0.325, 0.325},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r1.MetricName.Tags = []storage.Tag{{
|
||||
Key: []byte("foo"),
|
||||
Value: []byte("bar"),
|
||||
}}
|
||||
r2 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0.9166666666666666, 0.9166666666666666, 0.9166666666666666, 0.9166666666666666, 0.9166666666666666, 0.9166666666666666},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r2.MetricName.Tags = []storage.Tag{{
|
||||
Key: []byte("tag"),
|
||||
Value: []byte("xx"),
|
||||
}}
|
||||
resultExpected := []netstorage.Result{r1, r2}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_quantile(negative-bucket-count)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_quantile(0.6,
|
||||
@@ -4555,6 +4694,25 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_fraction(normal-bucket-count)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `histogram_fraction(22,35,
|
||||
label_set(0, "foo", "bar", "le", "10")
|
||||
or label_set(100, "foo", "bar", "le", "30")
|
||||
or label_set(300, "foo", "bar", "le", "+Inf")
|
||||
)`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0.1333333333333333, 0.1333333333333333, 0.1333333333333333, 0.1333333333333333, 0.1333333333333333, 0.1333333333333333},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r.MetricName.Tags = []storage.Tag{{
|
||||
Key: []byte("foo"),
|
||||
Value: []byte("bar"),
|
||||
}}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`histogram_quantile(normal-bucket-count, boundsLabel)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `sort(histogram_quantile(0.2,
|
||||
@@ -9827,7 +9985,7 @@ func TestExecError(t *testing.T) {
|
||||
Deadline: searchutil.NewDeadline(time.Now(), time.Minute, ""),
|
||||
RoundDigits: 100,
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
for range 4 {
|
||||
rv, err := Exec(nil, ec, q, false)
|
||||
if err == nil {
|
||||
t.Fatalf(`expecting non-nil error on %q`, q)
|
||||
|
||||
@@ -55,7 +55,7 @@ type parseCache struct {
|
||||
|
||||
func newParseCache() *parseCache {
|
||||
pc := new(parseCache)
|
||||
for i := 0; i < parseBucketCount; i++ {
|
||||
for i := range parseBucketCount {
|
||||
pc.buckets[i] = newParseBucket()
|
||||
}
|
||||
return pc
|
||||
@@ -75,7 +75,7 @@ func (pc *parseCache) get(q string) *parseCacheValue {
|
||||
|
||||
func (pc *parseCache) requests() uint64 {
|
||||
var n uint64
|
||||
for i := 0; i < parseBucketCount; i++ {
|
||||
for i := range parseBucketCount {
|
||||
n += pc.buckets[i].requests.Load()
|
||||
}
|
||||
return n
|
||||
@@ -83,7 +83,7 @@ func (pc *parseCache) requests() uint64 {
|
||||
|
||||
func (pc *parseCache) misses() uint64 {
|
||||
var n uint64
|
||||
for i := 0; i < parseBucketCount; i++ {
|
||||
for i := range parseBucketCount {
|
||||
n += pc.buckets[i].misses.Load()
|
||||
}
|
||||
return n
|
||||
@@ -91,7 +91,7 @@ func (pc *parseCache) misses() uint64 {
|
||||
|
||||
func (pc *parseCache) len() uint64 {
|
||||
var n uint64
|
||||
for i := 0; i < parseBucketCount; i++ {
|
||||
for i := range parseBucketCount {
|
||||
n += pc.buckets[i].len()
|
||||
}
|
||||
return n
|
||||
|
||||
@@ -17,7 +17,7 @@ func testGetParseCacheValue(q string) *parseCacheValue {
|
||||
|
||||
func testGenerateQueries(items int) []string {
|
||||
queries := make([]string, items)
|
||||
for i := 0; i < items; i++ {
|
||||
for i := range items {
|
||||
queries[i] = fmt.Sprintf(`node_time_seconds{instance="node%d", job="job%d"}`, i, i)
|
||||
}
|
||||
return queries
|
||||
@@ -102,7 +102,7 @@ func TestParseCacheBucketOverflow(t *testing.T) {
|
||||
v := testGetParseCacheValue(queries[0])
|
||||
|
||||
// Fill bucket
|
||||
for i := 0; i < parseBucketMaxLen; i++ {
|
||||
for i := range parseBucketMaxLen {
|
||||
b.put(queries[i], v)
|
||||
}
|
||||
expectedLen = uint64(parseBucketMaxLen)
|
||||
|
||||
@@ -15,7 +15,7 @@ func BenchmarkCachePutNoOverFlow(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for i := 0; i < items; i++ {
|
||||
for i := range items {
|
||||
pc.put(queries[i], v)
|
||||
}
|
||||
}
|
||||
@@ -32,14 +32,14 @@ func BenchmarkCacheGetNoOverflow(b *testing.B) {
|
||||
queries := testGenerateQueries(items)
|
||||
v := testGetParseCacheValue(queries[0])
|
||||
|
||||
for i := 0; i < len(queries); i++ {
|
||||
for i := range queries {
|
||||
pc.put(queries[i], v)
|
||||
}
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for i := 0; i < items; i++ {
|
||||
for i := range items {
|
||||
if v := pc.get(queries[i]); v == nil {
|
||||
b.Errorf("unexpected nil value obtained from cache for query: %s ", queries[i])
|
||||
}
|
||||
@@ -59,7 +59,7 @@ func BenchmarkCachePutGetNoOverflow(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for i := 0; i < items; i++ {
|
||||
for i := range items {
|
||||
pc.put(queries[i], v)
|
||||
if res := pc.get(queries[i]); res == nil {
|
||||
b.Errorf("unexpected nil value obtained from cache for query: %s ", queries[i])
|
||||
@@ -79,7 +79,7 @@ func BenchmarkCachePutOverflow(b *testing.B) {
|
||||
queries := testGenerateQueries(items)
|
||||
v := testGetParseCacheValue(queries[0])
|
||||
|
||||
for i := 0; i < parseCacheMaxLen; i++ {
|
||||
for i := range parseCacheMaxLen {
|
||||
c.put(queries[i], v)
|
||||
}
|
||||
|
||||
@@ -105,7 +105,7 @@ func BenchmarkCachePutGetOverflow(b *testing.B) {
|
||||
queries := testGenerateQueries(items)
|
||||
v := testGetParseCacheValue(queries[0])
|
||||
|
||||
for i := 0; i < parseCacheMaxLen; i++ {
|
||||
for i := range parseCacheMaxLen {
|
||||
c.put(queries[i], v)
|
||||
}
|
||||
|
||||
@@ -141,8 +141,8 @@ var testSimpleQueries = []string{
|
||||
|
||||
func BenchmarkParsePromQLWithCacheSimple(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
for j := 0; j < len(testSimpleQueries); j++ {
|
||||
for range b.N {
|
||||
for j := range testSimpleQueries {
|
||||
_, err := parsePromQLWithCache(testSimpleQueries[j])
|
||||
if err != nil {
|
||||
b.Errorf("unexpected error: %s", err)
|
||||
@@ -155,7 +155,7 @@ func BenchmarkParsePromQLWithCacheSimpleParallel(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for i := 0; i < len(testSimpleQueries); i++ {
|
||||
for i := range testSimpleQueries {
|
||||
_, err := parsePromQLWithCache(testSimpleQueries[i])
|
||||
if err != nil {
|
||||
b.Errorf("unexpected error: %s", err)
|
||||
@@ -210,8 +210,8 @@ var testComplexQueries = []string{
|
||||
|
||||
func BenchmarkParsePromQLWithCacheComplex(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
for j := 0; j < len(testComplexQueries); j++ {
|
||||
for range b.N {
|
||||
for j := range testComplexQueries {
|
||||
_, err := parsePromQLWithCache(testComplexQueries[j])
|
||||
if err != nil {
|
||||
b.Errorf("unexpected error: %s", err)
|
||||
@@ -224,7 +224,7 @@ func BenchmarkParsePromQLWithCacheComplexParallel(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for i := 0; i < len(testComplexQueries); i++ {
|
||||
for i := range testComplexQueries {
|
||||
_, err := parsePromQLWithCache(testComplexQueries[i])
|
||||
if err != nil {
|
||||
b.Errorf("unexpected error: %s", err)
|
||||
|
||||
@@ -739,7 +739,7 @@ func (mi *rollupResultCacheMetainfo) Unmarshal(src []byte) error {
|
||||
entriesLen := int(encoding.UnmarshalUint32(src))
|
||||
src = src[4:]
|
||||
mi.entries = slicesutil.SetLength(mi.entries, entriesLen)
|
||||
for i := 0; i < entriesLen; i++ {
|
||||
for i := range entriesLen {
|
||||
tail, err := mi.entries[i].Unmarshal(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot unmarshal entry #%d: %w", i, err)
|
||||
|
||||
@@ -11,14 +11,14 @@ import (
|
||||
|
||||
func TestRollupResultCacheInitStop(t *testing.T) {
|
||||
t.Run("inmemory", func(_ *testing.T) {
|
||||
for i := 0; i < 5; i++ {
|
||||
for range 5 {
|
||||
InitRollupResultCache("")
|
||||
StopRollupResultCache()
|
||||
}
|
||||
})
|
||||
t.Run("file-based", func(_ *testing.T) {
|
||||
cacheFilePath := "test-rollup-result-cache"
|
||||
for i := 0; i < 3; i++ {
|
||||
for range 3 {
|
||||
InitRollupResultCache(cacheFilePath)
|
||||
StopRollupResultCache()
|
||||
}
|
||||
@@ -241,12 +241,12 @@ func TestRollupResultCache(t *testing.T) {
|
||||
t.Run("big-timeseries", func(t *testing.T) {
|
||||
ResetRollupResultCache()
|
||||
var tss []*timeseries
|
||||
for i := 0; i < 1000; i++ {
|
||||
for i := range 1000 {
|
||||
ts := ×eries{
|
||||
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
||||
Values: []float64{1, 2, 3, 4, 5, 6},
|
||||
}
|
||||
ts.MetricName.MetricGroup = []byte(fmt.Sprintf("metric %d", i))
|
||||
ts.MetricName.MetricGroup = fmt.Appendf(nil, "metric %d", i)
|
||||
tss = append(tss, ts)
|
||||
}
|
||||
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss)
|
||||
|
||||
@@ -240,7 +240,7 @@ func testRollupFunc(t *testing.T, funcName string, args []any, vExpected float64
|
||||
if rollupFuncsRemoveCounterResets[funcName] {
|
||||
removeCounterResets(rfa.values, rfa.timestamps, 0)
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
for range 5 {
|
||||
v := rf(&rfa)
|
||||
if math.IsNaN(vExpected) {
|
||||
if !math.IsNaN(v) {
|
||||
@@ -1493,7 +1493,7 @@ func TestRollupBigNumberOfValues(t *testing.T) {
|
||||
rc.Timestamps = rc.getTimestamps()
|
||||
srcValues := make([]float64, srcValuesCount)
|
||||
srcTimestamps := make([]int64, srcValuesCount)
|
||||
for i := 0; i < srcValuesCount; i++ {
|
||||
for i := range int(srcValuesCount) {
|
||||
srcValues[i] = float64(i)
|
||||
srcTimestamps[i] = int64(i / 2)
|
||||
}
|
||||
|
||||
@@ -51,6 +51,7 @@ var transformFuncs = map[string]transformFunc{
|
||||
"exp": newTransformFuncOneArg(transformExp),
|
||||
"floor": newTransformFuncOneArg(transformFloor),
|
||||
"histogram_avg": transformHistogramAvg,
|
||||
"histogram_fraction": transformHistogramFraction,
|
||||
"histogram_quantile": transformHistogramQuantile,
|
||||
"histogram_quantiles": transformHistogramQuantiles,
|
||||
"histogram_share": transformHistogramShare,
|
||||
@@ -451,7 +452,7 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
|
||||
sort.Slice(leGroup, func(i, j int) bool {
|
||||
return leGroup[i].le < leGroup[j].le
|
||||
})
|
||||
for n := 0; n < pointsCount; n++ {
|
||||
for n := range pointsCount {
|
||||
prevValue := float64(0)
|
||||
for i := range leGroup {
|
||||
xx := &leGroup[i]
|
||||
@@ -662,13 +663,13 @@ func transformHistogramShare(tfa *transformFuncArg) ([]*timeseries, error) {
|
||||
if math.IsNaN(leReq) || len(xss) == 0 {
|
||||
return nan, nan, nan
|
||||
}
|
||||
fixBrokenBuckets(i, xss)
|
||||
if leReq < 0 {
|
||||
return 0, 0, 0
|
||||
}
|
||||
if math.IsInf(leReq, 1) {
|
||||
return 1, 1, 1
|
||||
}
|
||||
fixBrokenBuckets(i, xss)
|
||||
var vPrev, lePrev float64
|
||||
for _, xs := range xss {
|
||||
v := xs.ts.Values[i]
|
||||
@@ -729,6 +730,85 @@ func transformHistogramShare(tfa *transformFuncArg) ([]*timeseries, error) {
|
||||
return rvs, nil
|
||||
}
|
||||
|
||||
// histogram_fraction is a shortcut for `histogram_share(upperLe, buckets) - histogram_share(lowerLe, buckets)`;
|
||||
// histogram_fraction(x, y) = histogram_fraction(-Inf, y) - histogram_fraction(-Inf, x) = histogram_share(y) - histogram_share(x).
|
||||
// This function is supported by PromQL.
|
||||
func transformHistogramFraction(tfa *transformFuncArg) ([]*timeseries, error) {
|
||||
args := tfa.args
|
||||
if err := expectTransformArgsNum(args, 3); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lowerles, err := getScalar(args[0], 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse lower le: %w", err)
|
||||
}
|
||||
upperles, err := getScalar(args[1], 1)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse upper le: %w", err)
|
||||
}
|
||||
if lowerles[0] >= upperles[0] {
|
||||
return nil, fmt.Errorf("lower le cannot be greater than upper le; got lower le: %f, upper le: %f", lowerles[0], upperles[0])
|
||||
}
|
||||
|
||||
// Convert buckets with `vmrange` labels to buckets with `le` labels.
|
||||
tss := vmrangeBucketsToLE(args[2])
|
||||
|
||||
// Group metrics by all tags excluding "le"
|
||||
m := groupLeTimeseries(tss)
|
||||
|
||||
fraction := func(i int, lowerle, upperle float64, xss []leTimeseries) (q float64) {
|
||||
if math.IsNaN(lowerle) || math.IsNaN(upperle) || len(xss) == 0 {
|
||||
return nan
|
||||
}
|
||||
fixBrokenBuckets(i, xss)
|
||||
share := func(leReq float64) float64 {
|
||||
if leReq < 0 {
|
||||
return 0
|
||||
}
|
||||
if math.IsInf(leReq, 1) {
|
||||
return 1
|
||||
}
|
||||
var vPrev, lePrev float64
|
||||
for _, xs := range xss {
|
||||
v := xs.ts.Values[i]
|
||||
le := xs.le
|
||||
if leReq >= le {
|
||||
vPrev = v
|
||||
lePrev = le
|
||||
continue
|
||||
}
|
||||
// precondition: lePrev <= leReq < le
|
||||
vLast := xss[len(xss)-1].ts.Values[i]
|
||||
lower := vPrev / vLast
|
||||
if math.IsInf(le, 1) {
|
||||
return lower
|
||||
}
|
||||
if lePrev == leReq {
|
||||
return lower
|
||||
}
|
||||
q = lower + (v-vPrev)/vLast*(leReq-lePrev)/(le-lePrev)
|
||||
return q
|
||||
}
|
||||
return 1
|
||||
}
|
||||
return share(upperle) - share(lowerle)
|
||||
}
|
||||
rvs := make([]*timeseries, 0, len(m))
|
||||
for _, xss := range m {
|
||||
sort.Slice(xss, func(i, j int) bool {
|
||||
return xss[i].le < xss[j].le
|
||||
})
|
||||
xss = mergeSameLE(xss)
|
||||
dst := xss[0].ts
|
||||
for i := range dst.Values {
|
||||
q := fraction(i, lowerles[i], upperles[i], xss)
|
||||
dst.Values[i] = q
|
||||
}
|
||||
rvs = append(rvs, dst)
|
||||
}
|
||||
return rvs, nil
|
||||
}
|
||||
|
||||
func transformHistogramAvg(tfa *transformFuncArg) ([]*timeseries, error) {
|
||||
args := tfa.args
|
||||
if err := expectTransformArgsNum(args, 1); err != nil {
|
||||
@@ -1192,7 +1272,7 @@ func transformInterpolate(tfa *transformFuncArg) ([]*timeseries, error) {
|
||||
}
|
||||
prevValue := nan
|
||||
var nextValue float64
|
||||
for i := 0; i < len(values); i++ {
|
||||
for i := range values {
|
||||
if !math.IsNaN(values[i]) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1227,7 +1227,10 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
|
||||
#### buckets_limit
|
||||
|
||||
`buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
|
||||
of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`.
|
||||
of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`.
|
||||
|
||||
The result will preserve the first and the last bucket to improve accuracy for min and max values.
|
||||
So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
|
||||
|
||||
See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
|
||||
|
||||
@@ -1381,6 +1384,15 @@ It can be used for calculating the average over the given time range across mult
|
||||
For example, `histogram_avg(sum(histogram_over_time(response_time_duration_seconds[5m])) by (vmrange,job))` would return the average response time
|
||||
per each `job` over the last 5 minutes.
|
||||
|
||||
#### histogram_fraction
|
||||
|
||||
`histogram_fraction(lowerLe, upperLe, buckets)` is a [transform function](#transform-functions), which calculates the share (in the range `[0...1]`) for `buckets` that fall between `lowerLe` and `upperLe`.
|
||||
The result of `histogram_fraction(lowerLe, upperLe, buckets)` is equivalent to `histogram_share(upperLe, buckets) - histogram_share(lowerLe, buckets)`.
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [histogram_share](#histogram_share).
|
||||
|
||||
#### histogram_quantile
|
||||
|
||||
`histogram_quantile(phi, buckets)` is a [transform function](#transform-functions), which calculates `phi`-[percentile](https://en.wikipedia.org/wiki/Percentile)
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -37,10 +37,10 @@
|
||||
<meta property="og:title" content="UI for VictoriaMetrics">
|
||||
<meta property="og:url" content="https://victoriametrics.com/">
|
||||
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
|
||||
<script type="module" crossorigin src="./assets/index-C1hTBemk.js"></script>
|
||||
<script type="module" crossorigin src="./assets/index-DeVEZ1fy.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-BR6Q0Fin.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-D7CzMv1O.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-DffVfcrT.css">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
|
||||
@@ -62,7 +62,7 @@ var (
|
||||
"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . "+
|
||||
"See also -storage.maxHourlySeries")
|
||||
|
||||
minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which the storage stops accepting new data")
|
||||
minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 100e6, "The minimum free disk space at -storageDataPath after which the storage stops accepting new data")
|
||||
|
||||
cacheSizeStorageTSID = flagutil.NewBytes("storage.cacheSizeStorageTSID", 0, "Overrides max size for storage/tsid cache. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26.0 AS build-web-stage
|
||||
FROM golang:1.26.1 AS build-web-stage
|
||||
COPY build /build
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
244
app/vmui/packages/vmui/package-lock.json
generated
244
app/vmui/packages/vmui/package-lock.json
generated
@@ -1681,9 +1681,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@rollup/rollup-android-arm-eabi": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.52.5.tgz",
|
||||
"integrity": "sha512-8c1vW4ocv3UOMp9K+gToY5zL2XiiVw3k7f1ksf4yO1FlDFQ1C2u72iACFnSOceJFsWskc2WZNqeRhFRPzv+wtQ==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz",
|
||||
"integrity": "sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -1694,9 +1694,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-android-arm64": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.52.5.tgz",
|
||||
"integrity": "sha512-mQGfsIEFcu21mvqkEKKu2dYmtuSZOBMmAl5CFlPGLY94Vlcm+zWApK7F/eocsNzp8tKmbeBP8yXyAbx0XHsFNA==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.59.0.tgz",
|
||||
"integrity": "sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1707,9 +1707,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-darwin-arm64": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.52.5.tgz",
|
||||
"integrity": "sha512-takF3CR71mCAGA+v794QUZ0b6ZSrgJkArC+gUiG6LB6TQty9T0Mqh3m2ImRBOxS2IeYBo4lKWIieSvnEk2OQWA==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.59.0.tgz",
|
||||
"integrity": "sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1720,9 +1720,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-darwin-x64": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.52.5.tgz",
|
||||
"integrity": "sha512-W901Pla8Ya95WpxDn//VF9K9u2JbocwV/v75TE0YIHNTbhqUTv9w4VuQ9MaWlNOkkEfFwkdNhXgcLqPSmHy0fA==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.59.0.tgz",
|
||||
"integrity": "sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1733,9 +1733,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-freebsd-arm64": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.52.5.tgz",
|
||||
"integrity": "sha512-QofO7i7JycsYOWxe0GFqhLmF6l1TqBswJMvICnRUjqCx8b47MTo46W8AoeQwiokAx3zVryVnxtBMcGcnX12LvA==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.59.0.tgz",
|
||||
"integrity": "sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1746,9 +1746,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-freebsd-x64": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.52.5.tgz",
|
||||
"integrity": "sha512-jr21b/99ew8ujZubPo9skbrItHEIE50WdV86cdSoRkKtmWa+DDr6fu2c/xyRT0F/WazZpam6kk7IHBerSL7LDQ==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.59.0.tgz",
|
||||
"integrity": "sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1759,9 +1759,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-arm-gnueabihf": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.52.5.tgz",
|
||||
"integrity": "sha512-PsNAbcyv9CcecAUagQefwX8fQn9LQ4nZkpDboBOttmyffnInRy8R8dSg6hxxl2Re5QhHBf6FYIDhIj5v982ATQ==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.59.0.tgz",
|
||||
"integrity": "sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -1772,9 +1772,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-arm-musleabihf": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.52.5.tgz",
|
||||
"integrity": "sha512-Fw4tysRutyQc/wwkmcyoqFtJhh0u31K+Q6jYjeicsGJJ7bbEq8LwPWV/w0cnzOqR2m694/Af6hpFayLJZkG2VQ==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.59.0.tgz",
|
||||
"integrity": "sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -1785,9 +1785,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-arm64-gnu": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.52.5.tgz",
|
||||
"integrity": "sha512-a+3wVnAYdQClOTlyapKmyI6BLPAFYs0JM8HRpgYZQO02rMR09ZcV9LbQB+NL6sljzG38869YqThrRnfPMCDtZg==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.59.0.tgz",
|
||||
"integrity": "sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1798,9 +1798,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-arm64-musl": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.52.5.tgz",
|
||||
"integrity": "sha512-AvttBOMwO9Pcuuf7m9PkC1PUIKsfaAJ4AYhy944qeTJgQOqJYJ9oVl2nYgY7Rk0mkbsuOpCAYSs6wLYB2Xiw0Q==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.59.0.tgz",
|
||||
"integrity": "sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1811,9 +1811,22 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-loong64-gnu": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.52.5.tgz",
|
||||
"integrity": "sha512-DkDk8pmXQV2wVrF6oq5tONK6UHLz/XcEVow4JTTerdeV1uqPeHxwcg7aFsfnSm9L+OO8WJsWotKM2JJPMWrQtA==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.59.0.tgz",
|
||||
"integrity": "sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==",
|
||||
"cpu": [
|
||||
"loong64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-loong64-musl": {
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.59.0.tgz",
|
||||
"integrity": "sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==",
|
||||
"cpu": [
|
||||
"loong64"
|
||||
],
|
||||
@@ -1824,9 +1837,22 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-ppc64-gnu": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.52.5.tgz",
|
||||
"integrity": "sha512-W/b9ZN/U9+hPQVvlGwjzi+Wy4xdoH2I8EjaCkMvzpI7wJUs8sWJ03Rq96jRnHkSrcHTpQe8h5Tg3ZzUPGauvAw==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.59.0.tgz",
|
||||
"integrity": "sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-ppc64-musl": {
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.59.0.tgz",
|
||||
"integrity": "sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
@@ -1837,9 +1863,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-riscv64-gnu": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.52.5.tgz",
|
||||
"integrity": "sha512-sjQLr9BW7R/ZiXnQiWPkErNfLMkkWIoCz7YMn27HldKsADEKa5WYdobaa1hmN6slu9oWQbB6/jFpJ+P2IkVrmw==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.59.0.tgz",
|
||||
"integrity": "sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -1850,9 +1876,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-riscv64-musl": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.52.5.tgz",
|
||||
"integrity": "sha512-hq3jU/kGyjXWTvAh2awn8oHroCbrPm8JqM7RUpKjalIRWWXE01CQOf/tUNWNHjmbMHg/hmNCwc/Pz3k1T/j/Lg==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.59.0.tgz",
|
||||
"integrity": "sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -1863,9 +1889,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-s390x-gnu": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.52.5.tgz",
|
||||
"integrity": "sha512-gn8kHOrku8D4NGHMK1Y7NA7INQTRdVOntt1OCYypZPRt6skGbddska44K8iocdpxHTMMNui5oH4elPH4QOLrFQ==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.59.0.tgz",
|
||||
"integrity": "sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
@@ -1876,9 +1902,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-x64-gnu": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.52.5.tgz",
|
||||
"integrity": "sha512-hXGLYpdhiNElzN770+H2nlx+jRog8TyynpTVzdlc6bndktjKWyZyiCsuDAlpd+j+W+WNqfcyAWz9HxxIGfZm1Q==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.59.0.tgz",
|
||||
"integrity": "sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1889,9 +1915,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-linux-x64-musl": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.52.5.tgz",
|
||||
"integrity": "sha512-arCGIcuNKjBoKAXD+y7XomR9gY6Mw7HnFBv5Rw7wQRvwYLR7gBAgV7Mb2QTyjXfTveBNFAtPt46/36vV9STLNg==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.59.0.tgz",
|
||||
"integrity": "sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1901,10 +1927,23 @@
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-openbsd-x64": {
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.59.0.tgz",
|
||||
"integrity": "sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"openbsd"
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-openharmony-arm64": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.52.5.tgz",
|
||||
"integrity": "sha512-QoFqB6+/9Rly/RiPjaomPLmR/13cgkIGfA40LHly9zcH1S0bN2HVFYk3a1eAyHQyjs3ZJYlXvIGtcCs5tko9Cw==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.59.0.tgz",
|
||||
"integrity": "sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1915,9 +1954,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-win32-arm64-msvc": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.52.5.tgz",
|
||||
"integrity": "sha512-w0cDWVR6MlTstla1cIfOGyl8+qb93FlAVutcor14Gf5Md5ap5ySfQ7R9S/NjNaMLSFdUnKGEasmVnu3lCMqB7w==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.59.0.tgz",
|
||||
"integrity": "sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1928,9 +1967,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-win32-ia32-msvc": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.52.5.tgz",
|
||||
"integrity": "sha512-Aufdpzp7DpOTULJCuvzqcItSGDH73pF3ko/f+ckJhxQyHtp67rHw3HMNxoIdDMUITJESNE6a8uh4Lo4SLouOUg==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.59.0.tgz",
|
||||
"integrity": "sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
@@ -1941,9 +1980,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-win32-x64-gnu": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.52.5.tgz",
|
||||
"integrity": "sha512-UGBUGPFp1vkj6p8wCRraqNhqwX/4kNQPS57BCFc8wYh0g94iVIW33wJtQAx3G7vrjjNtRaxiMUylM0ktp/TRSQ==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.59.0.tgz",
|
||||
"integrity": "sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1954,9 +1993,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@rollup/rollup-win32-x64-msvc": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.52.5.tgz",
|
||||
"integrity": "sha512-TAcgQh2sSkykPRWLrdyy2AiceMckNf5loITqXxFI5VuQjS5tSuw3WlwdN8qv8vzjLAUTvYaH/mVjSFpbkFbpTg==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.59.0.tgz",
|
||||
"integrity": "sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -2377,13 +2416,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": {
|
||||
"version": "9.0.5",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
|
||||
"integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==",
|
||||
"version": "9.0.9",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz",
|
||||
"integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^2.0.1"
|
||||
"brace-expansion": "^2.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16 || 14 >=14.17"
|
||||
@@ -4617,9 +4656,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/immutable": {
|
||||
"version": "5.1.4",
|
||||
"resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.4.tgz",
|
||||
"integrity": "sha512-p6u1bG3YSnINT5RQmx/yRZBpenIl30kVxkTLDyHLIMk0gict704Q9n+thfDI7lTRm9vXdDYutVzXhzcThxTnXA==",
|
||||
"version": "5.1.5",
|
||||
"resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.5.tgz",
|
||||
"integrity": "sha512-t7xcm2siw+hlUM68I+UEOK+z84RzmN59as9DZ7P1l0994DKUWV7UXBMQZVxaoMSRQ+PBZbHCOoBt7a2wxOMt+A==",
|
||||
"devOptional": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
@@ -5458,9 +5497,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"version": "3.1.5",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz",
|
||||
"integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
@@ -6180,9 +6219,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/rollup": {
|
||||
"version": "4.52.5",
|
||||
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.52.5.tgz",
|
||||
"integrity": "sha512-3GuObel8h7Kqdjt0gxkEzaifHTqLVW56Y/bjN7PSQtkKr0w3V/QYSdt6QWYtd7A1xUtYQigtdUfgj1RvWVtorw==",
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz",
|
||||
"integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/estree": "1.0.8"
|
||||
@@ -6195,28 +6234,31 @@
|
||||
"npm": ">=8.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@rollup/rollup-android-arm-eabi": "4.52.5",
|
||||
"@rollup/rollup-android-arm64": "4.52.5",
|
||||
"@rollup/rollup-darwin-arm64": "4.52.5",
|
||||
"@rollup/rollup-darwin-x64": "4.52.5",
|
||||
"@rollup/rollup-freebsd-arm64": "4.52.5",
|
||||
"@rollup/rollup-freebsd-x64": "4.52.5",
|
||||
"@rollup/rollup-linux-arm-gnueabihf": "4.52.5",
|
||||
"@rollup/rollup-linux-arm-musleabihf": "4.52.5",
|
||||
"@rollup/rollup-linux-arm64-gnu": "4.52.5",
|
||||
"@rollup/rollup-linux-arm64-musl": "4.52.5",
|
||||
"@rollup/rollup-linux-loong64-gnu": "4.52.5",
|
||||
"@rollup/rollup-linux-ppc64-gnu": "4.52.5",
|
||||
"@rollup/rollup-linux-riscv64-gnu": "4.52.5",
|
||||
"@rollup/rollup-linux-riscv64-musl": "4.52.5",
|
||||
"@rollup/rollup-linux-s390x-gnu": "4.52.5",
|
||||
"@rollup/rollup-linux-x64-gnu": "4.52.5",
|
||||
"@rollup/rollup-linux-x64-musl": "4.52.5",
|
||||
"@rollup/rollup-openharmony-arm64": "4.52.5",
|
||||
"@rollup/rollup-win32-arm64-msvc": "4.52.5",
|
||||
"@rollup/rollup-win32-ia32-msvc": "4.52.5",
|
||||
"@rollup/rollup-win32-x64-gnu": "4.52.5",
|
||||
"@rollup/rollup-win32-x64-msvc": "4.52.5",
|
||||
"@rollup/rollup-android-arm-eabi": "4.59.0",
|
||||
"@rollup/rollup-android-arm64": "4.59.0",
|
||||
"@rollup/rollup-darwin-arm64": "4.59.0",
|
||||
"@rollup/rollup-darwin-x64": "4.59.0",
|
||||
"@rollup/rollup-freebsd-arm64": "4.59.0",
|
||||
"@rollup/rollup-freebsd-x64": "4.59.0",
|
||||
"@rollup/rollup-linux-arm-gnueabihf": "4.59.0",
|
||||
"@rollup/rollup-linux-arm-musleabihf": "4.59.0",
|
||||
"@rollup/rollup-linux-arm64-gnu": "4.59.0",
|
||||
"@rollup/rollup-linux-arm64-musl": "4.59.0",
|
||||
"@rollup/rollup-linux-loong64-gnu": "4.59.0",
|
||||
"@rollup/rollup-linux-loong64-musl": "4.59.0",
|
||||
"@rollup/rollup-linux-ppc64-gnu": "4.59.0",
|
||||
"@rollup/rollup-linux-ppc64-musl": "4.59.0",
|
||||
"@rollup/rollup-linux-riscv64-gnu": "4.59.0",
|
||||
"@rollup/rollup-linux-riscv64-musl": "4.59.0",
|
||||
"@rollup/rollup-linux-s390x-gnu": "4.59.0",
|
||||
"@rollup/rollup-linux-x64-gnu": "4.59.0",
|
||||
"@rollup/rollup-linux-x64-musl": "4.59.0",
|
||||
"@rollup/rollup-openbsd-x64": "4.59.0",
|
||||
"@rollup/rollup-openharmony-arm64": "4.59.0",
|
||||
"@rollup/rollup-win32-arm64-msvc": "4.59.0",
|
||||
"@rollup/rollup-win32-ia32-msvc": "4.59.0",
|
||||
"@rollup/rollup-win32-x64-gnu": "4.59.0",
|
||||
"@rollup/rollup-win32-x64-msvc": "4.59.0",
|
||||
"fsevents": "~2.3.2"
|
||||
}
|
||||
},
|
||||
|
||||
@@ -1227,7 +1227,10 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
|
||||
#### buckets_limit
|
||||
|
||||
`buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
|
||||
of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`.
|
||||
of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`.
|
||||
|
||||
The result will preserve the first and the last bucket to improve accuracy for min and max values.
|
||||
So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
|
||||
|
||||
See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
|
||||
|
||||
@@ -1381,6 +1384,15 @@ It can be used for calculating the average over the given time range across mult
|
||||
For example, `histogram_avg(sum(histogram_over_time(response_time_duration_seconds[5m])) by (vmrange,job))` would return the average response time
|
||||
per each `job` over the last 5 minutes.
|
||||
|
||||
#### histogram_fraction
|
||||
|
||||
`histogram_fraction(lowerLe, upperLe, buckets)` is a [transform function](#transform-functions), which calculates the share (in the range `[0...1]`) for `buckets` that fall between `lowerLe` and `upperLe`.
|
||||
The result of `histogram_fraction(lowerLe, upperLe, buckets)` is equivalent to `histogram_share(upperLe, buckets) - histogram_share(lowerLe, buckets)`.
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [histogram_share](#histogram_share).
|
||||
|
||||
#### histogram_quantile
|
||||
|
||||
`histogram_quantile(phi, buckets)` is a [transform function](#transform-functions), which calculates `phi`-[percentile](https://en.wikipedia.org/wiki/Percentile)
|
||||
|
||||
@@ -7,6 +7,7 @@ import { AUTOCOMPLETE_LIMITS } from "../../../constants/queryAutocomplete";
|
||||
import { QueryEditorAutocompleteProps } from "./QueryEditor";
|
||||
import { getExprLastPart, getValueByContext, getContext } from "./autocompleteUtils";
|
||||
import { extractCurrentLabel, extractLabelMatchers, extractMetric, splitByCursor } from "./utils/parser";
|
||||
import { escapeLabelName } from "../../../utils/metric";
|
||||
|
||||
const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
value,
|
||||
@@ -90,6 +91,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
}
|
||||
|
||||
if (context === QueryContextType.label) {
|
||||
insert = escapeLabelName(insert);
|
||||
valueAfterCursor = valueAfterCursor.replace(/^[^\s=!,{}()"|+\-/*^]*/, "");
|
||||
}
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ const ExploreMetricItem: FC<ExploreMetricItemGraphProps> = ({
|
||||
|
||||
const base = `{${params.join(",")}}`;
|
||||
if (isBucket) {
|
||||
return [`sum(rate(${base})) by (vmrange, le)`];
|
||||
return [`sum(increase_pure(${base})) by (vmrange, le)`];
|
||||
}
|
||||
const queryBase = rateEnabled ? `rollup_rate(${base})` : `rollup(${base})`;
|
||||
return [`
|
||||
|
||||
@@ -27,6 +27,7 @@ interface TextFieldProps {
|
||||
endIcon?: ReactNode
|
||||
startIcon?: ReactNode
|
||||
disabled?: boolean
|
||||
readonly?: boolean
|
||||
autofocus?: boolean
|
||||
helperText?: string
|
||||
inputmode?: "search" | "text" | "email" | "tel" | "url" | "none" | "numeric" | "decimal"
|
||||
@@ -50,6 +51,7 @@ const TextField: FC<TextFieldProps> = ({
|
||||
endIcon,
|
||||
startIcon,
|
||||
disabled = false,
|
||||
readonly = false,
|
||||
autofocus = false,
|
||||
inputmode = "text",
|
||||
caretPosition,
|
||||
@@ -148,6 +150,7 @@ const TextField: FC<TextFieldProps> = ({
|
||||
<textarea
|
||||
className={inputClasses}
|
||||
disabled={disabled}
|
||||
readOnly={readonly}
|
||||
ref={textareaRef}
|
||||
value={value}
|
||||
rows={1}
|
||||
@@ -166,6 +169,7 @@ const TextField: FC<TextFieldProps> = ({
|
||||
<input
|
||||
className={inputClasses}
|
||||
disabled={disabled}
|
||||
readOnly={readonly}
|
||||
ref={inputRef}
|
||||
value={value}
|
||||
type={type}
|
||||
|
||||
@@ -80,7 +80,7 @@ export default class AppConfigurator {
|
||||
|
||||
let keys: string[] = [];
|
||||
if (focusLabel || isMetricWithLabel) {
|
||||
keys = keys.concat("seriesCountByFocusLabelValue");
|
||||
keys = keys.concat("seriesCountByMetricName", "seriesCountByFocusLabelValue");
|
||||
} else if (isMetric) {
|
||||
keys = keys.concat("labelValueCountByLabelName");
|
||||
} else if (isLabel) {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { QueryUpdater } from "./types";
|
||||
import { escapeLabelName } from "../../utils/metric";
|
||||
|
||||
export const queryUpdater: QueryUpdater = {
|
||||
seriesCountByMetricName: ({ query }): string => {
|
||||
@@ -28,5 +29,5 @@ const getSeriesSelector = (label: string | null, value: string): string => {
|
||||
if (!label) {
|
||||
return "";
|
||||
}
|
||||
return "{" + label + "=" + JSON.stringify(value) + "}";
|
||||
return "{" + escapeLabelName(label) + "=" + JSON.stringify(value) + "}";
|
||||
};
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
TipHighNumberOfValues
|
||||
} from "./CardinalityTips";
|
||||
import useSearchParamsFromObject from "../../hooks/useSearchParamsFromObject";
|
||||
import { escapeLabelName } from "../../utils/metric";
|
||||
|
||||
const spinnerMessage = `Please wait while cardinality stats is calculated.
|
||||
This may take some time if the db contains big number of time series.`;
|
||||
@@ -36,12 +37,17 @@ const CardinalityPanel: FC = () => {
|
||||
const defaultState = getDefaultState(match, focusLabel);
|
||||
|
||||
const handleFilterClick = (key: string) => (query: string) => {
|
||||
const rawQuery = query;
|
||||
|
||||
const isLabelKey = ["labelValueCountByLabelName", "seriesCountByLabelName"].includes(key);
|
||||
if (isLabelKey) query = escapeLabelName(query);
|
||||
|
||||
const value = queryUpdater[key]({ query, focusLabel, match });
|
||||
const params: Record<string, string> = { match: value };
|
||||
if (key === "labelValueCountByLabelName" || key == "seriesCountByLabelName") {
|
||||
params.focusLabel = query;
|
||||
if (isLabelKey) {
|
||||
params.focusLabel = rawQuery;
|
||||
}
|
||||
if (key == "seriesCountByFocusLabelValue") {
|
||||
if (key === "seriesCountByFocusLabelValue") {
|
||||
params.focusLabel = "";
|
||||
}
|
||||
setSearchParamsFromKeys(params);
|
||||
|
||||
@@ -115,16 +115,20 @@ const DownsamplingFilters: FC = () => {
|
||||
</div>
|
||||
<div className="vm-downsampling-filters-body-top">
|
||||
<a
|
||||
className="vm-link vm-link_with-icon"
|
||||
target="_blank"
|
||||
href="https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling"
|
||||
rel="help noreferrer"
|
||||
>
|
||||
<WikiIcon/>
|
||||
Documentation
|
||||
<Button
|
||||
variant="text"
|
||||
color="gray"
|
||||
startIcon={<WikiIcon/>}
|
||||
>
|
||||
Documentation
|
||||
</Button>
|
||||
</a>
|
||||
<Button
|
||||
variant="text"
|
||||
variant="outlined"
|
||||
onClick={handleRunExample}
|
||||
>
|
||||
Try example
|
||||
@@ -134,7 +138,7 @@ const DownsamplingFilters: FC = () => {
|
||||
onClick={handleApplyFilters}
|
||||
startIcon={<PlayIcon/>}
|
||||
>
|
||||
Apply
|
||||
Preview
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -90,25 +90,33 @@ const Relabel: FC = () => {
|
||||
</div>
|
||||
<div className="vm-relabeling-header-bottom">
|
||||
<a
|
||||
className="vm-link vm-link_with-icon"
|
||||
target="_blank"
|
||||
href="https://docs.victoriametrics.com/victoriametrics/relabeling/"
|
||||
rel="help noreferrer"
|
||||
>
|
||||
<InfoIcon/>
|
||||
Relabeling cookbook
|
||||
<Button
|
||||
variant="text"
|
||||
color="gray"
|
||||
startIcon={<InfoIcon/>}
|
||||
>
|
||||
Relabeling cookbook
|
||||
</Button>
|
||||
</a>
|
||||
<a
|
||||
className="vm-link vm-link_with-icon"
|
||||
target="_blank"
|
||||
href="https://docs.victoriametrics.com/victoriametrics/relabeling/"
|
||||
rel="help noreferrer"
|
||||
>
|
||||
<WikiIcon/>
|
||||
Documentation
|
||||
<Button
|
||||
variant="text"
|
||||
color="gray"
|
||||
startIcon={<WikiIcon/>}
|
||||
>
|
||||
Documentation
|
||||
</Button>
|
||||
</a>
|
||||
<Button
|
||||
variant="text"
|
||||
variant="outlined"
|
||||
onClick={handleRunExample}
|
||||
>
|
||||
Try example
|
||||
@@ -118,7 +126,7 @@ const Relabel: FC = () => {
|
||||
onClick={handleRunQuery}
|
||||
startIcon={<PlayIcon/>}
|
||||
>
|
||||
Submit
|
||||
Preview
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
gap: $padding-global;
|
||||
gap: $padding-small;
|
||||
|
||||
a {
|
||||
color: $color-text-secondary;
|
||||
|
||||
@@ -107,16 +107,20 @@ const RetentionFilters: FC = () => {
|
||||
</div>
|
||||
<div className="vm-retention-filters-body-top">
|
||||
<a
|
||||
className="vm-link vm-link_with-icon"
|
||||
target="_blank"
|
||||
href="https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention-filters"
|
||||
rel="help noreferrer"
|
||||
>
|
||||
<WikiIcon/>
|
||||
Documentation
|
||||
<Button
|
||||
variant="text"
|
||||
color="gray"
|
||||
startIcon={<WikiIcon/>}
|
||||
>
|
||||
Documentation
|
||||
</Button>
|
||||
</a>
|
||||
<Button
|
||||
variant="text"
|
||||
variant="outlined"
|
||||
onClick={handleRunExample}
|
||||
>
|
||||
Try example
|
||||
@@ -126,7 +130,7 @@ const RetentionFilters: FC = () => {
|
||||
onClick={handleApplyFilters}
|
||||
startIcon={<PlayIcon/>}
|
||||
>
|
||||
Apply
|
||||
Preview
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -48,7 +48,7 @@ const WithTemplate: FC = () => {
|
||||
type="textarea"
|
||||
label="MetricsQL query after expanding WITH expressions and applying other optimizations"
|
||||
value={data}
|
||||
disabled
|
||||
readonly
|
||||
/>
|
||||
</div>
|
||||
<div className="vm-with-template-body-top">
|
||||
|
||||
@@ -52,3 +52,7 @@ export const isHistogramData = (result: MetricBase[]) => {
|
||||
|
||||
return isHistogram && result.every(r => histogramLabels.some(l => l in r.metric));
|
||||
};
|
||||
|
||||
export const escapeLabelName = (s: string) => {
|
||||
return s.replace(/([\\./-])/g, "\\$1");
|
||||
};
|
||||
|
||||
@@ -21,7 +21,7 @@ const getProxy = (): Record<string, ProxyOptions> | undefined => {
|
||||
};
|
||||
|
||||
return {
|
||||
"^/prometheus/(api|vmalert)/.*": { ...commonProxy },
|
||||
"^/prometheus/.*": { ...commonProxy },
|
||||
"/prometheus/vmui/config.json": { ...commonProxy },
|
||||
};
|
||||
};
|
||||
|
||||
@@ -33,6 +33,8 @@ type PrometheusQuerier interface {
|
||||
// separate interface or rename this interface to allow for multiple querier
|
||||
// types.
|
||||
GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse
|
||||
GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts)
|
||||
GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts)
|
||||
}
|
||||
|
||||
// Writer contains methods for writing new data
|
||||
|
||||
@@ -60,3 +60,60 @@ func TestClusterMetricsIndex(t *testing.T) {
|
||||
|
||||
testMetricsIndex(tc.T(), sut)
|
||||
}
|
||||
|
||||
// testTagSeries tests the registration of new time series in index.
|
||||
//
|
||||
// See https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb.
|
||||
func testTagSeries(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, getStorageMetric func(string) int) {
|
||||
t := tc.T()
|
||||
|
||||
assertNewTimeseriesCreatedTotal := func(want int) {
|
||||
tc.Assert(&apptest.AssertOptions{
|
||||
Msg: "unexpected vm_new_timeseries_created_total",
|
||||
Got: func() any {
|
||||
return getStorageMetric("vm_new_timeseries_created_total")
|
||||
},
|
||||
Want: want,
|
||||
})
|
||||
}
|
||||
|
||||
rec := "disk.used;rack=a1;datacenter=dc1;server=web01"
|
||||
sut.GraphiteTagsTagSeries(t, rec, apptest.QueryOpts{})
|
||||
assertNewTimeseriesCreatedTotal(0)
|
||||
|
||||
recs := []string{
|
||||
"metric.yyy;t2=a;t1=b;t3=c",
|
||||
"metric.zzz;t5=d;t4=e;t6=f",
|
||||
"metric.xxx;t8=g;t7=h;t9=i",
|
||||
}
|
||||
sut.GraphiteTagsTagMultiSeries(t, recs, apptest.QueryOpts{})
|
||||
assertNewTimeseriesCreatedTotal(0)
|
||||
}
|
||||
|
||||
func TestSingleTagSeries(t *testing.T) {
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
sut := tc.MustStartDefaultVmsingle()
|
||||
getStorageMetric := func(name string) int {
|
||||
return sut.GetIntMetric(t, name)
|
||||
}
|
||||
|
||||
testTagSeries(tc, sut, getStorageMetric)
|
||||
}
|
||||
|
||||
func TestClusterTagSeries(t *testing.T) {
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
sut := tc.MustStartDefaultCluster()
|
||||
getStorageMetric := func(name string) int {
|
||||
var v int
|
||||
for _, s := range sut.Vmstorages {
|
||||
v += s.GetIntMetric(t, name)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
testTagSeries(tc, sut, getStorageMetric)
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ func TestSingleMetricsMetadata(t *testing.T) {
|
||||
sut := tc.MustStartVmsingle("vmsingle", []string{
|
||||
"-storageDataPath=" + tc.Dir(),
|
||||
"-retentionPeriod=100y",
|
||||
"-enableMetadata",
|
||||
})
|
||||
// verify empty stats
|
||||
resp := sut.PrometheusAPIV1Metadata(t, "", 0, apptest.QueryOpts{})
|
||||
@@ -120,15 +119,12 @@ func TestClusterMetricsMetadata(t *testing.T) {
|
||||
|
||||
vminsert1 := tc.MustStartVminsert("vminsert1", []string{
|
||||
fmt.Sprintf("-storageNode=%s,%s", vmstorage1.VminsertAddr(), vmstorage2.VminsertAddr()),
|
||||
"-enableMetadata",
|
||||
})
|
||||
vminsert2 := tc.MustStartVminsert("vminsert-2", []string{
|
||||
fmt.Sprintf("-storageNode=%s,%s", vmstorage1.VminsertAddr(), vmstorage2.VminsertAddr()),
|
||||
"-enableMetadata",
|
||||
})
|
||||
vminsertGlobal := tc.MustStartVminsert("vminsert-global", []string{
|
||||
fmt.Sprintf("-storageNode=%s,%s", vminsert1.ClusternativeListenAddr(), vminsert2.ClusternativeListenAddr()),
|
||||
"-enableMetadata",
|
||||
})
|
||||
vmselect := tc.MustStartVmselect("vmselect", []string{
|
||||
fmt.Sprintf("-storageNode=%s,%s", vmstorage1.VmselectAddr(), vmstorage2.VmselectAddr()),
|
||||
|
||||
@@ -32,6 +32,8 @@ func TestSingleInstantQuery(t *testing.T) {
|
||||
testInstantQueryDoesNotReturnStaleNaNs(t, sut)
|
||||
|
||||
testQueryRangeWithAtModifier(t, sut)
|
||||
|
||||
testLabelValuesWithUTFNames(t, sut)
|
||||
}
|
||||
|
||||
func TestClusterInstantQuery(t *testing.T) {
|
||||
@@ -44,6 +46,8 @@ func TestClusterInstantQuery(t *testing.T) {
|
||||
testInstantQueryDoesNotReturnStaleNaNs(t, sut)
|
||||
|
||||
testQueryRangeWithAtModifier(t, sut)
|
||||
|
||||
testLabelValuesWithUTFNames(t, sut)
|
||||
}
|
||||
|
||||
func testInstantQueryWithUTFNames(t *testing.T, sut apptest.PrometheusWriteQuerier) {
|
||||
@@ -236,3 +240,46 @@ func testQueryRangeWithAtModifier(t *testing.T, sut apptest.PrometheusWriteQueri
|
||||
t.Fatalf("unexpected error: %q", resp.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// This test checks that label values are decoded from UTF-8 according to Prometheus spec.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10446
|
||||
// Spec: https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
|
||||
func testLabelValuesWithUTFNames(t *testing.T, sut apptest.PrometheusWriteQuerier) {
|
||||
|
||||
timestamp := millis("2025-01-01T00:00:00Z")
|
||||
data := prompb.WriteRequest{
|
||||
Timeseries: []prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "__name__", Value: "labelvals"},
|
||||
{Name: "kubernetes_something/special&' chars", Value: "漢©®€£"},
|
||||
{Name: "3👋tfにちは", Value: "漢©®€£"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: timestamp},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
sut.PrometheusAPIV1Write(t, data, apptest.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
|
||||
cmpOptions := []cmp.Option{}
|
||||
|
||||
// encoded via prometheus model.EscapeName(string,model.ValueEncodingEscaping)
|
||||
want := map[string][]string{
|
||||
"__name__": {"labelvals"},
|
||||
"U__kubernetes__something_2f_special_26__27__20_chars": {"漢©®€£"},
|
||||
"U___33__1f44b_tf_306b__3061__306f_": {"漢©®€£"},
|
||||
}
|
||||
for labelName, expected := range want {
|
||||
got := sut.PrometheusAPIV1LabelValues(t, labelName, `{__name__="labelvals"}`, apptest.QueryOpts{
|
||||
Start: fmt.Sprintf("%d", timestamp),
|
||||
End: fmt.Sprintf("%d", timestamp),
|
||||
})
|
||||
if diff := cmp.Diff(expected, got.Data, cmpOptions...); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,6 +171,26 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
|
||||
// /api/v1/label/../value with extra_filters
|
||||
|
||||
wantVR := apptest.NewPrometheusAPIV1LabelValuesResponse(t,
|
||||
`{"data": [
|
||||
"5"
|
||||
]
|
||||
}`)
|
||||
wantSR.Sort()
|
||||
gotVR := vmselect.PrometheusAPIV1LabelValues(t, "vm_account_id", "foo", apptest.QueryOpts{
|
||||
Start: "2022-05-10T08:00:00.000Z",
|
||||
End: "2022-05-10T08:30:00.000Z",
|
||||
ExtraFilters: []string{`{vm_account_id="5"}`},
|
||||
Tenant: "multitenant",
|
||||
})
|
||||
gotSR.Sort()
|
||||
|
||||
if diff := cmp.Diff(wantVR, gotVR, cmpopts.IgnoreFields(apptest.PrometheusAPIV1LabelValuesResponse{}, "Status", "IsPartial")); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
|
||||
// Delete series from specific tenant
|
||||
vmselect.APIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
|
||||
Tenant: "5:15",
|
||||
|
||||
@@ -61,8 +61,8 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
|
||||
|
||||
type startSUTFunc func(name string, disablePerDayIndex bool) apptest.PrometheusWriteQuerier
|
||||
|
||||
// testDisablePerDayIndex_Search shows what search results to expect when data
|
||||
// is first inserted with per-day index enabled and then with per-day index
|
||||
// testSearchWithDisabledPerDayIndex shows what search results to expect when
|
||||
// data is first inserted with per-day index enabled and then with per-day index
|
||||
// disabled.
|
||||
//
|
||||
// The data inserted with enabled per-day index must be searchable with disabled
|
||||
@@ -112,8 +112,8 @@ func testSearchWithDisabledPerDayIndex(tc *apptest.TestCase, start startSUTFunc)
|
||||
})
|
||||
}
|
||||
|
||||
// Start vmsingle with enabled per-day index, insert sample1, and confirm it
|
||||
// is searchable.
|
||||
// Start SUT with enabled per-day index, insert sample1, and confirm it is
|
||||
// searchable.
|
||||
sut := start("with-per-day-index", false)
|
||||
sample1 := []string{"metric1 111 1704067200000"} // 2024-01-01T00:00:00Z
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, sample1, apptest.QueryOpts{})
|
||||
@@ -130,8 +130,8 @@ func testSearchWithDisabledPerDayIndex(tc *apptest.TestCase, start startSUTFunc)
|
||||
},
|
||||
})
|
||||
|
||||
// Restart vmsingle with disabled per-day index, insert sample2, and confirm
|
||||
// that both sample1 and sample2 is searchable.
|
||||
// Restart SUT with disabled per-day index, insert sample2, and confirm that
|
||||
// both sample1 and sample2 is searchable.
|
||||
tc.StopPrometheusWriteQuerier(sut)
|
||||
sut = start("without-per-day-index", true)
|
||||
sample2 := []string{"metric2 222 1704067200000"} // 2024-01-01T00:00:00Z
|
||||
@@ -156,8 +156,8 @@ func testSearchWithDisabledPerDayIndex(tc *apptest.TestCase, start startSUTFunc)
|
||||
},
|
||||
})
|
||||
|
||||
// Insert sample1 but for a different date, restart vmsingle with enabled
|
||||
// per-day index and confirm that:
|
||||
// Insert sample1 but for a different date, restart SUT with enabled per-day
|
||||
// index and confirm that:
|
||||
// - sample1 is searchable within the time range of Jan 1st
|
||||
// - sample1 is not searchable within the time range of Jan 20th
|
||||
// - sample1 is searchable within the time range of Jan 1st-20th (because
|
||||
|
||||
@@ -22,7 +22,7 @@ func NewPrometheusMockStorage(series []*prompb.TimeSeries) *PrometheusMockStorag
|
||||
return &PrometheusMockStorage{store: series}
|
||||
}
|
||||
|
||||
// ReadMultiple implemnets the storage.ReadClient interface for reading time series data.
|
||||
// ReadMultiple implements the storage.ReadClient interface for reading time series data.
|
||||
func (ms *PrometheusMockStorage) ReadMultiple(ctx context.Context, queries []*prompb.Query, sortSeries bool) (storage.SeriesSet, error) {
|
||||
if len(queries) != 1 {
|
||||
panic(fmt.Errorf("reading multiple queries isn't implemented"))
|
||||
|
||||
@@ -278,7 +278,7 @@ func validateStreamReadHeaders(t *testing.T, r *http.Request) bool {
|
||||
func GenerateRemoteReadSeries(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries {
|
||||
var ts []*prompb.TimeSeries
|
||||
j := 0
|
||||
for i := 0; i < int(numOfSeries); i++ {
|
||||
for i := range int(numOfSeries) {
|
||||
if i%3 == 0 {
|
||||
j++
|
||||
}
|
||||
|
||||
@@ -313,7 +313,7 @@ func TestSingleVMAgentDropOnOverload(t *testing.T) {
|
||||
|
||||
waitFor := func(f func() bool) {
|
||||
t.Helper()
|
||||
for i := 0; i < retries; i++ {
|
||||
for range retries {
|
||||
if f() {
|
||||
return
|
||||
}
|
||||
@@ -339,7 +339,7 @@ func TestSingleVMAgentDropOnOverload(t *testing.T) {
|
||||
|
||||
// Send 2 more requests, the first RW endpoint should receive everything, the second should add them to the queue
|
||||
// since worker is busy with the first request.
|
||||
for i := 0; i < 2; i++ {
|
||||
for i := range 2 {
|
||||
vmagent.APIV1ImportPrometheusNoWaitFlush(t, []string{
|
||||
"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
|
||||
}, apptest.QueryOpts{})
|
||||
|
||||
@@ -145,7 +145,7 @@ func (app *Vmagent) ReloadRelabelConfigs(t *testing.T) {
|
||||
}
|
||||
|
||||
var currTotal float64
|
||||
for i := 0; i < 30; i++ {
|
||||
for range 30 {
|
||||
currTotal = app.GetMetric(t, "vmagent_relabel_config_reloads_total")
|
||||
if currTotal > prevTotal {
|
||||
return
|
||||
|
||||
@@ -298,13 +298,14 @@ func (app *Vminsert) String() string {
|
||||
func (app *Vminsert) sendBlocking(t *testing.T, numRecordsToSend int, send func()) {
|
||||
t.Helper()
|
||||
|
||||
wantRowsSentCount := app.rpcRowsSentTotal(t) + numRecordsToSend
|
||||
|
||||
send()
|
||||
|
||||
const (
|
||||
retries = 20
|
||||
period = 100 * time.Millisecond
|
||||
)
|
||||
wantRowsSentCount := app.rpcRowsSentTotal(t) + numRecordsToSend
|
||||
for range retries {
|
||||
d := app.rpcRowsSentTotal(t)
|
||||
if d >= wantRowsSentCount {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user