apptest/tests: add test to verify sparse cache usage

Sparse cache is only used for "final merges" - merge of data of previous months, so tests verify that.

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

app/vlinsert/insertutils/common_params.go

@@ -199,8 +199,8 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []l
 	lmp.bytesIngestedTotal.Add(n)
 
 	if len(fields) > *MaxFieldsPerLine {
-		rf := logstorage.RowFormatter(fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
+		line := logstorage.MarshalFieldsToJSON(nil, fields)
+		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, line)
 		rowsDroppedTotalTooManyFields.Inc()
 		return
 	}

app/vlinsert/insertutils/flags.go

@@ -8,8 +8,10 @@ import (
 
 var (
 	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
-	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
+	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers; "+
+		"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have")
 
 	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
-	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
+	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers; "+
+		"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain")
 )

app/vlogscli/main.go

@@ -270,7 +270,7 @@ func printCommandsHelp(w io.Writer) {
 \h - show this help
 \s - singleline json output mode
 \m - multiline json output mode
-\c - compact output
+\c - compact output mode
 \logfmt - logfmt output mode
 \wrap_long_lines - toggles wrapping long lines
 \tail <query> - live tail <query> results

apptest/tests/sparse_cache_test.go

@@ -0,0 +1,76 @@
+package tests
+
+import (
+	"fmt"
+	"math/rand/v2"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestStorageUsesSparseCacheForFinalMerge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartVmsingle("sparse-cache-final-merge", []string{`-retentionPeriod=100y`, `-downsampling.period={__name__=~"metric.*"}:5m:1s`})
+
+	// insert metrics daily over past 2 months
+	const metricsPerStep = 10
+	const steps = 35
+	const stepSize = int64(1 * 24 * 60 * 60 * 1000)
+	records := make([]string, metricsPerStep)
+	ts := time.Now().Add(-2 * 30 * 24 * time.Hour).UnixMilli()
+	for range steps {
+		for i := range metricsPerStep {
+			name := fmt.Sprintf("metric_%d", i)
+			records[i] = fmt.Sprintf("%s %d %d", name, rand.IntN(1000), ts)
+		}
+		sut.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{})
+		ts += stepSize
+	}
+	sut.ForceFlush(t)
+	sut.ForceMerge(t)
+
+	// todo: replace with a more reliable way to check if the merge is completed
+	// wait for merge to be completed
+	time.Sleep(5 * time.Second)
+
+	v := sut.GetIntMetric(t, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`)
+	if v <= 0 {
+		t.Fatalf(`unexpected vm_cache_requests_total{type="indexdb/dataBlocksSparse"} value: %d`, v)
+	}
+}
+
+func TestStorageDoesNotUseSparseCacheForRegularMerge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartVmsingle("sparse-cache-regular-merge", []string{`-retentionPeriod=100y`, `-downsampling.period={__name__=~"metric.*"}:5m:1s`})
+
+	// insert metrics into current month only
+	const metricsPerStep = 10
+	const steps = 2
+	const stepSize = 60 * 1000
+	records := make([]string, metricsPerStep)
+	ts := time.Now().Add(-1 * time.Hour).UnixMilli()
+	for range steps {
+		for i := range metricsPerStep {
+			name := fmt.Sprintf("metric_%d", i)
+			records[i] = fmt.Sprintf("%s %d %d", name, rand.IntN(1000), ts)
+		}
+		sut.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{})
+		ts += stepSize
+	}
+	sut.ForceFlush(t)
+	sut.ForceMerge(t)
+
+	// todo: replace with a more reliable way to check if the merge is completed
+	// wait for merge to be completed
+	time.Sleep(5 * time.Second)
+
+	v := sut.GetIntMetric(t, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`)
+	if v == 0 {
+		t.Fatalf(`unexpected vm_cache_requests_total{type="indexdb/dataBlocksSparse"} value: %d`, v)
+	}
+}

apptest/vmsingle.go

@@ -24,6 +24,7 @@ type Vmsingle struct {
 
 	// vmstorage URLs.
 	forceFlushURL string
+	forceMergeURL string
 
 	// vminsert URLs.
 	influxLineWriteURL                 string
@@ -65,6 +66,7 @@ func StartVmsingle(instance string, flags []string, cli *Client) (*Vmsingle, err
 		httpListenAddr:  stderrExtracts[1],
 
 		forceFlushURL:                      fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
+		forceMergeURL:                      fmt.Sprintf("http://%s/internal/force_merge", stderrExtracts[1]),
 		influxLineWriteURL:                 fmt.Sprintf("http://%s/influx/write", stderrExtracts[1]),
 		prometheusAPIV1ImportPrometheusURL: fmt.Sprintf("http://%s/prometheus/api/v1/import/prometheus", stderrExtracts[1]),
 		prometheusAPIV1WriteURL:            fmt.Sprintf("http://%s/prometheus/api/v1/write", stderrExtracts[1]),
@@ -83,6 +85,13 @@ func (app *Vmsingle) ForceFlush(t *testing.T) {
 	app.cli.Get(t, app.forceFlushURL, http.StatusOK)
 }
 
+// ForceMerge is a test helper function that forces the merging of parts.
+func (app *Vmsingle) ForceMerge(t *testing.T) {
+	t.Helper()
+
+	app.cli.Get(t, app.forceMergeURL, http.StatusOK)
+}
+
 // InfluxWrite is a test helper function that inserts a
 // collection of records in Influx line format by sending a HTTP
 // POST request to /influx/write vmsingle endpoint.

apptest/vmstorage.go

@@ -21,6 +21,7 @@ type Vmstorage struct {
 	vmselectAddr    string
 
 	forceFlushURL string
+	forceMergeURL string
 }
 
 // StartVmstorage starts an instance of vmstorage with the given flags. It also
@@ -57,6 +58,7 @@ func StartVmstorage(instance string, flags []string, cli *Client) (*Vmstorage, e
 		vmselectAddr:    stderrExtracts[3],
 
 		forceFlushURL: fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
+		forceMergeURL: fmt.Sprintf("http://%s/internal/force_merge", stderrExtracts[1]),
 	}, nil
 }
 
@@ -80,6 +82,12 @@ func (app *Vmstorage) ForceFlush(t *testing.T) {
 	app.cli.Get(t, app.forceFlushURL, http.StatusOK)
 }
 
+// ForceMerge is a test helper function that forces the merging of parts.
+func (app *Vmstorage) ForceMerge(t *testing.T) {
+	t.Helper()
+	app.cli.Get(t, app.forceMergeURL, http.StatusOK)
+}
+
 // String returns the string representation of the vmstorage app state.
 func (app *Vmstorage) String() string {
 	return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q vminsertAddr: %q vmselectAddr: %q}", []any{

dashboards/victorialogs.json

@@ -1686,7 +1686,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/victoriametrics-cluster.json

@@ -1933,7 +1933,7 @@
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/victoriametrics.json

@@ -1981,7 +1981,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/vm/victorialogs.json

@@ -1687,7 +1687,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/vm/victoriametrics-cluster.json

@@ -1934,7 +1934,7 @@
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/vm/victoriametrics.json

@@ -1982,7 +1982,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/vm/vmagent.json

@@ -1666,7 +1666,7 @@
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/vm/vmalert.json

@@ -1473,7 +1473,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n  rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n  / \n  vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+              "expr": "max(\n  rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n  / \n  process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/vm/vmauth.json

@@ -1361,7 +1361,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/vmagent.json

@@ -1665,7 +1665,7 @@
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/vmalert.json

@@ -1472,7 +1472,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n  rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n  / \n  vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+              "expr": "max(\n  rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n  / \n  process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/vmauth.json

@@ -1360,7 +1360,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

docs/Articles.md

@@ -86,6 +86,7 @@ See also [case studies](https://docs.victoriametrics.com/casestudies/).
 * [Persistent Data Structures in VictoriaMetrics (Part 2): vmselect](https://medium.com/@jiekun/persistent-data-structures-in-victoriametrics-part-2-vmselect-9e3de39a4d20)
 * [Migrating to VictoriaMetrics (by Zomato): A Complete Overhaul for Enhanced Observability](https://blog.zomato.com/migrating-to-victoriametrics-a-complete-overhaul-for-enhanced-observability)
 * [Harness the Power of VictoriaMetrics and Grafana Operators for Metrics Management](https://blog.ogenki.io/post/series/observability/metrics/)
+* [Reducing Inter-AZ traffic in VictoriaMetrics with Zonekeeper](https://tanmay-bhat.medium.com/reducing-inter-az-traffic-in-victoriametrics-with-zonekeeper-3bd7e1526796)
 
 ## Our articles
 

docs/BestPractices.md

@@ -48,6 +48,21 @@ VictoriaMetrics can run also on MacOS for testing and development purposes.
 * **MacOS**: amd64, arm64 (for testing and development purposes)
 * **Windows**: amd64
 
+## Kubernetes
+
+VictoriaMetrics natively supports deployment in Kubernetes via [helm charts](https://docs.victoriametrics.com/helm/)
+and [kubernetes operator](https://docs.victoriametrics.com/operator/). See how to [start using k8s operator](https://docs.victoriametrics.com/guides/getting-started-with-vm-operator/).
+
+Common recommendations:
+1. Prefer setting [requests equal to limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits)
+for stateful components like [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/#architecture-overview) to avoid unnecessary
+component restarts.
+
+1. Avoid using [fractional CPU units](https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units) 
+when setting resources for optimal performance. VictoriaMetrics is written in Go and its runtime requires specifying
+[integer number](https://pkg.go.dev/runtime#GOMAXPROCS) of concurrently running threads. 
+When fractional CPU unit is specified, VictoriaMetrics will automatically round it down. 
+
 ## Upgrade procedure
 
 It is safe to upgrade VictoriaMetrics to new versions unless the [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) say otherwise.

docs/VictoriaLogs/CHANGELOG.md

@@ -19,6 +19,8 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
 * FEATURE: [`block_stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#block_stats-pipe): return the path to the part where every data block is stored. The path to the part is returned in the `part_path` field. This allows investigating the distribution of data blocks among parts.
 * FEATURE: reduce VictoriaLogs startup time by multiple times when it opens a large datastore with big [retention](https://docs.victoriametrics.com/victorialogs/#retention).
 
+* BUGFIX: [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/): drop log entries with too long field names and log the dropped log entries with the `ignoring log entry with too long field name` message, so human operators could notice and fix the ingestion of incorrect logs ASAP. Previously too long field names were silently truncated to shorter values. This isn't what most users expect. See [why VictoriaLogs has a limit on the field name length](https://docs.victoriametrics.com/victorialogs/faq/#what-is-the-maximum-supported-field-name-length).
+
 ## [v1.8.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.8.0-victorialogs)
 
 Released at 2025-01-24

docs/VictoriaLogs/FAQ.md

@@ -184,16 +184,36 @@ VictoriaLogs works optimally with log records of up to `10KB`. It works OK with
 log records of up to `100KB`. It works not so optimal with log records exceeding
 `100KB`.
 
-The max size of a log record VictoriaLogs can handle is `2MB`. This is
-because VictoriaLogs stores log records in blocks and `2MB` is the max size of a
-block. Blocks of this size fit the L2 cache of a typical CPU, which gives an
-optimal processing performance.
+The max size of a log record VictoriaLogs can accept during [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/)
+is `2MB`, because log records are stored in blocks of up to `2MB` size.
+Blocks of this size fit the L2 cache of a typical CPU, which gives an
+optimal performance during data ingestion and querying.
 
-However, log records whose size is close to `2MB` aren't handled efficiently by
+Note that log records with sizes close to `2MB` aren't handled efficiently by
 VictoriaLogs because per-block overhead translates to a single log record, and
-this overhead is big. 
+this overhead is big.
 
-The `2MB` limit is hadrcoded and is unlikely to change.
+The `2MB` limit is hadrcoded and is unlikely to increase.
+
+The limit can be set to the lower value during [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/)
+via `-insert.maxLineSizeBytes` command-line flag.
+
+## What is the maximum supported field name length
+
+VictoriaLogs limits [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) name length to 128 bytes -
+Log entries with longer field names are ignored during [date ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/).
+
+The maximum length of a field name is hardcoded and is unikely to increase, since this may increase RAM and CPU usage.
+
+## How many fields a single log entry may contain
+
+A single log entry may contain up to 2000 fields. This fits well the majority of use cases for structured logs and
+for [wide events](https://jeremymorrell.dev/blog/a-practitioners-guide-to-wide-events/).
+
+The maximum number of fields per log entry is hardcoded and is unlikely to increase, since this may increase RAM and CPU usage.
+
+The limit can be set to the lower value during [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/)
+via `-insert.maxFieldsPerLine` command-line flag.
 
 ## How to determine which log fields occupy the most of disk space?
 

docs/changelog/CHANGELOG.md

@@ -20,6 +20,28 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 
 * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): improve startup times when opening a storage with the [retention](https://docs.victoriametrics.com/#retention) exceeding a few months.
 
+## [v1.102.12](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.12)
+
+Released at 2025-01-28
+
+**v1.102.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.102.x line will be supported for at least 12 months since [v1.102.0](https://docs.victoriametrics.com/changelog/#v11020) release**
+
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): log metric names for signals with unsupported delta temporality on ingestion via [OpenTelemetry protocol for metrics](https://docs.victoriametrics.com/#sending-data-via-opentelemetry). Thanks to @chenlujjj for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8018).
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): respect staleness detection in increase, increase_pure and delta functions when time series has gaps and `-search.maxStalenessInterval` is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8072) for details.
+
+## [v1.97.17](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.17)
+
+Released at 2025-01-28
+
+**v1.97.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.97.x line will be supported for at least 12 months since [v1.97.0](https://docs.victoriametrics.com/CHANGELOG.html#v1970) release**
+
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): log metric names for signals with unsupported delta temporality on ingestion via [OpenTelemetry protocol for metrics](https://docs.victoriametrics.com/#sending-data-via-opentelemetry). Thanks to @chenlujjj for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8018).
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): respect staleness detection in increase, increase_pure and delta functions when time series has gaps and `-search.maxStalenessInterval` is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8072) for details.
+
 ## [v1.110.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.0)
 
 Released at 2025-01-24

docs/victoriametrics-cloud/tiers-parameters.md

@@ -24,10 +24,10 @@ For a detailed explanation of each parameter, visit the guide on [Understanding
 
 ## Flag Parameters Configuration
 
-| **Flag**                          | **Default Value**         | **Description**                                                                                                                                                                                                                                                                                                                                    |
-|-----------------------------------|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Max Label Value Length**        | `<= 1kb` (Default: `4kb`) | Maximum length of label values. Longer values are truncated. Large label values can lead to high RAM consumption. This can be adjusted via [support](mailto:support-cloud@victoriametrics.com).                                                                                                                                                          |
-| **Max Labels per Time Series**    | `<= 30`                   | Maximum number of labels per time series. Excess labels are dropped. Higher values can increase [cardinality](https://docs.victoriametrics.com/keyconcepts/#cardinality) and resource usage. This can be configured in [deployment settings](https://docs.victoriametrics.com/victoriametrics-cloud/quickstart/#modifying-an-existing-deployment). |
+| **Flag**                          | **Default Value**         | **Description**                                                                                                                                                                                                                                                                                                                                                     |
+|-----------------------------------|---------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **Max Label Value Length**        | `<= 1kb` (Default: `4kb`) | Maximum length of label values. Time series with longer values are dropped. Large label values can lead to high RAM consumption. This can be adjusted via [support](mailto:support-cloud@victoriametrics.com).                                                                                                                                                      |
+| **Max Labels per Time Series**    | `<= 30`                   | Maximum number of labels per time series. Time series with excess labels are dropped. Higher values can increase [cardinality](https://docs.victoriametrics.com/keyconcepts/#cardinality) and resource usage. This can be configured in [deployment settings](https://docs.victoriametrics.com/victoriametrics-cloud/quickstart/#modifying-an-existing-deployment). |
 
 
 ## Terms and definitions:

docs/victoriametrics-datasource.md

@@ -46,7 +46,7 @@ Installing VictoriaMetrics Grafana datasource [requires](https://grafana.com/doc
 
 ``` ini
 [plugins]
-allow_loading_unsigned_plugins = victoriametrics-datasource
+allow_loading_unsigned_plugins = victoriametrics-metrics-datasource
 ```
 
 For `grafana-operator` users, please adjust `config:` section in your `kind=Grafana` resource as below
@@ -54,7 +54,7 @@ For `grafana-operator` users, please adjust `config:` section in your `kind=Graf
 ```
   config:
     plugins:
-      allow_loading_unsigned_plugins: "victoriametrics-datasource"
+      allow_loading_unsigned_plugins: "victoriametrics-metrics-datasource"
 ```
 
 See [why VictoriaMetrics datasource is unsigned](#why-victoriaMetrics-datasource-is-unsigned).
@@ -83,7 +83,7 @@ datasources:
   # displayed in Grafana panels and queries.
   - name: VictoriaMetrics
     # <string, required> Sets the data source type.
-    type: victoriametrics-datasource
+    type: victoriametrics-metrics-datasource
       # <string, required> Sets the access mode, either
       # proxy or direct (Server or Browser in the UI).
       # Some data sources are incompatible with any setting
@@ -99,7 +99,7 @@ datasources:
     # displayed in Grafana panels and queries.
   - name: VictoriaMetrics - cluster
     # <string, required> Sets the data source type.
-    type: victoriametrics-datasource
+    type: victoriametrics-metrics-datasource
     # <string, required> Sets the access mode, either
     # proxy or direct (Server or Browser in the UI).
     # Some data sources are incompatible with any setting
@@ -124,8 +124,8 @@ Please find the example of provisioning Grafana instance with VictoriaMetrics da
        grafana:
          image: grafana/grafana:11.0.0
          environment:
-         - GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.3/victoriametrics-datasource-v0.10.3.zip;victoriametrics-datasource
-         - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-datasource
+         - GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.13.0/victoriametrics-metrics-datasource-v0.13.0.zip;victoriametrics-metrics-datasource
+         - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-metrics-datasource
          ports:
          - 3000:3000/tcp
          volumes:
@@ -152,14 +152,14 @@ Option 1. Using Grafana provisioning:
 
 ``` yaml
 env:
-  GF_INSTALL_PLUGINS: "https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.3/victoriametrics-datasource-v0.10.3.zip;victoriametrics-datasource"
+  GF_INSTALL_PLUGINS: "https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.13.0/victoriametrics-metrics-datasource-v0.13.0.zip;victoriametrics-metrics-datasource"
 ```
 
 Option 2. Using Grafana plugins section in `values.yaml`:
 
 ``` yaml
 plugins:
-  - https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.3/victoriametrics-datasource-v0.10.3.zip;victoriametrics-datasource
+  - https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.13.0/victoriametrics-metrics-datasource-v0.13.0.zip;victoriametrics-metrics-datasource
 ```
 
 Option 3. Using init container:
@@ -179,7 +179,7 @@ extraInitContainers:
        set -ex
        mkdir -p /var/lib/grafana/plugins/
        ver=$(curl -s -L https://api.github.com/repos/VictoriaMetrics/victoriametrics-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
-       curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
+       curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-metrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
        tar -xf /var/lib/grafana/plugins/vm-plugin.tar.gz -C /var/lib/grafana/plugins/
        rm /var/lib/grafana/plugins/vm-plugin.tar.gz
     volumeMounts:
@@ -239,7 +239,7 @@ spec:
                   set -ex
                   mkdir -p /var/lib/grafana/plugins/
                   ver=$(curl -s https://api.github.com/repos/VictoriaMetrics/victoriametrics-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
-                  curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
+                  curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-metrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
                   tar -xf /var/lib/grafana/plugins/vm-plugin.tar.gz -C /var/lib/grafana/plugins/
                   rm /var/lib/grafana/plugins/vm-plugin.tar.gz
               volumeMounts:
@@ -247,7 +247,7 @@ spec:
                   mountPath: /var/lib/grafana
   config:
     plugins:
-      allow_loading_unsigned_plugins: victoriametrics-datasource
+      allow_loading_unsigned_plugins: victoriametrics-metrics-datasource
 ```
 
 See [Grafana operator reference](https://grafana-operator.github.io/grafana-operator/docs/grafana/) to find more about  Grafana operator.
@@ -259,7 +259,7 @@ This example uses init container to download and install plugin.
 
    ```sh
    ver=$(curl -s https://api.github.com/repos/VictoriaMetrics/victoriametrics-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
-   curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
+   curl -L https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/$ver/victoriametrics-metrics-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vm-plugin.tar.gz
    tar -xf /var/lib/grafana/plugins/vm-plugin.tar.gz -C /var/lib/grafana/plugins/
    rm /var/lib/grafana/plugins/vm-plugin.tar.gz
    ```
@@ -279,7 +279,7 @@ plugins = {{path to directory with plugin}}
 
 ``` ini
 [plugins]
-allow_loading_unsigned_plugins = victoriametrics-datasource
+allow_loading_unsigned_plugins = victoriametrics-metrics-datasource
 ```
 
 ### 2. Run the plugin

go.mod

@@ -1,6 +1,6 @@
 module github.com/VictoriaMetrics/VictoriaMetrics
 
-go 1.23.3
+go 1.23.5
 
 // This is needed in order to avoid vmbackup and vmrestore binary size increase by 20MB
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8008

lib/cgroup/cpu.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 
 	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 // AvailableCPUs returns the number of available CPU cores for the app.
@@ -44,6 +46,9 @@ func updateGOMAXPROCSToCPUQuota(cpuQuota float64) {
 	if gomaxprocs <= 0 {
 		gomaxprocs = 1
 	}
+	if cpuQuota > float64(gomaxprocs) {
+		logger.Warnf("rounding CPU quota %.1f to %d CPUs for performance reasons - see https://docs.victoriametrics.com/bestpractices/#kubernetes", cpuQuota, gomaxprocs)
+	}
 
 	numCPU := runtime.NumCPU()
 	if gomaxprocs > numCPU {

lib/logstorage/consts.go

@@ -29,10 +29,10 @@ const maxRowsPerBlock = 8 * 1024 * 1024
 // in excess memory usage during data ingestion and significant slowdown during query execution.
 const maxColumnsPerBlock = 2_000
 
-// MaxFieldNameSize is the maximum size in bytes for field name.
+// maxFieldNameSize is the maximum size in bytes for field name.
 //
-// Longer field names are truncated during data ingestion to MaxFieldNameSize length.
-const MaxFieldNameSize = 128
+// Log entries with longer field names are rejected during data ingestion.
+const maxFieldNameSize = 128
 
 // maxConstColumnValueSize is the maximum size in bytes for const column value.
 //

lib/logstorage/log_rows.go

@@ -65,15 +65,6 @@ func (sf *sortedFields) Swap(i, j int) {
 	a[i], a[j] = a[j], a[i]
 }
 
-// RowFormatter implementes fmt.Stringer for []Field aka a single log row
-type RowFormatter []Field
-
-// String returns user-readable representation for rf
-func (rf *RowFormatter) String() string {
-	result := MarshalFieldsToJSON(nil, *rf)
-	return string(result)
-}
-
 // Reset resets lr with all its settings.
 //
 // Call ResetKeepSettings() for resetting lr without resetting its settings.
@@ -142,23 +133,33 @@ func (lr *LogRows) NeedFlush() bool {
 // It is OK to modify the args after returning from the function,
 // since lr copies all the args to internal data.
 //
-// Field names longer than MaxFieldNameSize are automatically truncated to MaxFieldNameSize length.
-//
-// Log entries with too big number of fields are ignored.
-// Loo long log entries are ignored.
+// Log entries are dropped with the warning message in the following cases:
+// - if there are too many log fields
+// - if there are too long log field names
+// - if the total length of log entries is too long
 func (lr *LogRows) MustAdd(tenantID TenantID, timestamp int64, fields, streamFields []Field) {
+	// Verify that the log entry doesn't exceed limits.
 	if len(fields) > maxColumnsPerBlock {
-		fieldNames := make([]string, len(fields))
-		for i, f := range fields {
-			fieldNames[i] = f.Name
-		}
-		logger.Infof("ignoring log entry with too big number of fields, which exceeds %d; fieldNames=%q", maxColumnsPerBlock, fieldNames)
+		line := MarshalFieldsToJSON(nil, fields)
+		logger.Warnf("ignoring log entry with too big number of fields %d, since it exceeds the limit %d; "+
+			"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain ; log entry: %s", len(fields), maxColumnsPerBlock, line)
 		return
 	}
+	for i := range fields {
+		fieldName := fields[i].Name
+		if len(fieldName) > maxFieldNameSize {
+			line := MarshalFieldsToJSON(nil, fields)
+			logger.Warnf("ignoring log entry with too long field name %q, since its length (%d) exceeds the limit %d bytes; "+
+				"see https://docs.victoriametrics.com/victorialogs/faq/#what-is-the-maximum-supported-field-name-length ; log entry: %s",
+				fieldName, len(fieldName), maxFieldNameSize, line)
+			return
+		}
+	}
 	rowLen := uncompressedRowSizeBytes(fields)
 	if rowLen > maxUncompressedBlockSize {
-		logger.Infof("ignoring too long log record with the estimated size %d bytes, since it exceeds the limit %d; "+
-			"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have", rowLen, maxUncompressedBlockSize)
+		line := MarshalFieldsToJSON(nil, fields)
+		logger.Warnf("ignoring too long log entry with the estimated length of %d bytes, since it exceeds the limit %d bytes; "+
+			"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have ; log entry: %s", rowLen, maxUncompressedBlockSize, line)
 		return
 	}
 
@@ -248,9 +249,6 @@ func (lr *LogRows) addFieldsInternal(fields []Field, ignoreFields map[string]str
 		dstField := &fb[len(fb)-1]
 
 		fieldName := f.Name
-		if len(fieldName) > MaxFieldNameSize {
-			fieldName = fieldName[:MaxFieldNameSize]
-		}
 		if fieldName == "_msg" {
 			fieldName = ""
 			hasMsgField = true
@@ -267,20 +265,21 @@ func (lr *LogRows) addFieldsInternal(fields []Field, ignoreFields map[string]str
 func (lr *LogRows) GetRowString(idx int) string {
 	tf := TimeFormatter(lr.timestamps[idx])
 	streamTags := getStreamTagsString(lr.streamTagsCanonicals[idx])
-	var rf RowFormatter
-	rf = append(rf[:0], lr.rows[idx]...)
-	rf = append(rf, Field{
+	var fields []Field
+	fields = append(fields[:0], lr.rows[idx]...)
+	fields = append(fields, Field{
 		Name:  "_time",
 		Value: tf.String(),
 	})
-	rf = append(rf, Field{
+	fields = append(fields, Field{
 		Name:  "_stream",
 		Value: streamTags,
 	})
-	sort.Slice(rf, func(i, j int) bool {
-		return rf[i].Name < rf[j].Name
+	sort.Slice(fields, func(i, j int) bool {
+		return fields[i].Name < fields[j].Name
 	})
-	return rf.String()
+	line := MarshalFieldsToJSON(nil, fields)
+	return string(line)
 }
 
 // GetLogRows returns LogRows from the pool for the given streamFields.

lib/logstorage/partition.go

@@ -147,8 +147,8 @@ func (pt *partition) mustAddRows(lr *LogRows) {
 
 func (pt *partition) logNewStream(streamTagsCanonical []byte, fields []Field) {
 	streamTags := getStreamTagsString(streamTagsCanonical)
-	rf := RowFormatter(fields)
-	logger.Infof("partition %s: new stream %s for log entry %s", pt.path, streamTags, &rf)
+	line := MarshalFieldsToJSON(nil, fields)
+	logger.Infof("partition %s: new stream %s for log entry %s", pt.path, streamTags, line)
 }
 
 func (pt *partition) logIngestedRows(lr *LogRows) {

lib/logstorage/storage.go

@@ -544,22 +544,22 @@ func (s *Storage) MustAddRows(lr *LogRows) {
 	for i, ts := range lr.timestamps {
 		day := ts / nsecsPerDay
 		if day < minAllowedDay {
-			rf := RowFormatter(lr.rows[i])
+			line := MarshalFieldsToJSON(nil, lr.rows[i])
 			tsf := TimeFormatter(ts)
 			minAllowedTsf := TimeFormatter(minAllowedDay * nsecsPerDay)
 			tooSmallTimestampLogger.Warnf("skipping log entry with too small timestamp=%s; it must be bigger than %s according "+
 				"to the configured -retentionPeriod=%dd. See https://docs.victoriametrics.com/victorialogs/#retention ; "+
-				"log entry: %s", &tsf, &minAllowedTsf, durationToDays(s.retention), &rf)
+				"log entry: %s", &tsf, &minAllowedTsf, durationToDays(s.retention), line)
 			s.rowsDroppedTooSmallTimestamp.Add(1)
 			continue
 		}
 		if day > maxAllowedDay {
-			rf := RowFormatter(lr.rows[i])
+			line := MarshalFieldsToJSON(nil, lr.rows[i])
 			tsf := TimeFormatter(ts)
 			maxAllowedTsf := TimeFormatter(maxAllowedDay * nsecsPerDay)
 			tooBigTimestampLogger.Warnf("skipping log entry with too big timestamp=%s; it must be smaller than %s according "+
 				"to the configured -futureRetention=%dd; see https://docs.victoriametrics.com/victorialogs/#retention ; "+
-				"log entry: %s", &tsf, &maxAllowedTsf, durationToDays(s.futureRetention), &rf)
+				"log entry: %s", &tsf, &maxAllowedTsf, durationToDays(s.futureRetention), line)
 			s.rowsDroppedTooBigTimestamp.Add(1)
 			continue
 		}