mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-08-04 23:07:25 +03:00
Compare commits
4 Commits
fix-missin
...
issue-5914
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e54c3ecfb6 | ||
|
|
d98336e581 | ||
|
|
13ef9c3ba8 | ||
|
|
5a87c85093 |
@@ -311,6 +311,8 @@ var (
|
||||
|
||||
const (
|
||||
influxAddr = "influx-addr"
|
||||
influxVersion = "influx-version"
|
||||
influxToken = "influx-token"
|
||||
influxUser = "influx-user"
|
||||
influxPassword = "influx-password"
|
||||
influxDB = "influx-database"
|
||||
@@ -337,6 +339,19 @@ var (
|
||||
Value: "http://localhost:8086",
|
||||
Usage: "InfluxDB server addr",
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: influxVersion,
|
||||
Usage: "Major version of the source InfluxDB: 1 or 2.\n" +
|
||||
"InfluxDB 2.x is migrated via its InfluxDB 1.x compatibility API, which requires -influx-token " +
|
||||
"and a database/retention policy mapping (DBRP) pointing at the bucket to migrate.\n" +
|
||||
"See https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/",
|
||||
Value: 1,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: influxToken,
|
||||
Usage: "InfluxDB v2 API token. Requires -influx-version=2",
|
||||
EnvVars: []string{"INFLUX_TOKEN"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: influxUser,
|
||||
Usage: "InfluxDB user",
|
||||
@@ -353,9 +368,11 @@ var (
|
||||
Required: true,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: influxRetention,
|
||||
Usage: "InfluxDB retention policy",
|
||||
Value: "autogen",
|
||||
Name: influxRetention,
|
||||
Usage: "InfluxDB retention policy.\n" +
|
||||
"Defaults to 'autogen' for -influx-version=1.\n" +
|
||||
"For -influx-version=2 it is the retention policy of a DBRP mapping; if empty, " +
|
||||
"the default mapping of -influx-database is used",
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: influxChunkSize,
|
||||
|
||||
@@ -11,6 +11,27 @@ import (
|
||||
influx "github.com/influxdata/influxdb/client/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
// VersionV1 is InfluxDB 1.x, queried via its native /query endpoint.
|
||||
VersionV1 = 1
|
||||
|
||||
// VersionV2 is InfluxDB 2.x, queried via its InfluxDB 1.x compatibility
|
||||
// API. It authenticates with an API token instead of a password.
|
||||
//
|
||||
// See https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/
|
||||
VersionV2 = 2
|
||||
|
||||
// defaultV1CompatUser is sent as the username when migrating from InfluxDB 2.x
|
||||
// with an API token.
|
||||
//
|
||||
// The InfluxDB 1.x compatibility API requires a username whenever an API token
|
||||
// is used as the password, but the value itself is ignored.
|
||||
// See https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/
|
||||
defaultV1CompatUser = "vmctl"
|
||||
|
||||
defaultRetentionV1 = "autogen"
|
||||
)
|
||||
|
||||
// Client represents a wrapper over
|
||||
// influx HTTP client
|
||||
type Client struct {
|
||||
@@ -27,7 +48,9 @@ type Client struct {
|
||||
// Config contains fields required
|
||||
// for Client configuration
|
||||
type Config struct {
|
||||
Version int
|
||||
Addr string
|
||||
Token string
|
||||
Username string
|
||||
Password string
|
||||
Database string
|
||||
@@ -88,10 +111,18 @@ type LabelPair struct {
|
||||
// NewClient creates and returns influx client
|
||||
// configured with passed Config
|
||||
func NewClient(cfg Config) (*Client, error) {
|
||||
if err := cfg.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// InfluxDB 2.x authenticates with an API token passed as the password
|
||||
// of its InfluxDB 1.x compatibility API.
|
||||
username, password := resolveAuth(cfg.Username, cfg.Password, cfg.Token)
|
||||
|
||||
c := influx.HTTPConfig{
|
||||
Addr: cfg.Addr,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
Username: username,
|
||||
Password: password,
|
||||
TLSConfig: cfg.TLSConfig,
|
||||
}
|
||||
hc, err := influx.NewHTTPClient(c)
|
||||
@@ -99,6 +130,7 @@ func NewClient(cfg Config) (*Client, error) {
|
||||
return nil, fmt.Errorf("failed to establish conn: %w", err)
|
||||
}
|
||||
if _, _, err := hc.Ping(time.Second); err != nil {
|
||||
_ = hc.Close()
|
||||
return nil, fmt.Errorf("ping failed: %w", err)
|
||||
}
|
||||
|
||||
@@ -110,7 +142,7 @@ func NewClient(cfg Config) (*Client, error) {
|
||||
client := &Client{
|
||||
Client: hc,
|
||||
database: cfg.Database,
|
||||
retention: cfg.Retention,
|
||||
retention: resolveRetention(cfg.Version, cfg.Retention),
|
||||
chunkSize: chunkSize,
|
||||
filterTime: timeFilter(cfg.Filter.TimeStart, cfg.Filter.TimeEnd),
|
||||
filterSeries: cfg.Filter.Series,
|
||||
@@ -464,3 +496,52 @@ func (c *Client) do(q influx.Query) ([]queryValues, error) {
|
||||
}
|
||||
return parseResult(res.Results[0])
|
||||
}
|
||||
|
||||
// resolveAuth returns the credentials to authenticate with.
|
||||
//
|
||||
// InfluxDB 2.x is queried via its InfluxDB 1.x compatibility API, which accepts
|
||||
// an API token in place of the password. Therefore a non-empty token replaces
|
||||
// the password, and a placeholder username is substituted when none is given.
|
||||
func resolveAuth(username, password, token string) (string, string) {
|
||||
if token == "" {
|
||||
return username, password
|
||||
}
|
||||
if username == "" {
|
||||
username = defaultV1CompatUser
|
||||
}
|
||||
return username, token
|
||||
}
|
||||
|
||||
// resolveRetention returns the retention policy to query.
|
||||
//
|
||||
// In InfluxDB 1.x `autogen` is the retention policy created together with a
|
||||
// database, so it is a meaningful default. In InfluxDB 2.x the retention policy
|
||||
// is one half of a DBRP mapping and its name is arbitrary, so no default can be
|
||||
// assumed: an empty value makes InfluxDB use the default mapping of the
|
||||
// database instead of failing on a non-existent one.
|
||||
func resolveRetention(version int, retention string) string {
|
||||
if retention == "" && version == VersionV1 {
|
||||
return defaultRetentionV1
|
||||
}
|
||||
return retention
|
||||
}
|
||||
|
||||
// validate checks that the configuration is self-consistent.
|
||||
func (cfg *Config) validate() error {
|
||||
if cfg.Version != VersionV1 && cfg.Version != VersionV2 {
|
||||
return fmt.Errorf("unsupported InfluxDB version %d; supported versions are %d and %d",
|
||||
cfg.Version, VersionV1, VersionV2)
|
||||
}
|
||||
if cfg.Version == VersionV2 && cfg.Token == "" {
|
||||
return fmt.Errorf("-influx-token is required for InfluxDB v2")
|
||||
}
|
||||
if cfg.Version == VersionV1 && cfg.Token != "" {
|
||||
return fmt.Errorf("-influx-token is only supported for InfluxDB v2; pass -influx-version=2 to use it")
|
||||
}
|
||||
// The database is the `db` parameter of the query API. For InfluxDB 2.x
|
||||
// it is the database name of a DBRP mapping, which points at a bucket.
|
||||
if cfg.Database == "" {
|
||||
return fmt.Errorf("-influx-database cannot be empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
package influx
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchQuery(t *testing.T) {
|
||||
f := func(s *Series, timeFilter, resultExpected string) {
|
||||
@@ -126,3 +132,255 @@ func TestGetSeriesCommand(t *testing.T) {
|
||||
f("from cpu where arch='x86'", "time >= '2020-01-01T20:07:00Z'", "show series from cpu where arch='x86' AND time >= '2020-01-01T20:07:00Z'")
|
||||
f("from cpu where arch='x86' AND hostname='host_2753'", "time >= '2020-01-01T20:07:00Z'", "show series from cpu where arch='x86' AND hostname='host_2753' AND time >= '2020-01-01T20:07:00Z'")
|
||||
}
|
||||
|
||||
func TestResolveAuth(t *testing.T) {
|
||||
f := func(username, password, token, userExpected, passExpected string) {
|
||||
t.Helper()
|
||||
|
||||
user, pass := resolveAuth(username, password, token)
|
||||
if user != userExpected || pass != passExpected {
|
||||
t.Fatalf("unexpected credentials for (username=%q, password=%q, token=%q)\ngot\n(%q, %q)\nwant\n(%q, %q)",
|
||||
username, password, token, user, pass, userExpected, passExpected)
|
||||
}
|
||||
}
|
||||
|
||||
// InfluxDB 1.x: no token, credentials are passed through unchanged.
|
||||
f("", "", "", "", "")
|
||||
f("user", "pass", "", "user", "pass")
|
||||
|
||||
// InfluxDB 2.x: the API token is sent as the password. The username is
|
||||
// required by the v1 compatibility API but may be any value, so a
|
||||
// placeholder is used when the caller did not provide one.
|
||||
f("", "", "token", defaultV1CompatUser, "token")
|
||||
|
||||
// An explicitly provided username is preserved.
|
||||
f("myuser", "", "token", "myuser", "token")
|
||||
|
||||
// The token takes precedence over a password.
|
||||
f("user", "pass", "token", "user", "token")
|
||||
}
|
||||
|
||||
func TestConfigValidateSuccess(t *testing.T) {
|
||||
f := func(cfg Config) {
|
||||
t.Helper()
|
||||
|
||||
if err := cfg.validate(); err != nil {
|
||||
t.Fatalf("unexpected error for config %+v: %s", cfg, err)
|
||||
}
|
||||
}
|
||||
|
||||
// InfluxDB 1.x with and without credentials.
|
||||
f(Config{Version: 1, Database: "mydb"})
|
||||
f(Config{Version: 1, Database: "mydb", Username: "user", Password: "pass"})
|
||||
|
||||
// InfluxDB 2.x requires a token.
|
||||
f(Config{Version: 2, Database: "mydb", Token: "my-token"})
|
||||
}
|
||||
|
||||
func TestConfigValidateFailure(t *testing.T) {
|
||||
f := func(cfg Config, errStrExpected string) {
|
||||
t.Helper()
|
||||
|
||||
err := cfg.validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error for config %+v", cfg)
|
||||
}
|
||||
if !strings.Contains(err.Error(), errStrExpected) {
|
||||
t.Fatalf("unexpected error for config %+v\ngot\n%s\nwant it to contain\n%s", cfg, err, errStrExpected)
|
||||
}
|
||||
}
|
||||
|
||||
// unsupported versions
|
||||
f(Config{Version: 0, Database: "mydb"}, "unsupported InfluxDB version")
|
||||
f(Config{Version: 3, Database: "mydb"}, "unsupported InfluxDB version")
|
||||
|
||||
// InfluxDB 2.x without a token cannot authenticate
|
||||
f(Config{Version: 2, Database: "mydb"}, "influx-token")
|
||||
|
||||
// a token is meaningless without opting into v2
|
||||
f(Config{Version: 1, Database: "mydb", Token: "my-token"}, "influx-version=2")
|
||||
|
||||
// the database is mandatory for both versions, since it is the `db`
|
||||
// parameter of the query API
|
||||
f(Config{Version: 1}, "influx-database")
|
||||
f(Config{Version: 2, Token: "my-token"}, "influx-database")
|
||||
}
|
||||
|
||||
func TestResolveRetention(t *testing.T) {
|
||||
f := func(version int, retention, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
result := resolveRetention(version, retention)
|
||||
if result != resultExpected {
|
||||
t.Fatalf("unexpected retention policy for (version=%d, retention=%q)\ngot\n%q\nwant\n%q",
|
||||
version, retention, result, resultExpected)
|
||||
}
|
||||
}
|
||||
|
||||
// InfluxDB 1.x keeps its historical default.
|
||||
f(VersionV1, "", "autogen")
|
||||
f(VersionV1, "all_data", "all_data")
|
||||
|
||||
// For InfluxDB 2.x the retention policy is part of a DBRP mapping whose
|
||||
// name is arbitrary, so no default may be assumed: an empty value lets
|
||||
// InfluxDB pick the default mapping of the database.
|
||||
f(VersionV2, "", "")
|
||||
f(VersionV2, "myrp", "myrp")
|
||||
}
|
||||
|
||||
// TestNewClientValidatesConfig ensures an invalid configuration is rejected
|
||||
// before any connection to InfluxDB is attempted.
|
||||
func TestNewClientValidatesConfig(t *testing.T) {
|
||||
f := func(cfg Config, errStrExpected string) {
|
||||
t.Helper()
|
||||
|
||||
cfg.Addr = "http://127.0.0.1:1"
|
||||
|
||||
c, err := NewClient(cfg)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error for config %+v", cfg)
|
||||
}
|
||||
if c != nil {
|
||||
t.Fatalf("expecting nil client for config %+v", cfg)
|
||||
}
|
||||
if !strings.Contains(err.Error(), errStrExpected) {
|
||||
t.Fatalf("unexpected error for config %+v\ngot\n%s\nwant it to contain\n%s", cfg, err, errStrExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f(Config{Version: 0, Database: "mydb"}, "unsupported InfluxDB version")
|
||||
f(Config{Version: 2, Database: "mydb"}, "influx-token")
|
||||
f(Config{Version: 1, Database: "mydb", Token: "my-token"}, "influx-version=2")
|
||||
}
|
||||
|
||||
// TestNewClientTokenAuth ensures the API token is used as the password when
|
||||
// migrating from InfluxDB 2.x.
|
||||
func TestNewClientTokenAuth(t *testing.T) {
|
||||
f := func(cfg Config, userExpected, passExpected string) {
|
||||
t.Helper()
|
||||
|
||||
user, pass := resolveAuth(cfg.Username, cfg.Password, cfg.Token)
|
||||
if user != userExpected || pass != passExpected {
|
||||
t.Fatalf("unexpected credentials for config %+v\ngot\n(%q, %q)\nwant\n(%q, %q)",
|
||||
cfg, user, pass, userExpected, passExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f(Config{Version: 2, Database: "mydb", Token: "my-token"}, defaultV1CompatUser, "my-token")
|
||||
f(Config{Version: 1, Database: "mydb", Username: "user", Password: "pass"}, "user", "pass")
|
||||
}
|
||||
|
||||
func TestQueryRequestAuthAndRetention(t *testing.T) {
|
||||
f := func(cfg Config, userExpected, passExpected, rpExpected string) {
|
||||
t.Helper()
|
||||
|
||||
var lastQuery queryRequest
|
||||
s := newTestServer(t, &lastQuery)
|
||||
defer s.Close()
|
||||
|
||||
cfg.Addr = s.URL
|
||||
c, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error creating client: %s", err)
|
||||
}
|
||||
|
||||
if _, err := c.fieldsByMeasurement(); err != nil {
|
||||
t.Fatalf("unexpected error querying field keys: %s", err)
|
||||
}
|
||||
|
||||
got := lastQuery.get()
|
||||
if !got.seen {
|
||||
t.Fatalf("the client did not issue a /query request")
|
||||
}
|
||||
if !got.hasAuth {
|
||||
t.Fatalf("expecting basic auth to be set on the request")
|
||||
}
|
||||
if got.user != userExpected || got.password != passExpected {
|
||||
t.Fatalf("unexpected credentials on the wire\ngot\n(%q, %q)\nwant\n(%q, %q)",
|
||||
got.user, got.password, userExpected, passExpected)
|
||||
}
|
||||
if got.rp != rpExpected {
|
||||
t.Fatalf("unexpected rp parameter\ngot\n%q\nwant\n%q", got.rp, rpExpected)
|
||||
}
|
||||
if got.db != cfg.Database {
|
||||
t.Fatalf("unexpected db parameter\ngot\n%q\nwant\n%q", got.db, cfg.Database)
|
||||
}
|
||||
}
|
||||
|
||||
// InfluxDB 1.x: credentials are passed through and `autogen` is the
|
||||
// default retention policy.
|
||||
f(Config{Version: VersionV1, Database: "mydb", Username: "user", Password: "pass"},
|
||||
"user", "pass", "autogen")
|
||||
|
||||
// An explicit retention policy is honoured.
|
||||
f(Config{Version: VersionV1, Database: "mydb", Username: "user", Password: "pass", Retention: "all_data"},
|
||||
"user", "pass", "all_data")
|
||||
|
||||
// InfluxDB 2.x: the API token is sent as the password with a placeholder
|
||||
// username, and no retention policy is assumed so that the default DBRP
|
||||
// mapping of the database applies.
|
||||
f(Config{Version: VersionV2, Database: "mydb", Token: "my-token"},
|
||||
defaultV1CompatUser, "my-token", "")
|
||||
|
||||
// An explicit retention policy names the DBRP mapping to query.
|
||||
f(Config{Version: VersionV2, Database: "mydb", Token: "my-token", Retention: "myrp"},
|
||||
defaultV1CompatUser, "my-token", "myrp")
|
||||
}
|
||||
|
||||
// queryRequest holds what a /query request carried on the wire. Only the values
|
||||
// under test are kept, so nothing of the *http.Request outlives the handler.
|
||||
type queryRequest struct {
|
||||
mu sync.Mutex
|
||||
|
||||
seen bool
|
||||
user string
|
||||
password string
|
||||
hasAuth bool
|
||||
rp string
|
||||
db string
|
||||
}
|
||||
|
||||
func (q *queryRequest) set(r *http.Request) {
|
||||
user, password, hasAuth := r.BasicAuth()
|
||||
params := r.URL.Query()
|
||||
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
q.seen = true
|
||||
q.user, q.password, q.hasAuth = user, password, hasAuth
|
||||
q.rp = params.Get("rp")
|
||||
q.db = params.Get("db")
|
||||
}
|
||||
|
||||
func (q *queryRequest) get() queryRequest {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
return queryRequest{
|
||||
seen: q.seen,
|
||||
user: q.user,
|
||||
password: q.password,
|
||||
hasAuth: q.hasAuth,
|
||||
rp: q.rp,
|
||||
db: q.db,
|
||||
}
|
||||
}
|
||||
|
||||
func newTestServer(t *testing.T, lastQuery *queryRequest) *httptest.Server {
|
||||
t.Helper()
|
||||
|
||||
const fieldKeysResponse = `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["fieldKey","fieldType"],"values":[["value","float"]]}]}]}`
|
||||
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Influxdb-Version", "test")
|
||||
switch r.URL.Path {
|
||||
case "/ping":
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
case "/query":
|
||||
lastQuery.set(r)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(fieldKeysResponse))
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -141,7 +141,9 @@ func main() {
|
||||
}
|
||||
|
||||
iCfg := influx.Config{
|
||||
Version: c.Int(influxVersion),
|
||||
Addr: c.String(influxAddr),
|
||||
Token: c.String(influxToken),
|
||||
Username: c.String(influxUser),
|
||||
Password: c.String(influxPassword),
|
||||
Database: c.String(influxDB),
|
||||
|
||||
File diff suppressed because one or more lines are too long
197
app/vmselect/vmui/assets/index-D5egN2id.js
Normal file
197
app/vmselect/vmui/assets/index-D5egN2id.js
Normal file
File diff suppressed because one or more lines are too long
@@ -37,7 +37,7 @@
|
||||
<meta property="og:title" content="UI for VictoriaMetrics">
|
||||
<meta property="og:url" content="https://victoriametrics.com/">
|
||||
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
|
||||
<script type="module" crossorigin src="./assets/index-B1dXK3k7.js"></script>
|
||||
<script type="module" crossorigin src="./assets/index-D5egN2id.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-CNC7AqOf.js">
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-DwJYpOdw.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
|
||||
|
||||
342
apptest/tests/influx_server_test.go
Normal file
342
apptest/tests/influx_server_test.go
Normal file
@@ -0,0 +1,342 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// influxPoint is a single field value at a single timestamp of a single series,
|
||||
// as served by the mock InfluxDB server.
|
||||
type influxPoint struct {
|
||||
Measurement string
|
||||
Tags map[string]string
|
||||
Field string
|
||||
FieldType string
|
||||
Timestamp int64 // unix seconds
|
||||
Value any // float64, int64, bool or string
|
||||
}
|
||||
|
||||
// influxRequest is a /query request recorded by the mock server, used to assert
|
||||
// how vmctl authenticates and which database and retention policy it asks for.
|
||||
type influxRequest struct {
|
||||
Query url.Values
|
||||
User string
|
||||
Password string
|
||||
HasAuth bool
|
||||
}
|
||||
|
||||
// influxMockServer implements the subset of the InfluxDB 1.x query API used by
|
||||
// `vmctl influx`: /ping plus /query serving `SHOW FIELD KEYS`, `SHOW TAG KEYS`,
|
||||
// `SHOW SERIES` and `SELECT`.
|
||||
//
|
||||
// InfluxDB 2.x is migrated through the very same API - its 1.x compatibility
|
||||
// endpoint - so one mock covers both -influx-version=1 and -influx-version=2.
|
||||
type influxMockServer struct {
|
||||
server *httptest.Server
|
||||
points []influxPoint
|
||||
|
||||
mu sync.Mutex
|
||||
requests []influxRequest
|
||||
}
|
||||
|
||||
// newInfluxMockServer starts an httptest server serving the given points.
|
||||
func newInfluxMockServer(t *testing.T, points []influxPoint) *influxMockServer {
|
||||
t.Helper()
|
||||
s := &influxMockServer{points: points}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/ping", s.handlePing)
|
||||
mux.HandleFunc("/query", s.handleQuery)
|
||||
s.server = httptest.NewServer(mux)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *influxMockServer) close() { s.server.Close() }
|
||||
|
||||
func (s *influxMockServer) httpAddr() string { return s.server.URL }
|
||||
|
||||
// recordQuery stores a /query request for later assertions.
|
||||
//
|
||||
// Every request is kept, not just the last one: vmctl issues several kinds of
|
||||
// query - `SHOW FIELD KEYS` unchunked, `SHOW TAG KEYS` and `SHOW SERIES`
|
||||
// chunked, and one `SELECT` per series - and the assertions verify that all of
|
||||
// them authenticate and address the database the same way. The assertions
|
||||
// cannot live in this handler because it runs on a server goroutine, where
|
||||
// t.Fatalf must not be called.
|
||||
func (s *influxMockServer) recordQuery(r *http.Request) {
|
||||
user, pass, ok := r.BasicAuth()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.requests = append(s.requests, influxRequest{
|
||||
Query: r.URL.Query(),
|
||||
User: user,
|
||||
Password: pass,
|
||||
HasAuth: ok,
|
||||
})
|
||||
}
|
||||
|
||||
// queryRequests returns a copy of the recorded /query requests.
|
||||
func (s *influxMockServer) queryRequests() []influxRequest {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return slices.Clone(s.requests)
|
||||
}
|
||||
|
||||
// handlePing must answer 204: the client library treats any other status as a
|
||||
// failed ping, and vmctl pings before querying.
|
||||
//
|
||||
// X-Influxdb-Version is set because real InfluxDB sets it. The client returns
|
||||
// the value from Ping, but vmctl discards it, so it is not asserted anywhere and
|
||||
// its content is arbitrary.
|
||||
//
|
||||
// The request is not recorded: /ping carries no db or rp to assert.
|
||||
func (s *influxMockServer) handlePing(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("X-Influxdb-Version", "1.8.0-mock")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// influxResponse mirrors the InfluxDB 1.x JSON query response.
|
||||
type influxResponse struct {
|
||||
Results []influxResult `json:"results"`
|
||||
}
|
||||
|
||||
type influxResult struct {
|
||||
StatementID int `json:"statement_id"`
|
||||
Series []influxRow `json:"series,omitempty"`
|
||||
Err string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type influxRow struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Columns []string `json:"columns"`
|
||||
Values [][]any `json:"values,omitempty"`
|
||||
}
|
||||
|
||||
// parseSelect splits a SELECT built by vmctl into its field, measurement and
|
||||
// WHERE clause. The statement shape is fixed by Series.fetchQuery:
|
||||
//
|
||||
// select "value" from "cpu" where "host"::tag='h1' and "env"::tag=''
|
||||
//
|
||||
// Identifiers containing a double quote are not supported, which is fine
|
||||
// because no test fixture uses one.
|
||||
func parseSelect(q string) (field, measurement, where string, err error) {
|
||||
rest, ok := strings.CutPrefix(q, `select "`)
|
||||
if !ok {
|
||||
return "", "", "", fmt.Errorf("cannot parse select statement: %s", q)
|
||||
}
|
||||
field, rest, ok = strings.Cut(rest, `" from "`)
|
||||
if !ok {
|
||||
return "", "", "", fmt.Errorf("cannot parse measurement in: %s", q)
|
||||
}
|
||||
measurement, rest, ok = strings.Cut(rest, `"`)
|
||||
if !ok {
|
||||
return "", "", "", fmt.Errorf("unterminated measurement in: %s", q)
|
||||
}
|
||||
return field, measurement, strings.TrimPrefix(rest, " where "), nil
|
||||
}
|
||||
|
||||
// handleQuery serves the InfluxQL statements vmctl issues. Both the plain and
|
||||
// the chunked code paths of the client accept a single JSON response object,
|
||||
// so one implementation covers both.
|
||||
func (s *influxMockServer) handleQuery(w http.ResponseWriter, r *http.Request) {
|
||||
s.recordQuery(r)
|
||||
|
||||
q := strings.TrimSpace(r.URL.Query().Get("q"))
|
||||
lower := strings.ToLower(q)
|
||||
|
||||
// Content-Type is load-bearing: the client rejects any response that is not
|
||||
// application/json. X-Influxdb-Version only matters for 5xx replies, where
|
||||
// the client uses its absence to report a downstream proxy error instead of
|
||||
// an InfluxDB one; it is set here to mirror real InfluxDB.
|
||||
w.Header().Set("X-Influxdb-Version", "1.8.0-mock")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
var resp influxResponse
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "show field keys"):
|
||||
resp = influxResponse{Results: []influxResult{{Series: s.fieldKeys()}}}
|
||||
case strings.HasPrefix(lower, "show tag keys"):
|
||||
resp = influxResponse{Results: []influxResult{{Series: s.tagKeys()}}}
|
||||
case strings.HasPrefix(lower, "show series"):
|
||||
resp = influxResponse{Results: []influxResult{{Series: s.series()}}}
|
||||
case strings.HasPrefix(lower, "select"):
|
||||
rows, err := s.selectRows(q)
|
||||
if err != nil {
|
||||
resp = influxResponse{Results: []influxResult{{Err: err.Error()}}}
|
||||
break
|
||||
}
|
||||
resp = influxResponse{Results: []influxResult{{Series: rows}}}
|
||||
default:
|
||||
resp = influxResponse{Results: []influxResult{{Err: fmt.Sprintf("unsupported query: %s", q)}}}
|
||||
}
|
||||
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
// fieldKeys serves `SHOW FIELD KEYS`: one row per measurement listing every
|
||||
// field key together with its type. vmctl uses the type to skip string fields.
|
||||
//
|
||||
// Rows are built by walking the points in order, so the output is deterministic
|
||||
// without sorting.
|
||||
func (s *influxMockServer) fieldKeys() []influxRow {
|
||||
rows := make(map[string]*influxRow)
|
||||
var order []string
|
||||
seen := make(map[string]struct{})
|
||||
for _, p := range s.points {
|
||||
row, ok := rows[p.Measurement]
|
||||
if !ok {
|
||||
row = &influxRow{Name: p.Measurement, Columns: []string{"fieldKey", "fieldType"}}
|
||||
rows[p.Measurement] = row
|
||||
order = append(order, p.Measurement)
|
||||
}
|
||||
key := p.Measurement + "\x00" + p.Field
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
row.Values = append(row.Values, []any{p.Field, p.FieldType})
|
||||
}
|
||||
out := make([]influxRow, 0, len(order))
|
||||
for _, m := range order {
|
||||
out = append(out, *rows[m])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tagKeys serves `SHOW TAG KEYS`: one row per measurement listing its tag keys.
|
||||
//
|
||||
// Tag keys come from a map, so each row is sorted to keep the output stable.
|
||||
func (s *influxMockServer) tagKeys() []influxRow {
|
||||
rows := make(map[string]*influxRow)
|
||||
var order []string
|
||||
seen := make(map[string]struct{})
|
||||
for _, p := range s.points {
|
||||
row, ok := rows[p.Measurement]
|
||||
if !ok {
|
||||
row = &influxRow{Name: p.Measurement, Columns: []string{"tagKey"}}
|
||||
rows[p.Measurement] = row
|
||||
order = append(order, p.Measurement)
|
||||
}
|
||||
for k := range p.Tags {
|
||||
key := p.Measurement + "\x00" + k
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
row.Values = append(row.Values, []any{k})
|
||||
}
|
||||
}
|
||||
out := make([]influxRow, 0, len(order))
|
||||
for _, m := range order {
|
||||
row := *rows[m]
|
||||
sort.Slice(row.Values, func(a, b int) bool {
|
||||
return row.Values[a][0].(string) < row.Values[b][0].(string)
|
||||
})
|
||||
out = append(out, row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// series serves `SHOW SERIES`: a single row of series keys in the
|
||||
// `measurement,tag=value,...` form.
|
||||
func (s *influxMockServer) series() []influxRow {
|
||||
seen := make(map[string]struct{})
|
||||
var keys []string
|
||||
for _, p := range s.points {
|
||||
key := influxSeriesKey(p.Measurement, p.Tags)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
row := influxRow{Columns: []string{"key"}}
|
||||
for _, k := range keys {
|
||||
row.Values = append(row.Values, []any{k})
|
||||
}
|
||||
return []influxRow{row}
|
||||
}
|
||||
|
||||
// selectRows serves the per-series SELECT issued for every measurement/field
|
||||
// combination, returning the `time` and field columns.
|
||||
func (s *influxMockServer) selectRows(q string) ([]influxRow, error) {
|
||||
field, measurement, where, err := parseSelect(q)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Tags the series must carry. An empty value means the tag must be absent,
|
||||
// which is how vmctl addresses series that lack a tag of the measurement.
|
||||
wantTags := make(map[string]string)
|
||||
for _, cond := range strings.Split(where, " and ") {
|
||||
// Conditions that are not tag comparisons - such as the time filter
|
||||
// added by -influx-filter-time-start/-end - are not series selectors.
|
||||
name, value, ok := strings.Cut(cond, "::tag=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
wantTags[strings.Trim(name, `"`)] = strings.Trim(value, `'`)
|
||||
}
|
||||
|
||||
row := influxRow{Name: measurement, Columns: []string{"time", field}}
|
||||
for _, p := range s.points {
|
||||
if p.Measurement != measurement || p.Field != field {
|
||||
continue
|
||||
}
|
||||
if !influxTagsMatch(p.Tags, wantTags) {
|
||||
continue
|
||||
}
|
||||
ts := time.Unix(p.Timestamp, 0).UTC().Format(time.RFC3339)
|
||||
row.Values = append(row.Values, []any{ts, p.Value})
|
||||
}
|
||||
if len(row.Values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return []influxRow{row}, nil
|
||||
}
|
||||
|
||||
// influxTagsMatch reports whether the series tags satisfy the conditions of a
|
||||
// SELECT built by vmctl. Every requested tag must match exactly; a requested
|
||||
// empty value requires the tag to be absent from the series.
|
||||
func influxTagsMatch(got, want map[string]string) bool {
|
||||
for k, v := range want {
|
||||
if v == "" {
|
||||
if _, ok := got[k]; ok {
|
||||
return false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if got[k] != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// Every tag of the series must be constrained, otherwise the SELECT would
|
||||
// address more than one series.
|
||||
for k := range got {
|
||||
if _, ok := want[k]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// influxSeriesKey builds the `measurement,tag=value,...` key used by SHOW SERIES.
|
||||
//
|
||||
// The tags must be sorted: the key doubles as the deduplication key, so an
|
||||
// unstable order would report the same series more than once. tagsKey already
|
||||
// sorts, so it is reused here.
|
||||
func influxSeriesKey(measurement string, tags map[string]string) string {
|
||||
if len(tags) == 0 {
|
||||
return measurement
|
||||
}
|
||||
return measurement + "," + tagsKey(tags)
|
||||
}
|
||||
@@ -26,20 +26,33 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
|
||||
defer tc.Stop()
|
||||
|
||||
testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) apptest.PrometheusWriteQuerier {
|
||||
vmstorage := tc.MustStartVmstorage("vmstorage-"+name, []string{
|
||||
"-storageDataPath=" + tc.Dir() + "/vmstorage",
|
||||
// Using static ports for vmstorage because random ports may cause
|
||||
// changes in how data is sharded.
|
||||
vmstorage1 := tc.MustStartVmstorage("vmstorage1-"+name, []string{
|
||||
"-storageDataPath=" + tc.Dir() + "/vmstorage1",
|
||||
"-retentionPeriod=100y",
|
||||
"-httpListenAddr=127.0.0.1:61001",
|
||||
"-vminsertAddr=127.0.0.1:61002",
|
||||
"-vmselectAddr=127.0.0.1:61003",
|
||||
fmt.Sprintf("-disablePerDayIndex=%t", disablePerDayIndex),
|
||||
})
|
||||
vmstorage2 := tc.MustStartVmstorage("vmstorage2-"+name, []string{
|
||||
"-storageDataPath=" + tc.Dir() + "/vmstorage2",
|
||||
"-retentionPeriod=100y",
|
||||
"-httpListenAddr=127.0.0.1:62001",
|
||||
"-vminsertAddr=127.0.0.1:62002",
|
||||
"-vmselectAddr=127.0.0.1:62003",
|
||||
fmt.Sprintf("-disablePerDayIndex=%t", disablePerDayIndex),
|
||||
})
|
||||
vminsert := tc.MustStartVminsert("vminsert-"+name, []string{
|
||||
"-storageNode=" + vmstorage.VminsertAddr(),
|
||||
"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
|
||||
})
|
||||
vmselect := tc.MustStartVmselect("vmselect"+name, []string{
|
||||
"-storageNode=" + vmstorage.VmselectAddr(),
|
||||
"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
|
||||
"-search.maxStalenessInterval=1m",
|
||||
})
|
||||
return &apptest.Vmcluster{
|
||||
Vmstorages: []*apptest.Vmstorage{vmstorage},
|
||||
Vmstorages: []*apptest.Vmstorage{vmstorage1, vmstorage2},
|
||||
Vminsert: vminsert,
|
||||
Vmselect: vmselect,
|
||||
}
|
||||
|
||||
272
apptest/tests/vmctl_influx_migration_test.go
Normal file
272
apptest/tests/vmctl_influx_migration_test.go
Normal file
@@ -0,0 +1,272 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
)
|
||||
|
||||
const (
|
||||
influxTestDatabase = "testdb"
|
||||
influxTestToken = "my-secret-token"
|
||||
// defaultV1CompatUser mirrors the username vmctl sends when authenticating
|
||||
// to InfluxDB 2.x with an API token. The 1.x compatibility API requires a
|
||||
// username but ignores its value.
|
||||
influxV1CompatUser = "vmctl"
|
||||
)
|
||||
|
||||
func TestSingleVmctlInfluxV2Migration(t *testing.T) {
|
||||
fs.MustRemoveDir(t.Name())
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
vmsingleDst := tc.MustStartDefaultVmsingle()
|
||||
vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
|
||||
|
||||
baseTS := time.Now().Add(-2 * time.Hour).Truncate(time.Minute).Unix()
|
||||
points := newInfluxTestPoints(baseTS)
|
||||
|
||||
influx := newInfluxMockServer(t, points)
|
||||
defer influx.close()
|
||||
|
||||
vmctlFlags := []string{
|
||||
`influx`,
|
||||
`--influx-version=2`,
|
||||
`--influx-addr=` + influx.httpAddr(),
|
||||
`--influx-token=` + influxTestToken,
|
||||
`--influx-database=` + influxTestDatabase,
|
||||
`--vm-addr=` + vmAddr,
|
||||
`--disable-progress-bar=true`,
|
||||
`-s`,
|
||||
}
|
||||
|
||||
testVmctlInfluxMigration(tc, vmsingleDst, vmctlFlags, points, baseTS)
|
||||
|
||||
assertInfluxRequests(t, influx, influxV1CompatUser, influxTestToken, "")
|
||||
}
|
||||
|
||||
func TestSingleVmctlInfluxV1Migration(t *testing.T) {
|
||||
fs.MustRemoveDir(t.Name())
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
vmsingleDst := tc.MustStartDefaultVmsingle()
|
||||
vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
|
||||
|
||||
baseTS := time.Now().Add(-2 * time.Hour).Truncate(time.Minute).Unix()
|
||||
points := newInfluxTestPoints(baseTS)
|
||||
|
||||
influx := newInfluxMockServer(t, points)
|
||||
defer influx.close()
|
||||
|
||||
vmctlFlags := []string{
|
||||
`influx`,
|
||||
`--influx-addr=` + influx.httpAddr(),
|
||||
`--influx-user=user`,
|
||||
`--influx-password=pass`,
|
||||
`--influx-database=` + influxTestDatabase,
|
||||
`--vm-addr=` + vmAddr,
|
||||
`--disable-progress-bar=true`,
|
||||
`-s`,
|
||||
}
|
||||
|
||||
testVmctlInfluxMigration(tc, vmsingleDst, vmctlFlags, points, baseTS)
|
||||
assertInfluxRequests(t, influx, "user", "pass", "autogen")
|
||||
}
|
||||
|
||||
func TestClusterVmctlInfluxV2Migration(t *testing.T) {
|
||||
fs.MustRemoveDir(t.Name())
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
cluster := tc.MustStartDefaultCluster()
|
||||
vmAddr := fmt.Sprintf("http://%s/", cluster.Vminsert.HTTPAddr())
|
||||
|
||||
baseTS := time.Now().Add(-2 * time.Hour).Truncate(time.Minute).Unix()
|
||||
points := newInfluxTestPoints(baseTS)
|
||||
|
||||
influx := newInfluxMockServer(t, points)
|
||||
defer influx.close()
|
||||
|
||||
vmctlFlags := []string{
|
||||
`influx`,
|
||||
`--influx-version=2`,
|
||||
`--influx-addr=` + influx.httpAddr(),
|
||||
`--influx-token=` + influxTestToken,
|
||||
`--influx-database=` + influxTestDatabase,
|
||||
`--vm-addr=` + vmAddr,
|
||||
`--vm-account-id=0`,
|
||||
`--disable-progress-bar=true`,
|
||||
`-s`,
|
||||
}
|
||||
|
||||
testVmctlInfluxMigration(tc, cluster, vmctlFlags, points, baseTS)
|
||||
assertInfluxRequests(t, influx, influxV1CompatUser, influxTestToken, "")
|
||||
}
|
||||
|
||||
func testVmctlInfluxMigration(
|
||||
tc *apptest.TestCase,
|
||||
queries apptest.PrometheusWriteQuerier,
|
||||
vmctlFlags []string,
|
||||
points []influxPoint,
|
||||
baseTS int64,
|
||||
) {
|
||||
t := tc.T()
|
||||
t.Helper()
|
||||
|
||||
queryStart := time.Unix(baseTS-3600, 0).UTC().Format(time.RFC3339)
|
||||
queryEnd := time.Unix(baseTS+7200, 0).UTC().Format(time.RFC3339)
|
||||
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
|
||||
// Nothing is stored before the migration runs.
|
||||
got := queries.PrometheusAPIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
|
||||
Step: "5m",
|
||||
Time: queryStart,
|
||||
})
|
||||
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
|
||||
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
|
||||
t.Errorf("unexpected response before migration (-want, +got):\n%s", diff)
|
||||
}
|
||||
|
||||
tc.MustStartVmctl("vmctl", vmctlFlags)
|
||||
queries.ForceFlush(t)
|
||||
|
||||
tc.Assert(&apptest.AssertOptions{
|
||||
Retries: 300,
|
||||
Msg: `unexpected metrics migrated from influx`,
|
||||
Got: func() any {
|
||||
r := queries.PrometheusAPIV1Export(t, `{__name__!=""}`, apptest.QueryOpts{
|
||||
Start: queryStart,
|
||||
End: queryEnd,
|
||||
})
|
||||
r.Sort()
|
||||
return r.Data.Result
|
||||
},
|
||||
Want: buildExpectedInfluxResult(t, points),
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// buildExpectedInfluxResult derives the expected VictoriaMetrics contents from
|
||||
// the same points the mock server serves: metric name is
|
||||
// `<measurement>_<field>`, tags become labels, the source database is added as
|
||||
// the `db` label, booleans become 1/0 and string fields are skipped.
|
||||
func buildExpectedInfluxResult(t *testing.T, points []influxPoint) []*apptest.QueryResult {
|
||||
t.Helper()
|
||||
|
||||
grouped := map[string]*apptest.QueryResult{}
|
||||
for _, p := range points {
|
||||
if p.FieldType == "string" {
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("%s_%s", p.Measurement, p.Field)
|
||||
metric := map[string]string{
|
||||
"__name__": name,
|
||||
"db": influxTestDatabase,
|
||||
}
|
||||
for k, v := range p.Tags {
|
||||
metric[k] = v
|
||||
}
|
||||
key := tagsKey(metric)
|
||||
if _, ok := grouped[key]; !ok {
|
||||
grouped[key] = &apptest.QueryResult{Metric: metric}
|
||||
}
|
||||
grouped[key].Samples = append(grouped[key].Samples, &apptest.Sample{
|
||||
Timestamp: p.Timestamp * 1000,
|
||||
Value: influxExpectedValue(t, p.Value),
|
||||
})
|
||||
}
|
||||
out := make([]*apptest.QueryResult, 0, len(grouped))
|
||||
for _, v := range grouped {
|
||||
out = append(out, v)
|
||||
}
|
||||
resp := apptest.PrometheusAPIV1QueryResponse{
|
||||
Data: &apptest.QueryData{Result: out},
|
||||
}
|
||||
resp.Sort()
|
||||
return resp.Data.Result
|
||||
}
|
||||
|
||||
// influxExpectedValue converts a field value the way vmctl does: numbers pass
|
||||
// through and booleans become 1/0.
|
||||
func influxExpectedValue(t *testing.T, v any) float64 {
|
||||
t.Helper()
|
||||
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
return val
|
||||
case int64:
|
||||
return float64(val)
|
||||
case bool:
|
||||
if val {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
default:
|
||||
t.Fatalf("unexpected field value type %T in test fixture; only float64, int64 and bool are supported", v)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// assertInfluxRequests verifies how vmctl authenticated to InfluxDB and which
|
||||
// database and retention policy it requested.
|
||||
func assertInfluxRequests(t *testing.T, s *influxMockServer, wantUser, wantPass, wantRP string) {
|
||||
t.Helper()
|
||||
|
||||
reqs := s.queryRequests()
|
||||
if len(reqs) == 0 {
|
||||
t.Fatalf("vmctl issued no /query requests")
|
||||
}
|
||||
for _, r := range reqs {
|
||||
if !r.HasAuth {
|
||||
t.Fatalf("no basic auth on %s", r.Query)
|
||||
}
|
||||
if r.User != wantUser || r.Password != wantPass {
|
||||
t.Fatalf("unexpected credentials for q=%q\ngot\n(%q, %q)\nwant\n(%q, %q)",
|
||||
r.Query.Get("q"), r.User, r.Password, wantUser, wantPass)
|
||||
}
|
||||
if rp := r.Query.Get("rp"); rp != wantRP {
|
||||
t.Fatalf("unexpected rp for q=%q\ngot\n%q\nwant\n%q", r.Query.Get("q"), rp, wantRP)
|
||||
}
|
||||
if db := r.Query.Get("db"); db != influxTestDatabase {
|
||||
t.Fatalf("unexpected db for q=%q\ngot\n%q\nwant\n%q", r.Query.Get("q"), db, influxTestDatabase)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// newInfluxTestPoints returns points covering the cases that matter for the
|
||||
// migration: numeric field types, a boolean, a string field which must be
|
||||
// skipped, a series missing one of the measurement tags, and a measurement name
|
||||
// containing special characters.
|
||||
func newInfluxTestPoints(baseTS int64) []influxPoint {
|
||||
var points []influxPoint
|
||||
for i := range 10 {
|
||||
ts := baseTS + int64(i*60)
|
||||
|
||||
// Two series of `cpu`: the second one has no `env` tag, which exercises
|
||||
// the empty-tag conditions vmctl adds to its SELECT statements.
|
||||
points = append(points,
|
||||
influxPoint{"cpu", map[string]string{"host": "h1", "env": "prod"}, "value", "float", ts, float64(i) + 0.5},
|
||||
influxPoint{"cpu", map[string]string{"host": "h1", "env": "prod"}, "count", "integer", ts, int64(i)},
|
||||
influxPoint{"cpu", map[string]string{"host": "h1", "env": "prod"}, "flag", "boolean", ts, i%2 == 0},
|
||||
influxPoint{"cpu", map[string]string{"host": "h1", "env": "prod"}, "note", "string", ts, "ignored"},
|
||||
influxPoint{"cpu", map[string]string{"host": "h2"}, "value", "float", ts, float64(i) - 2.25},
|
||||
influxPoint{"cpu", map[string]string{"host": "h2"}, "note", "string", ts, "ignored"},
|
||||
// Special characters in a measurement name, cf. issue #10892.
|
||||
influxPoint{"user_percent.mem.zwickel+", map[string]string{"host": "h1"}, "value", "float", ts, float64(i * 3)},
|
||||
)
|
||||
}
|
||||
return points
|
||||
}
|
||||
@@ -829,7 +829,7 @@ See also [minimum downtime strategy](#minimum-downtime-strategy).
|
||||
|
||||
## Slowness-based re-routing
|
||||
|
||||
By default{{% available_from "v1.149.0" %}}, `vminsert` automatically re-routes writes away from the slowest `vmstorage` node
|
||||
By default{{% available_from "#" %}}, `vminsert` automatically re-routes writes away from the slowest `vmstorage` node
|
||||
to preserve maximum ingestion throughput. This prevents a single slow `vmstorage` node
|
||||
from throttling the entire cluster.
|
||||
|
||||
@@ -843,7 +843,7 @@ Disable slowness-based re-routing with `-disableRerouting=true` when keeping met
|
||||
perfectly balanced across nodes or minimizing the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series)
|
||||
matters more than peak write throughput.
|
||||
|
||||
Slowness-based re-routing is automatically disabled{{% available_from "v1.149.0" %}} when `-replicationFactor` is greater than `1`,
|
||||
Slowness-based re-routing is automatically disabled{{% available_from "#" %}} when `-replicationFactor` is greater than `1`,
|
||||
because rerouting does not guarantee that replicated copies land on distinct storage nodes,
|
||||
which violates the replication contract.
|
||||
|
||||
|
||||
@@ -39,8 +39,6 @@ VictoriaMetrics has the following prominent features:
|
||||
* Easy and fast backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
|
||||
can be done with [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) / [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/) tools.
|
||||
See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
|
||||
* It supports storage and retrieval of samples with timestamps that fall within the `[1970-01-02T00:00:00.000Z, 2262-03-31T23:59:59.999Z]` time range with millisecond precision.
|
||||
See [Retention](#retention) for details.
|
||||
* It implements a PromQL-like query language - [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/), which provides improved functionality on top of PromQL.
|
||||
* It provides a global query view. Multiple Prometheus instances or any other data sources may ingest data into VictoriaMetrics. Later this data may be queried via a single query.
|
||||
* It provides high performance and good vertical and horizontal scalability for both
|
||||
@@ -1542,9 +1540,6 @@ It is safe to extend `-retentionPeriod` on existing data. If `-retentionPeriod`
|
||||
value than before, then data outside the configured period will be eventually deleted.
|
||||
|
||||
VictoriaMetrics does not support indefinite retention, but you can specify an arbitrarily high duration, e.g. `-retentionPeriod=100y`.
|
||||
Just keep in mind that VictoriaMetrics does not support samples with negative timestamps. Timestamps at `1970-01-01` are also not
|
||||
supported because this date has a special meaning internally. It therefore rejects samples with timestamps before
|
||||
`1970-01-02T00:00:00.000Z`.
|
||||
|
||||
By default, VictoriaMetrics doesn't accept samples with timestamps bigger than `now+2d`, e.g. 2 days in the future.
|
||||
If you need accepting samples with bigger timestamps, then specify the desired "future retention" via `-futureRetention` command-line flag.
|
||||
@@ -1556,9 +1551,6 @@ For example, the following command starts VictoriaMetrics, which accepts samples
|
||||
/path/to/victoria-metrics -futureRetention=1y
|
||||
```
|
||||
|
||||
VictoriaMetrics does not support stamples after `2262-03-31T23:59:59.999Z`. If the future retention includes dates after this timestamp,
|
||||
the samples for those dates will be rejected.
|
||||
|
||||
By default, VictoriaMetrics accepts samples with timestamps as old as the configured `-retentionPeriod` allows, e.g. it accepts backfilled
|
||||
historical data as long as it fits into the retention. If you need rejecting samples with historical timestamps older than the specified
|
||||
duration, then specify the desired duration via the `-maxBackfillAge` command-line flag. This can be useful for limiting ingestion of
|
||||
|
||||
@@ -26,10 +26,6 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
|
||||
## tip
|
||||
|
||||
## [v1.149.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.149.0)
|
||||
|
||||
Release candidate
|
||||
|
||||
**Update Note 1:** `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): the default value of `-disableRerouting` flag has changed from `true` to `false`, enabling [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. If you rely on the old behavior, pass `-disableRerouting` command-line flag to `vminsert`. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
|
||||
|
||||
**Update Note 2:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): the `/api/v1/admin/tsdb/delete_series`, `/tags/delSeries` endpoints now require `POST` method. Previously, it also accepted `GET` requests. If you use `GET` requests for this endpoint, update your scripts or tooling to use `POST` instead. See [#5552](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5552).
|
||||
@@ -40,18 +36,18 @@ Release candidate
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `name` label identifying the corresponding `-remoteWrite.url` target to the `vm_persistentqueue_*` metrics exposed by persistent queue. See [#7944](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7944). Thanks to @tIGO for contribution.
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-remoteWrite.obfuscateLabels` flag for hashing values of the specified labels before sending metrics to the corresponding `-remoteWrite.url`. This allows sharing metrics with external systems while keeping sensitive label values hidden. See [#10599](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10599).
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `-replay.continueWithExecutionErr` flag to allow continuing to replay other rules when a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit. See [11313](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11313).
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add template variable `$interval` to expose the alerting rule group's evaluation interval. This allows generating dashboard links with a lookback window relative to the rule's interval, for example `&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}`. See [#11232](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11232). Thanks to @1solomonwakhungu for contribution.
|
||||
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, `NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
|
||||
* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support [Prometheus native histograms](https://prometheus.io/docs/specs/native_histograms/) migration in [remote read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/). Native histograms are converted into `_count`, `_sum` and `_bucket` series with `vmrange` labels in the same way as VictoriaMetrics [converts native histograms received via Prometheus remote write protocol](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/#native-histograms), except that for native histograms with custom buckets the original bucket bounds are preserved instead of being estimated with the exponential formula. Previously native histograms were silently ignored in `SAMPLES` mode, while in stream mode the migration failed with `EOF` error. See [#11292](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11292). Thanks to @liuxu623 for contribution.
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add template variable `$interval` to expose the alerting rule group's evaluation interval. This allows generating dashboard links with a lookback window relative to the rule's interval, for example `&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}`. See this issue [#11232](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11232) for more details. Thanks to @1solomonwakhungu for contribution.
|
||||
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`,
|
||||
`NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
|
||||
* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support [Prometheus native histograms](https://prometheus.io/docs/specs/native_histograms/) migration in [remote read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/). Native histograms are converted into `_count`, `_sum` and `_bucket` series with `vmrange` labels in the same way as VictoriaMetrics converts native histograms received via Prometheus remote write protocol, except that for native histograms with custom buckets the original bucket bounds are preserved instead of being estimated with the exponential formula. Previously native histograms were silently ignored in `SAMPLES` mode, while in stream mode the migration failed with `EOF` error. See [#11292](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11292). Thanks to @liuxu623 for contribution.
|
||||
* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): enable [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Previously, `-disableRerouting` defaulted to `true`, which limited ingestion throughput to the slowest `vmstorage` node. Now `-disableRerouting` defaults to `false`, so `vminsert` automatically routes data away from the slowest `vmstorage` node, improving overall ingestion performance. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the selected auto-refresh interval in the URL. See [VictoriaLogs#1310](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1310).
|
||||
* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add the ability to migrate data from InfluxDB 2.x in the `influx` mode via the new `-influx-version` and `-influx-token` command-line flags. Pass `-influx-version=2` together with `-influx-token` to authenticate with an InfluxDB 2.x API token; `-influx-database` then accepts the bucket name. Migration goes through the [InfluxDB 1.x compatibility API](https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/), which InfluxDB OSS exposes for every bucket automatically, so no database and retention policy mapping has to be created manually. See [#5914](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5914).
|
||||
|
||||
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): ignore HTTP proxy environment variables when scraping targets over Unix domain sockets. See [#11318](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11318). Thanks to @lwmacct for contribution.
|
||||
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fixed the display of rule state badges on the `Groups` page in the web UI. See [#11160](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11160).
|
||||
* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): previously, `vmbackupmanager` was crashing on startup when it failed to restore backup state from remote storage, causing a crash loop. Now it logs the error and continues running, retrying the state restore before each scheduled backup. Added `vm_backup_errors_total{type="restoreState"}` metric to track backup state restore failures. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
|
||||
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix incorrect [sum_samples_total](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#sum_samples_total) results when `enable_windows: true` is set. See [#11261](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11261). Thanks to @beyond-infra for contribution.
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): accept scientific notation with sub-second precision (e.g. `1.784144612388E9`) for timestamp args such as `start` and `end` in `/api/v1/query_range` and `--vm-native-filter-time-start` and `--vm-native-filter-time-end` in `vmctl`. Previously, values with this pattern were rejected, which is incompatible with Prometheus. See [#11268](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11268). Thanks to @STiFLeR7 for contribution.
|
||||
|
||||
## [v1.148.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.0)
|
||||
|
||||
|
||||
@@ -688,7 +688,7 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
|
||||
## Obfuscating label values
|
||||
|
||||
`vmagent` can obfuscate the values of specified labels before sending metrics to `-remoteWrite.url`
|
||||
via `-remoteWrite.obfuscateLabels`{{% available_from "v1.149.0" %}}.
|
||||
via `-remoteWrite.obfuscateLabels`{{% available_from "#" %}}.
|
||||
|
||||
This is useful when one or more `-remoteWrite.url` endpoints point to external monitoring services
|
||||
outside the organization, and sensitive label values such as `ip`, `host`, `instance`, or `datacenter`
|
||||
|
||||
@@ -480,7 +480,7 @@ Clusters here are referred to as `source` and `destination`.
|
||||
|
||||
To verify that `vmbackupmanager` is executing backup tasks normally, the following metrics can help:
|
||||
|
||||
* `vm_backup_last_success_at{type="<backup_type>"}` - unix timestamp of the last successful backup{{% available_from "v1.149.0" %}}. Remains `0` if no backup has completed successfully since startup. Check error logs and verify remote storage accessibility if this persists.
|
||||
* `vm_backup_last_success_at{type="<backup_type>"}` - unix timestamp of the last successful backup{{% available_from "#" %}}. Remains `0` if no backup has completed successfully since startup. Check error logs and verify remote storage accessibility if this persists.
|
||||
* `vm_backup_last_run_failed{type="<backup_type>"}` - whether the last backup task for the given backup type failed. The value `1` means the last task failed. Check the error logs of `vmbackupmanager` for the root cause
|
||||
* `vm_backup_errors_total{type="<backup_type>"}` - total number of backup errors for the given backup type.
|
||||
|
||||
|
||||
@@ -7,9 +7,12 @@ menu:
|
||||
identifier: "vmctl-influxdb"
|
||||
weight: 2
|
||||
---
|
||||
`vmctl` can migrate historical data from InfluxDB (v1) to VictoriaMetrics. See `./vmctl influx --help` for details and
|
||||
`vmctl` can migrate historical data from InfluxDB v1 and v2 to VictoriaMetrics. See `./vmctl influx --help` for details and
|
||||
full list of flags. Also see [migrating data from InfluxDB to VictoriaMetrics](https://docs.victoriametrics.com/guides/migrate-from-influx/) article.
|
||||
|
||||
For InfluxDB v2 see [InfluxDB v2](#influxdb-v2) below. The examples in this section use InfluxDB v1,
|
||||
which is the default (`--influx-version=1`).
|
||||
|
||||
To start migration, specify the InfluxDB address `--influx-addr`, database `--influx-database` and VictoriaMetrics address `--vm-addr`:
|
||||
```sh
|
||||
./vmctl influx --influx-addr=http://<influx-addr>:8086 \
|
||||
@@ -92,8 +95,45 @@ See more about [time filtering in InfluxDB](https://docs.influxdata.com/influxdb
|
||||
|
||||
## InfluxDB v2
|
||||
|
||||
Migrating data from InfluxDB v2.x is not supported yet ([#32](https://github.com/VictoriaMetrics/vmctl/issues/32)).
|
||||
You may find useful a 3rd party solution for this - <https://github.com/jonppe/influx_to_victoriametrics>.
|
||||
Migrating data from InfluxDB v2 is supported {{% available_from "#" %}} via the `--influx-version=2` flag. In this mode vmctl reads
|
||||
data through the [InfluxDB 1.x compatibility API](https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/)
|
||||
of InfluxDB v2, so migration uses the same queries and produces the same
|
||||
[data mapping](#data-mapping) as for InfluxDB v1. Only authentication differs:
|
||||
|
||||
- `--influx-token` must be set to an InfluxDB v2 [API token](https://docs.influxdata.com/influxdb/v2/admin/tokens/).
|
||||
It is sent as the password of the compatibility API. The username is required by that API but its value
|
||||
is ignored, so `--influx-user` may be left unset.
|
||||
- `--influx-database` accepts the name of the **bucket** to migrate.
|
||||
- `--influx-retention-policy` should be left unset. InfluxDB then resolves the default database and
|
||||
retention policy (DBRP) mapping of the bucket. Set it only if the bucket is exposed through an explicit
|
||||
mapping with a non-default retention policy name.
|
||||
|
||||
```sh
|
||||
./vmctl influx --influx-version=2 \
|
||||
--influx-addr=http://<influx-addr>:8086 \
|
||||
--influx-token=<influx-token> \
|
||||
--influx-database=<bucket-name> \
|
||||
--vm-addr=http://<victoriametrics-addr>:8428
|
||||
```
|
||||
|
||||
InfluxDB OSS automatically creates a *virtual* DBRP mapping for every bucket, where the database name equals
|
||||
the bucket name. No mapping has to be created manually before the migration. Existing mappings can be listed
|
||||
with:
|
||||
|
||||
```sh
|
||||
curl -s 'http://<influx-addr>:8086/api/v2/dbrps?org=<org>' \
|
||||
--header 'Authorization: Token <influx-token>'
|
||||
```
|
||||
|
||||
If the bucket must be reachable under a different database name, or if several retention policies are mapped
|
||||
to different buckets, create the mapping explicitly. See
|
||||
[Database and retention policy mapping](https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/dbrp/)
|
||||
and pass its `database` and `retention_policy` values via `--influx-database` and `--influx-retention-policy`.
|
||||
|
||||
All the other `--influx-*` flags behave the same as for InfluxDB v1, including
|
||||
[filtering](#filtering) via `--influx-filter-series` and the time filters, since the compatibility API
|
||||
accepts the same InfluxQL statements. When using `--influx-filter-series` with an `ON <database_name>`
|
||||
clause, the database name is the one of the DBRP mapping, which for a virtual mapping is the bucket name.
|
||||
|
||||
## Configuration
|
||||
|
||||
|
||||
@@ -1290,9 +1290,6 @@ func (pt *partition) mergeParts(pws []*partWrapper, stopCh <-chan struct{}, isFi
|
||||
putBlockStreamReader(bsr)
|
||||
}
|
||||
if err != nil {
|
||||
if mpNew != nil {
|
||||
putInmemoryPart(mpNew)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if mpNew != nil {
|
||||
@@ -1447,8 +1444,6 @@ func (pt *partition) openCreatedPart(ph *partHeader, pws []*partWrapper, mpNew *
|
||||
// The created part is empty. Remove it
|
||||
if mpNew == nil {
|
||||
fs.MustRemoveDir(dstPartPath)
|
||||
} else {
|
||||
putInmemoryPart(mpNew)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1118,14 +1118,6 @@ func searchAndMerge[T any](qt *querytracer.Tracer, s *Storage, tr TimeRange, sea
|
||||
qt = qt.NewChild("search indexDBs: timeRange=%v", &tr)
|
||||
defer qt.Done()
|
||||
|
||||
var zeroValue T
|
||||
if tr.MinTimestamp < minUnixMilli {
|
||||
tr.MinTimestamp = minUnixMilli
|
||||
}
|
||||
if tr.MaxTimestamp < tr.MinTimestamp {
|
||||
return zeroValue, nil
|
||||
}
|
||||
|
||||
var idbts []indexDBWithType
|
||||
|
||||
ptws := s.tb.GetPartitions(tr)
|
||||
|
||||
@@ -402,10 +402,6 @@ func TestStorageDeletePendingSeries(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
const numMonths = 10
|
||||
start := time.Date(1971, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
middle := start.AddDate(0, (numMonths-1)/2, 0)
|
||||
end := start.AddDate(0, numMonths-1, 0)
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{})
|
||||
|
||||
var metricGroupName = []byte("metric")
|
||||
@@ -460,7 +456,7 @@ func TestStorageDeletePendingSeries(t *testing.T) {
|
||||
assertCountMonthsWithLabels := func(count int) {
|
||||
t.Helper()
|
||||
|
||||
ts := start
|
||||
ts := time.Unix(0, 0)
|
||||
n := 0
|
||||
for range numMonths {
|
||||
lns, err := s.SearchLabelNames(nil, nil, TimeRange{ts.UnixMilli(), ts.UnixMilli()}, 1e5, 1e9, noDeadline)
|
||||
@@ -485,7 +481,7 @@ func TestStorageDeletePendingSeries(t *testing.T) {
|
||||
var search Search
|
||||
defer search.MustClose()
|
||||
|
||||
search.Init(nil, s, []*TagFilters{tfs}, TimeRange{start.UnixMilli(), math.MaxInt64}, 1e5, noDeadline)
|
||||
search.Init(nil, s, []*TagFilters{tfs}, TimeRange{0, math.MaxInt64}, 1e5, noDeadline)
|
||||
n := 0
|
||||
for search.NextMetricBlock() {
|
||||
var b Block
|
||||
@@ -502,6 +498,10 @@ func TestStorageDeletePendingSeries(t *testing.T) {
|
||||
// Verify no metrics exist
|
||||
assertCountRows(0)
|
||||
|
||||
start := time.Unix(0, 0)
|
||||
middle := start.AddDate(0, (numMonths-1)/2, 0)
|
||||
end := start.AddDate(0, numMonths-1, 0)
|
||||
|
||||
// Add some rows and flush, so next DeleteSeries() can delete them
|
||||
addRows(start, middle, false)
|
||||
s.DebugFlush()
|
||||
@@ -3385,190 +3385,53 @@ func TestStorageQueryWithoutIndex(t *testing.T) {
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageAddRowsWithZeroDate(t *testing.T) {
|
||||
func TestStorageAddRows_SamplesWithZeroDate(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
f := func(t *testing.T, disablePerDayIndex bool) {
|
||||
t.Helper()
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
|
||||
mn := MetricName{MetricGroup: []byte("metric")}
|
||||
mr := MetricRow{MetricNameRaw: mn.marshalRaw(nil)}
|
||||
for range 10 {
|
||||
mr.Timestamp = rand.Int63n(msecPerDay)
|
||||
mr.Value = float64(rand.Intn(1000))
|
||||
s.AddRows([]MetricRow{mr}, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
// Reset TSID cache so that insertion takes the path that involves
|
||||
// checking whether the index contains metricName->TSID mapping.
|
||||
s.resetAndSaveTSIDCache()
|
||||
}
|
||||
|
||||
want := 1
|
||||
firstUnixDay := TimeRange{
|
||||
MinTimestamp: 0,
|
||||
MaxTimestamp: msecPerDay - 1,
|
||||
}
|
||||
if got := s.newTimeseriesCreated.Load(); got != uint64(want) {
|
||||
t.Errorf("unexpected new timeseries count: got %d, want %d", got, want)
|
||||
}
|
||||
if got := testCountAllMetricNames(s, firstUnixDay); got != want {
|
||||
t.Errorf("unexpected metric name count: got %d, want %d", got, want)
|
||||
}
|
||||
if got := testCountAllMetricIDs(s, firstUnixDay); got != want {
|
||||
t.Errorf("unexpected metric id count: got %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testStorageAddRowsWithZeroDate(t, disablePerDayIndex)
|
||||
f(t, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testStorageAddRowsWithZeroDate(t *testing.T, disablePerDayIndex bool) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
|
||||
const numDays = 4
|
||||
var metricNamesAll []string
|
||||
labelNamesAll := []string{"__name__", "label"}
|
||||
var labelValuesAll []string
|
||||
mrs := make([]MetricRow, numDays)
|
||||
for day := range numDays {
|
||||
metricName := fmt.Sprintf("metric_%02d", day)
|
||||
labelName := fmt.Sprintf("label_%02d", day)
|
||||
labelValue := fmt.Sprintf("value_%02d", day)
|
||||
|
||||
if day != 0 {
|
||||
metricNamesAll = append(metricNamesAll, metricName)
|
||||
labelNamesAll = append(labelNamesAll, labelName)
|
||||
labelValuesAll = append(labelValuesAll, labelValue)
|
||||
}
|
||||
|
||||
mn := MetricName{
|
||||
MetricGroup: []byte(metricName),
|
||||
Tags: []Tag{
|
||||
{Key: []byte(labelName), Value: []byte("value")},
|
||||
{Key: []byte("label"), Value: []byte(labelValue)},
|
||||
},
|
||||
}
|
||||
mn.sortTags()
|
||||
|
||||
mrs[day].MetricNameRaw = mn.marshalRaw(nil)
|
||||
mrs[day].Timestamp = int64(day * msecPerDay)
|
||||
}
|
||||
|
||||
s.AddRows(mrs, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
if got, want := s.newTimeseriesCreated.Load(), uint64(numDays-1); got != want {
|
||||
t.Fatalf("unexpected new timeseries count: got %d, want %d", got, want)
|
||||
}
|
||||
if got, want := s.tooSmallTimestampRows.Load(), uint64(1); got != want {
|
||||
t.Fatalf("unexpected rows with too small timestamp: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
assertMetricNames := func(tr TimeRange, want []string) {
|
||||
t.Helper()
|
||||
tfs := NewTagFilters()
|
||||
if err := tfs.Add(nil, []byte("metric_.*"), false, true); err != nil {
|
||||
t.Fatalf("unexpected error in TagFilters.Add: %v", err)
|
||||
}
|
||||
got, err := s.SearchMetricNames(nil, []*TagFilters{tfs}, tr, 1e9, noDeadline)
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMetricNames(%v, %v) failed unexpectedly: %v", tfs, &tr, err)
|
||||
}
|
||||
for i, name := range got {
|
||||
var mn MetricName
|
||||
if err := mn.UnmarshalString(name); err != nil {
|
||||
t.Fatalf("Could not unmarshal metric name %q: %v", name, err)
|
||||
}
|
||||
got[i] = string(mn.MetricGroup)
|
||||
}
|
||||
slices.Sort(got)
|
||||
slices.Sort(want)
|
||||
if diff := cmp.Diff(want, got); diff != "" {
|
||||
t.Fatalf("unexpected metric names (-want, +got):\n%s", diff)
|
||||
}
|
||||
}
|
||||
assertLabelNames := func(tr TimeRange, want []string) {
|
||||
t.Helper()
|
||||
tfs := NewTagFilters()
|
||||
if err := tfs.Add(nil, []byte("metric_.*"), false, true); err != nil {
|
||||
t.Fatalf("unexpected error in TagFilters.Add: %v", err)
|
||||
}
|
||||
got, err := s.SearchLabelNames(nil, []*TagFilters{tfs}, tr, 1e9, 1e9, noDeadline)
|
||||
if err != nil {
|
||||
t.Fatalf("SearchLabelNames(%v, %v) failed unexpectedly: %s", tfs, &tr, err)
|
||||
}
|
||||
slices.Sort(got)
|
||||
slices.Sort(want)
|
||||
if diff := cmp.Diff(want, got); diff != "" {
|
||||
t.Fatalf("unexpected label names (-want, +got):\n%s", diff)
|
||||
}
|
||||
}
|
||||
assertLabelValues := func(tr TimeRange, want []string) {
|
||||
t.Helper()
|
||||
tfs := NewTagFilters()
|
||||
if err := tfs.Add([]byte("label"), []byte("value_.*"), false, true); err != nil {
|
||||
t.Fatalf("unexpected error in TagFilters.Add: %v", err)
|
||||
}
|
||||
got, err := s.SearchLabelValues(nil, "label", []*TagFilters{tfs}, tr, 1e9, 1e9, noDeadline)
|
||||
if err != nil {
|
||||
t.Fatalf("SearchLabelValues(%v, %v) failed unexpectedly: %s", tfs, tr, err)
|
||||
}
|
||||
slices.Sort(got)
|
||||
slices.Sort(want)
|
||||
if diff := cmp.Diff(want, got); diff != "" {
|
||||
t.Fatalf("unexpected label values (-want, +got):\n%s", diff)
|
||||
}
|
||||
}
|
||||
assertData := func(tr TimeRange, want []MetricRow) {
|
||||
t.Helper()
|
||||
tfs := NewTagFilters()
|
||||
if err := tfs.Add(nil, []byte("metric_.*"), false, true); err != nil {
|
||||
t.Fatalf("TagFilters.Add() failed unexpectedly: %v", err)
|
||||
}
|
||||
if err := testAssertSearchResult(s, tr, tfs, want); err != nil {
|
||||
t.Fatalf("Search(%v, %v) failed unexpectedly: %v", tfs, tr, err)
|
||||
}
|
||||
}
|
||||
|
||||
var tr TimeRange
|
||||
|
||||
// Empty time range.
|
||||
// Expect empty search results
|
||||
tr = TimeRange{}
|
||||
assertMetricNames(tr, nil)
|
||||
assertLabelNames(tr, []string{})
|
||||
assertLabelValues(tr, []string{})
|
||||
assertData(tr, nil)
|
||||
|
||||
// First day time range.
|
||||
// Expect empty search results
|
||||
tr = TimeRange{
|
||||
MinTimestamp: 0,
|
||||
MaxTimestamp: msecPerDay - 1,
|
||||
}
|
||||
assertMetricNames(tr, nil)
|
||||
assertLabelNames(tr, []string{})
|
||||
assertLabelValues(tr, []string{})
|
||||
assertData(tr, nil)
|
||||
|
||||
// Second day time range.
|
||||
tr = TimeRange{
|
||||
MinTimestamp: msecPerDay,
|
||||
MaxTimestamp: 2*msecPerDay - 1,
|
||||
}
|
||||
if disablePerDayIndex {
|
||||
// Expect index search results for all days if per-day index is
|
||||
// disabled.
|
||||
assertMetricNames(tr, metricNamesAll)
|
||||
assertLabelNames(tr, labelNamesAll)
|
||||
assertLabelValues(tr, labelValuesAll)
|
||||
} else {
|
||||
// Expect index search results on second day only if per-day index is
|
||||
// enabled.
|
||||
assertMetricNames(tr, []string{"metric_01"})
|
||||
assertLabelNames(tr, []string{"__name__", "label", "label_01"})
|
||||
assertLabelValues(tr, []string{"value_01"})
|
||||
}
|
||||
assertData(tr, mrs[1:2])
|
||||
|
||||
// First two days time range.
|
||||
// Expect results on second day only.
|
||||
tr = TimeRange{
|
||||
MinTimestamp: 0,
|
||||
MaxTimestamp: 2*msecPerDay - 1,
|
||||
}
|
||||
if disablePerDayIndex {
|
||||
// Expect index search results for all days if per-day index is
|
||||
// disabled.
|
||||
assertMetricNames(tr, metricNamesAll)
|
||||
assertLabelNames(tr, labelNamesAll)
|
||||
assertLabelValues(tr, labelValuesAll)
|
||||
} else {
|
||||
// Expect index search results on second day only if per-day index is
|
||||
// enabled.
|
||||
assertMetricNames(tr, []string{"metric_01"})
|
||||
assertLabelNames(tr, []string{"__name__", "label", "label_01"})
|
||||
assertLabelValues(tr, []string{"value_01"})
|
||||
}
|
||||
assertData(tr, mrs[1:2])
|
||||
}
|
||||
|
||||
// testSearchMetricIDs returns metricIDs for the given tfss and tr.
|
||||
//
|
||||
// The returned metricIDs are sorted. The function panics in in case of error.
|
||||
|
||||
@@ -429,8 +429,9 @@ func (tb *table) getMinMaxIngestionTimestamps() (int64, int64) {
|
||||
func (tb *table) getMinMaxTimestampsForAge(minAgeMsecs int64) (int64, int64) {
|
||||
now := int64(fasttime.UnixTimestamp() * 1000)
|
||||
minTimestamp := now - minAgeMsecs
|
||||
if minTimestamp < minUnixMilli {
|
||||
minTimestamp = minUnixMilli
|
||||
if minTimestamp < 0 {
|
||||
// Negative timestamps aren't supported by the storage.
|
||||
minTimestamp = 0
|
||||
}
|
||||
maxTimestamp := int64(maxUnixMilli)
|
||||
if maxUnixMilli-now > tb.s.futureRetentionMsecs {
|
||||
|
||||
@@ -40,6 +40,12 @@ type TimeRange struct {
|
||||
MaxTimestamp int64
|
||||
}
|
||||
|
||||
// Zero time range and zero date are used to force global index search.
|
||||
var (
|
||||
globalIndexTimeRange = TimeRange{}
|
||||
globalIndexDate = uint64(0)
|
||||
)
|
||||
|
||||
// DateRange returns the date range for the given time range.
|
||||
func (tr *TimeRange) DateRange() (uint64, uint64) {
|
||||
minDate := uint64(tr.MinTimestamp) / msecPerDay
|
||||
@@ -111,29 +117,10 @@ func (tr *TimeRange) contains(timestamp int64) bool {
|
||||
return tr.MinTimestamp <= timestamp && timestamp <= tr.MaxTimestamp
|
||||
}
|
||||
|
||||
// Zero time range and zero date are used to force global index search.
|
||||
var (
|
||||
globalIndexDate = uint64(0)
|
||||
globalIndexTimeRange = TimeRange{}
|
||||
)
|
||||
|
||||
const (
|
||||
msecPerDay = 24 * 3600 * 1000
|
||||
msecPerHour = 3600 * 1000
|
||||
|
||||
// minUnixMilli is the min millisecond that is allowed to be used as the
|
||||
// sample timestamp.
|
||||
//
|
||||
// It corresponds to the first millisecond of the second day of the Unix
|
||||
// Epoch, i.e. 1970-01-02T00:00:00.000Z.
|
||||
//
|
||||
// The first day of the Unix Epoch is reserved: zero date and zero time
|
||||
// range are used for indicating that the the global index search is
|
||||
// required. See globalIndexDate and globalIndexTimeRange above.
|
||||
//
|
||||
// Negative timestamps aren't supported.
|
||||
minUnixMilli = msecPerDay
|
||||
|
||||
// maxUnixMilli is the max millisecond that is allowed to be used as the
|
||||
// sample timestamp.
|
||||
//
|
||||
@@ -143,6 +130,6 @@ const (
|
||||
// time.UnixMicro(math.MaxInt64/1000) == 2262-04-11 23:47:16.854775 UTC.
|
||||
//
|
||||
// Round it to the last millisecond of the last complete partition:
|
||||
// 2262-03-31T23:59:59.999Z.
|
||||
// 2262-03-31 23:59:59.999 UTC.
|
||||
maxUnixMilli = 9222422399999
|
||||
)
|
||||
|
||||
@@ -935,25 +935,4 @@ foo:1m_sum_samples{baz="qwe"} 10
|
||||
dedup_interval: 30s
|
||||
outputs: [sum_samples]
|
||||
`, "11111111")
|
||||
|
||||
// Reproduce issue #11261: sum_samples_total must be monotonic with enable_windows: true
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11262
|
||||
f([]string{`
|
||||
test_delta 1
|
||||
`, `
|
||||
test_delta 1
|
||||
`, `
|
||||
test_delta 1
|
||||
`, `
|
||||
test_delta 1
|
||||
`}, time.Minute, `test_delta 1
|
||||
test_delta 2
|
||||
test_delta 3
|
||||
test_delta 4
|
||||
`, `
|
||||
- interval: 1m
|
||||
keep_metric_names: true
|
||||
outputs: [sum_samples_total]
|
||||
enable_windows: true
|
||||
`, "1111")
|
||||
}
|
||||
|
||||
@@ -4,39 +4,30 @@ import (
|
||||
"math"
|
||||
)
|
||||
|
||||
type sumSamplesAggrValueShared struct {
|
||||
total float64
|
||||
}
|
||||
|
||||
type sumSamplesAggrValue struct {
|
||||
delta float64
|
||||
shared *sumSamplesAggrValueShared
|
||||
sum float64
|
||||
}
|
||||
|
||||
func (av *sumSamplesAggrValue) pushSample(_ aggrConfig, sample *pushSample, _ string, _ int64) {
|
||||
av.delta += sample.value
|
||||
if math.Abs(av.sum) >= (1 << 53) {
|
||||
// It is time to reset the entry, since it starts losing float64 precision
|
||||
av.sum = 0
|
||||
}
|
||||
av.sum += sample.value
|
||||
}
|
||||
|
||||
func (av *sumSamplesAggrValue) flush(c aggrConfig, ctx *flushCtx, key string, _ bool) {
|
||||
ac := c.(*sumSamplesAggrConfig)
|
||||
if ac.resetTotalOnFlush {
|
||||
ctx.appendSeries(key, "sum_samples", av.delta)
|
||||
av.delta = 0
|
||||
ctx.appendSeries(key, "sum_samples", av.sum)
|
||||
av.sum = 0
|
||||
return
|
||||
}
|
||||
total := av.shared.total + av.delta
|
||||
av.delta = 0
|
||||
if math.Abs(total) >= (1 << 53) {
|
||||
// It is time to reset the entry, since it starts losing float64 precision
|
||||
av.shared.total = 0
|
||||
} else {
|
||||
av.shared.total = total
|
||||
}
|
||||
ctx.appendSeries(key, "sum_samples_total", total)
|
||||
ctx.appendSeries(key, "sum_samples_total", av.sum)
|
||||
}
|
||||
|
||||
func (av *sumSamplesAggrValue) state() any {
|
||||
return av.shared
|
||||
func (*sumSamplesAggrValue) state() any {
|
||||
return nil
|
||||
}
|
||||
|
||||
func newSumSamplesAggrConfig(resetTotalOnFlush bool) aggrConfig {
|
||||
@@ -49,14 +40,6 @@ type sumSamplesAggrConfig struct {
|
||||
resetTotalOnFlush bool
|
||||
}
|
||||
|
||||
func (*sumSamplesAggrConfig) getValue(s any) aggrValue {
|
||||
var shared *sumSamplesAggrValueShared
|
||||
if s == nil {
|
||||
shared = &sumSamplesAggrValueShared{}
|
||||
} else {
|
||||
shared = s.(*sumSamplesAggrValueShared)
|
||||
}
|
||||
return &sumSamplesAggrValue{
|
||||
shared: shared,
|
||||
}
|
||||
func (*sumSamplesAggrConfig) getValue(_ any) aggrValue {
|
||||
return &sumSamplesAggrValue{}
|
||||
}
|
||||
|
||||
@@ -206,36 +206,17 @@ func tryParseScientificNumberForUnixTimestamp(s string, decimalExp int64) (int64
|
||||
return multiplyByDecimalExp(n, decimalExp)
|
||||
}
|
||||
|
||||
if decimalExp < 0 {
|
||||
// Negative exponents on a fractional mantissa are intentionally not
|
||||
// supported. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11268
|
||||
return 0, false
|
||||
}
|
||||
|
||||
intStr := s[:dotIdx]
|
||||
fracStr := s[dotIdx+1:]
|
||||
if decimalExp < int64(len(fracStr)) {
|
||||
return 0, false
|
||||
}
|
||||
n, ok := tryParseFractionalNumberForUnixTimestamp(intStr, fracStr)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if decimalExp >= int64(len(fracStr)) {
|
||||
// The exponent shifts the decimal point past every fractional digit,
|
||||
// so the value is an integer number of seconds (or coarser).
|
||||
decimalExp -= int64(len(fracStr))
|
||||
return multiplyByDecimalExp(n, decimalExp)
|
||||
}
|
||||
|
||||
// The exponent leaves fractional digits, e.g. 1.784144612388E9 == 1784144612.388
|
||||
// Pad n as plain fractional timestamps do.
|
||||
fracDigits := int64(len(fracStr)) - decimalExp
|
||||
for fracDigits%3 != 0 {
|
||||
if n >= 0 && n > math.MaxInt64/10 || n < 0 && n < math.MinInt64/10 {
|
||||
return 0, false
|
||||
}
|
||||
n *= 10
|
||||
fracDigits++
|
||||
}
|
||||
return n, true
|
||||
decimalExp -= int64(len(fracStr))
|
||||
return multiplyByDecimalExp(n, decimalExp)
|
||||
}
|
||||
|
||||
func tryParseFractionalNumberForUnixTimestamp(intStr, fracStr string) (int64, bool) {
|
||||
|
||||
@@ -69,17 +69,6 @@ func TestTryParseUnixTimestamp_Success(t *testing.T) {
|
||||
f("1.23e2", 123000000000)
|
||||
f("1.2e1", 12000000000)
|
||||
f("1123.456789123456789E15", 1123456789123456789)
|
||||
|
||||
// scientific notation with sub-second precision, i.e. more fractional digits
|
||||
// than the exponent shifts (https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11268).
|
||||
// These must match the equivalent plain fractional form.
|
||||
f("1.784144612388E9", 1784144612388000000) // == 1784144612.388
|
||||
f("1.784144612388e9", 1784144612388000000)
|
||||
f("-1.784144612388e9", -1784144612388000000)
|
||||
f("1.5000000005e9", 1500000000500000000) // == 1500000000.5
|
||||
f("1.23456789e9", 1234567890000000000) // exponent consumes all frac digits (integer result)
|
||||
f("1.23e1", 12300000000000) // == 12.3
|
||||
f("1.234e0", 1234000000000) // == 1.234
|
||||
}
|
||||
|
||||
func TestTryParseUnixTimestamp_Failure(t *testing.T) {
|
||||
@@ -126,7 +115,9 @@ func TestTryParseUnixTimestamp_Failure(t *testing.T) {
|
||||
f("1e19")
|
||||
f("1.3e123456789090123")
|
||||
|
||||
// negative decimal exponent
|
||||
// too small decimal exponent
|
||||
f("1.23e1")
|
||||
f("1.234e0")
|
||||
f("1E-1")
|
||||
f("1.3e-123456789090123")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user