Compare commits

..

3 Commits

Author SHA1 Message Date
Zakhar Bessarab
1d6340fe83 deps: go mod vendor after 1334eee4
Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
2025-06-10 18:29:25 +04:00
Zakhar Bessarab
1334eee433 make/tools: switch only qtc to "go tool"
Restore other tools to migrate step by step.

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
2025-06-10 18:28:31 +04:00
Zakhar Bessarab
2d67e14d59 make/tools: switch tooling to use go tool for external tools
Go 1.24 added first class support for management of tools via built-in `go tool` commands - https://tip.golang.org/doc/go1.24#tools

Switching tools management to go tooling addresses the following:
- version drift - previously make commands did not verify specific version of the tool
- global installation of dependencies - dependencies are now well scoped and do not require any globally installed parts

This commit sets up tooling to use `go tool` pattern for the same make targets as previously so there are no changes in workflow needed for existing users.

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
2025-06-06 15:04:44 +04:00
61 changed files with 2561 additions and 1367 deletions

View File

@@ -91,7 +91,6 @@ jobs:
uses: codecov/codecov-action@v5
with:
files: ./coverage.txt
fail_ci_if_error: false
integration-test:
name: integration-test

View File

@@ -528,8 +528,6 @@ vet:
check-all: fmt vet golangci-lint govulncheck
clean-checkers: remove-golangci-lint remove-govulncheck
test:
GOEXPERIMENT=synctest go test ./lib/... ./app/...
@@ -545,7 +543,7 @@ test-full:
test-full-386:
GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
integration-test: victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore
integration-test: victoria-metrics vmagent vmalert vmauth vmctl
go test ./apptest/... -skip="^TestCluster.*"
benchmark:
@@ -574,12 +572,11 @@ app-local-goos-goarch:
app-local-windows-goarch:
CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
quicktemplate-gen: install-qtc
qtc
install-qtc:
which qtc || go install github.com/valyala/quicktemplate/qtc@latest
quicktemplate-gen:
go tool qtc
golangci-lint:
GOEXPERIMENT=synctest go tool golangci-lint run
golangci-lint: install-golangci-lint
GOEXPERIMENT=synctest golangci-lint run

File diff suppressed because one or more lines are too long

View File

@@ -35,10 +35,10 @@
<meta property="og:title" content="UI for VictoriaLogs">
<meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
<meta property="og:description" content="Explore your log data with VictoriaLogs UI">
<script type="module" crossorigin src="./assets/index-DhqzKCNf.js"></script>
<script type="module" crossorigin src="./assets/index-BaRvaPfA.js"></script>
<link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
<link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
<link rel="stylesheet" crossorigin href="./assets/index-C85_NB5q.css">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>

View File

@@ -818,7 +818,6 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
LookbackDelta: lookbackDelta,
RoundDigits: getRoundDigits(r),
EnforcedTagFilterss: etfs,
CacheTagFilters: etfs,
GetRequestURI: func() string {
return httpserver.GetRequestURI(r)
},
@@ -928,7 +927,6 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
LookbackDelta: lookbackDelta,
RoundDigits: getRoundDigits(r),
EnforcedTagFilterss: etfs,
CacheTagFilters: etfs,
GetRequestURI: func() string {
return httpserver.GetRequestURI(r)
},

View File

@@ -140,13 +140,6 @@ type EvalConfig struct {
// EnforcedTagFilterss may contain additional label filters to use in the query.
EnforcedTagFilterss [][]storage.TagFilter
// CacheTagFilters stores the original tag-filter sets and extra_label from the request.
// The slice is never modified after creation and is used only to build
// the query-cache key.
//
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9001
CacheTagFilters [][]storage.TagFilter
// The callback, which returns the request URI during logging.
// The request URI isn't stored here because its' construction may take non-trivial amounts of CPU.
GetRequestURI func() string
@@ -173,7 +166,6 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
ec.LookbackDelta = src.LookbackDelta
ec.RoundDigits = src.RoundDigits
ec.EnforcedTagFilterss = src.EnforcedTagFilterss
ec.CacheTagFilters = src.CacheTagFilters
ec.GetRequestURI = src.GetRequestURI
ec.QueryStats = src.QueryStats

View File

@@ -291,7 +291,7 @@ func (rrc *rollupResultCache) GetSeries(qt *querytracer.Tracer, ec *EvalConfig,
bb := bbPool.Get()
defer bbPool.Put(bb)
bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.CacheTagFilters)
bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
metainfoBuf := rrc.c.Get(nil, bb.B)
if len(metainfoBuf) == 0 {
qt.Printf("nothing found")
@@ -313,7 +313,7 @@ func (rrc *rollupResultCache) GetSeries(qt *querytracer.Tracer, ec *EvalConfig,
if !ok {
mi.RemoveKey(key)
metainfoBuf = mi.Marshal(metainfoBuf[:0])
bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.CacheTagFilters)
bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
rrc.c.Set(bb.B, metainfoBuf)
return nil, ec.Start
}
@@ -419,7 +419,7 @@ func (rrc *rollupResultCache) PutSeries(qt *querytracer.Tracer, ec *EvalConfig,
metainfoBuf := bbPool.Get()
defer bbPool.Put(metainfoBuf)
metainfoKey.B = marshalRollupResultCacheKeyForSeries(metainfoKey.B[:0], expr, window, ec.Step, ec.CacheTagFilters)
metainfoKey.B = marshalRollupResultCacheKeyForSeries(metainfoKey.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
metainfoBuf.B = rrc.c.Get(metainfoBuf.B[:0], metainfoKey.B)
var mi rollupResultCacheMetainfo
if len(metainfoBuf.B) > 0 {

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -36,10 +36,10 @@
<meta property="og:title" content="UI for VictoriaMetrics">
<meta property="og:url" content="https://victoriametrics.com/">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<script type="module" crossorigin src="./assets/index-D-ssBbZq.js"></script>
<script type="module" crossorigin src="./assets/index-xmjGcv4-.js"></script>
<link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
<link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
<link rel="stylesheet" crossorigin href="./assets/index-C85_NB5q.css">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>

View File

@@ -1,4 +1,4 @@
FROM golang:1.24.4 AS build-web-stage
FROM golang:1.24.3 AS build-web-stage
COPY build /build
WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
FROM alpine:3.22.0
FROM alpine:3.21.3
USER root
COPY --from=build-web-stage /build/web-amd64 /app/web

View File

@@ -189,36 +189,6 @@ func (tc *TestCase) MustStartVmauth(instance string, flags []string, configFileY
return app
}
// MustStartVmbackup is a test helper that starts an instance of vmbackup
// and waits until the app exits. It fails the test if the app fails to start or
// exits with non zero code.
func (tc *TestCase) MustStartVmbackup(instance, storageDataPath, snapshotCreateURL, dst string) {
tc.t.Helper()
if err := StartVmbackup(instance, storageDataPath, snapshotCreateURL, dst); err != nil {
tc.t.Fatalf("vmbackup %q failed to start or exited with non-zero code: %v", instance, err)
}
// Do not add the process to the list of running apps using
// tc.addApp(instance, app), because the method blocks until the process
// exits.
}
// MustStartVmrestore is a test helper that starts an instance of vmrestore
// and waits until the app exits. It fails the test if the app fails to start or
// exits with non zero code.
func (tc *TestCase) MustStartVmrestore(instance, src, storageDataPath string) {
tc.t.Helper()
if err := StartVmrestore(instance, src, storageDataPath); err != nil {
tc.t.Fatalf("vmrestore %q failed to start or exited with non-zero code: %v", instance, err)
}
// Do not add the process to the list of running apps using
// tc.addApp(instance, app), because the method blocks until the process
// exits.
}
// MustStartDefaultCluster starts a typical cluster configuration with default
// flags.
func (tc *TestCase) MustStartDefaultCluster() *Vmcluster {

View File

@@ -1,239 +0,0 @@
package tests
import (
"fmt"
"path/filepath"
"slices"
"testing"
"time"
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
)
type testBackupRestoreOpts struct {
startSUT func() at.PrometheusWriteQuerier
stopSUT func()
storageDataPaths []string
snapshotCreateURLs func(at.PrometheusWriteQuerier) []string
}
func TestSingleBackupRestore(t *testing.T) {
tc := at.NewTestCase(t)
defer tc.Stop()
storageDataPath := filepath.Join(tc.Dir(), "vmsingle")
opts := testBackupRestoreOpts{
startSUT: func() at.PrometheusWriteQuerier {
return tc.MustStartVmsingle("vmsingle", []string{
"-storageDataPath=" + storageDataPath,
"-retentionPeriod=100y",
"-search.maxStalenessInterval=1m",
})
},
stopSUT: func() {
tc.StopApp("vmsingle")
},
storageDataPaths: []string{
storageDataPath,
},
snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
return []string{
sut.(*at.Vmsingle).SnapshotCreateURL(),
}
},
}
testBackupRestore(tc, opts)
}
func TestClusterBackupRestore(t *testing.T) {
tc := at.NewTestCase(t)
defer tc.Stop()
storage1DataPath := filepath.Join(tc.Dir(), "vmstorage1")
storage2DataPath := filepath.Join(tc.Dir(), "vmstorage2")
opts := testBackupRestoreOpts{
startSUT: func() at.PrometheusWriteQuerier {
return tc.MustStartCluster(&at.ClusterOptions{
Vmstorage1Instance: "vmstorage1",
Vmstorage1Flags: []string{
"-storageDataPath=" + storage1DataPath,
"-retentionPeriod=100y",
},
Vmstorage2Instance: "vmstorage2",
Vmstorage2Flags: []string{
"-storageDataPath=" + storage2DataPath,
"-retentionPeriod=100y",
},
VminsertInstance: "vminsert",
VminsertFlags: []string{},
VmselectInstance: "vmselect",
VmselectFlags: []string{
"-search.maxStalenessInterval=1m",
},
})
},
stopSUT: func() {
tc.StopApp("vminsert")
tc.StopApp("vmselect")
tc.StopApp("vmstorage1")
tc.StopApp("vmstorage2")
},
storageDataPaths: []string{
storage1DataPath,
storage2DataPath,
},
snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
c := sut.(*at.Vmcluster)
return []string{
c.Vmstorages[0].SnapshotCreateURL(),
c.Vmstorages[1].SnapshotCreateURL(),
}
},
}
testBackupRestore(tc, opts)
}
func testBackupRestore(tc *at.TestCase, opts testBackupRestoreOpts) {
t := tc.T()
const msecPerMinute = 60 * 1000
genData := func(count int, prefix string, start int64) (recs []string, wantSeries []map[string]string, wantQueryResults []*at.QueryResult) {
recs = make([]string, count)
wantSeries = make([]map[string]string, count)
wantQueryResults = make([]*at.QueryResult, count)
for i := range count {
name := fmt.Sprintf("%s_%03d", prefix, i)
value := float64(i)
timestamp := start + int64(i)*msecPerMinute
recs[i] = fmt.Sprintf("%s %f %d", name, value, timestamp)
wantSeries[i] = map[string]string{"__name__": name}
wantQueryResults[i] = &at.QueryResult{
Metric: map[string]string{"__name__": name},
Samples: []*at.Sample{{Timestamp: timestamp, Value: value}},
}
}
return recs, wantSeries, wantQueryResults
}
backupBaseDir, err := filepath.Abs(filepath.Join(tc.Dir(), "backups"))
if err != nil {
t.Fatalf("could not get absolute path for the backup base dir")
}
// assertSeries retrieves set of all metric names from the storage and
// compares it with the expected set.
assertSeries := func(app at.PrometheusQuerier, query string, start, end int64, want []map[string]string) {
t.Helper()
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
return app.PrometheusAPIV1Series(t, query, at.QueryOpts{
Start: fmt.Sprintf("%d", start),
End: fmt.Sprintf("%d", end),
}).Sort()
},
Want: &at.PrometheusAPIV1SeriesResponse{
Status: "success",
Data: want,
},
FailNow: true,
})
}
// assertSeries retrieves all data from the storage and compares it with the
// expected result.
assertQueryResults := func(app at.PrometheusQuerier, query string, start, end int64, want []*at.QueryResult) {
t.Helper()
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query_range response",
Got: func() any {
return app.PrometheusAPIV1QueryRange(t, query, at.QueryOpts{
Start: fmt.Sprintf("%d", start),
End: fmt.Sprintf("%d", end),
Step: "60s",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
Result: want,
},
},
FailNow: true,
Retries: 300,
})
}
createBackup := func(sut at.PrometheusWriteQuerier, name string) {
for i, storageDataPath := range opts.storageDataPaths {
replica := fmt.Sprintf("replica-%d", i)
instance := fmt.Sprintf("vmbackup-%s-%s", name, replica)
snapshotCreateURL := opts.snapshotCreateURLs(sut)[i]
backupPath := "fs://" + filepath.Join(backupBaseDir, name, replica)
tc.MustStartVmbackup(instance, storageDataPath, snapshotCreateURL, backupPath)
}
}
restoreFromBackup := func(name string) {
for i, storageDataPath := range opts.storageDataPaths {
replica := fmt.Sprintf("replica-%d", i)
instance := fmt.Sprintf("vmrestore-%s-%s", name, replica)
backupPath := "fs://" + filepath.Join(backupBaseDir, name, replica)
tc.MustStartVmrestore(instance, backupPath, storageDataPath)
}
}
// Use the same number of metrics and time range for all the data ingestions
// below.
const numMetrics = 1000
// With 1000 metrics (one per minute), the time range spans 2 months.
end := time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC).UnixMilli()
start := end - numMetrics*msecPerMinute
// Verify backup/restore:
//
// - Start vmsingle with empty storage data dir.
// - Ingest first batch or records (batch1) and ensure they can be queried.
// - Create batch1 backup
// - Ingest second batch of records (batch2) and ensure the queries return
// (batch1 + batch2) data.
// - Stop vmsingle
// - Restore batch1 from backup
// - Start vmsingle
// - Ensure that the queries return batch1 data only.
batch1Data, wantBatch1Series, wantBatch1QueryResults := genData(numMetrics, "batch1", start)
batch2Data, wantBatch2Series, wantBatch2QueryResults := genData(numMetrics, "batch2", start)
wantBatch12Series := slices.Concat(wantBatch1Series, wantBatch2Series)
wantBatch12QueryResults := slices.Concat(wantBatch1QueryResults, wantBatch2QueryResults)
sut := opts.startSUT()
sut.PrometheusAPIV1ImportPrometheus(t, batch1Data, at.QueryOpts{})
sut.ForceFlush(t)
assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
createBackup(sut, "batch1")
sut.PrometheusAPIV1ImportPrometheus(t, batch2Data, at.QueryOpts{})
sut.ForceFlush(t)
assertSeries(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12Series)
assertQueryResults(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12QueryResults)
createBackup(sut, "batch12")
opts.stopSUT()
restoreFromBackup("batch1")
sut = opts.startSUT()
assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
}

View File

@@ -1,78 +0,0 @@
package tests
import (
"os"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
)
func TestClusterRollupResultCache(t *testing.T) {
os.RemoveAll(t.Name())
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
tc := apptest.NewTestCase(t)
defer tc.Stop()
vmstorage := tc.MustStartVmstorage("vmstorage", []string{
"-storageDataPath=" + tc.Dir() + "/vmstorage",
"-retentionPeriod=100y",
})
vminsert := tc.MustStartVminsert("vminsert", []string{
"-storageNode=" + vmstorage.VminsertAddr(),
})
vmselect := tc.MustStartVmselect("vmselect", []string{
"-storageNode=" + vmstorage.VmselectAddr(),
"-search.tenantCacheExpireDuration=0",
})
var tenantLabelsSamples = []string{
`foo_bar{vm_account_id="5"} 1.00 1652169720000`, // 2022-05-10T08:00:00Z'
`foo_bar{vm_account_id="5",vm_project_id="15"} 3.00 1652169720000`, // 2022-05-10T08:02:00Z
}
vminsert.PrometheusAPIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
vmstorage.ForceFlush(t)
want := apptest.NewPrometheusAPIV1QueryResponse(t,
`{"data":
{"result":[
{"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id": "0"},"values":[[1652169720,"1"],[1652169780,"1"]]},
{"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id":"15"},"values":[[1652169720,"3"],[1652169780,"3"]]}
]
}
}`,
)
got := vmselect.PrometheusAPIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
Tenant: "multitenant",
Start: "2022-05-10T07:59:00.000Z",
End: "2022-05-10T08:05:00.000Z",
Step: "1m",
ExtraFilters: []string{`{vm_account_id="5",vm_project_id="15"}`, `{vm_account_id="5",vm_project_id="0"}`},
})
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
want = apptest.NewPrometheusAPIV1QueryResponse(t,
`{"data":
{"result":[]}
}`,
)
got = vmselect.PrometheusAPIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
Tenant: "multitenant",
Start: "2022-05-10T07:59:00.000Z",
End: "2022-05-10T08:05:00.000Z",
Step: "1m",
ExtraFilters: []string{`{vm_account_id="99",vm_project_id="99"}`},
})
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
}

View File

@@ -1,13 +0,0 @@
package apptest
// StartVmbackup starts an instance of vmbackup with the given flags and waits
// until it exits.
func StartVmbackup(instance, storageDataPath, snapshotCreateURL, dst string) error {
flags := []string{
"-storageDataPath=" + storageDataPath,
"-snapshot.createURL=" + snapshotCreateURL,
"-dst=" + dst,
}
_, _, err := startApp(instance, "../../bin/vmbackup", flags, &appOptions{wait: true})
return err
}

View File

@@ -1,12 +0,0 @@
package apptest
// StartVmrestore starts an instance of vmrestore with the given flags and waits
// until it exits.
func StartVmrestore(instance, src, storageDataPath string) error {
flags := []string{
"-src=" + src,
"-storageDataPath=" + storageDataPath,
}
_, _, err := startApp(instance, "../../bin/vmrestore", flags, &appOptions{wait: true})
return err
}

View File

@@ -363,7 +363,8 @@ func (app *Vmsingle) APIV1AdminStatusMetricNamesStatsReset(t *testing.T, opts Qu
func (app *Vmsingle) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
t.Helper()
data, statusCode := app.cli.Post(t, app.SnapshotCreateURL(), "", nil)
queryURL := fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
data, statusCode := app.cli.Post(t, queryURL, "", nil)
if got, want := statusCode, http.StatusOK; got != want {
t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
}
@@ -376,11 +377,6 @@ func (app *Vmsingle) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
return &res
}
// SnapshotCreateURL returns the URL for creating snapshots.
func (app *Vmsingle) SnapshotCreateURL() string {
return fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
}
// APIV1AdminTSDBSnapshot creates a database snapshot by sending a query to the
// /api/v1/admin/tsdb/snapshot endpoint.
//

View File

@@ -99,7 +99,8 @@ func (app *Vmstorage) ForceMerge(t *testing.T) {
func (app *Vmstorage) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
t.Helper()
data, statusCode := app.cli.Post(t, app.SnapshotCreateURL(), "", nil)
queryURL := fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
data, statusCode := app.cli.Post(t, queryURL, "", nil)
if got, want := statusCode, http.StatusOK; got != want {
t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
}
@@ -112,11 +113,6 @@ func (app *Vmstorage) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
return &res
}
// SnapshotCreateURL returns the URL for creating snapshots.
func (app *Vmstorage) SnapshotCreateURL() string {
return fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
}
// SnapshotList lists existing database snapshots by sending a query to the
// /snapshot/list endpoint.
//

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -3,11 +3,11 @@
DOCKER_REGISTRIES ?= docker.io quay.io
DOCKER_NAMESPACE ?= victoriametrics
ROOT_IMAGE ?= alpine:3.22.0
ROOT_IMAGE ?= alpine:3.21.3
ROOT_IMAGE_SCRATCH ?= scratch
CERTS_IMAGE := alpine:3.22.0
CERTS_IMAGE := alpine:3.21.3
GO_BUILDER_IMAGE := golang:1.24.4-alpine
GO_BUILDER_IMAGE := golang:1.24.3-alpine
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
DOCKER ?= docker

View File

@@ -6,7 +6,7 @@ RUN apk add git gcc musl-dev make wget --no-cache && \
cd /opt/cross-builder && \
for arch in aarch64 x86_64; do \
wget \
https://github.com/VictoriaMetrics/muslcc-mirror/releases/download/v1.0.0/${arch}-linux-musl-cross.tgz \
https://musl.cc/${arch}-linux-musl-cross.tgz \
-O /opt/cross-builder/${arch}-musl.tgz \
--no-verbose && \
tar zxf ${arch}-musl.tgz -C ./ && \

View File

@@ -68,7 +68,7 @@ services:
# VictoriaMetrics instance, a single process responsible for
# scraping, storing metrics and serve read requests.
victoriametrics:
image: victoriametrics/victoria-metrics:v1.119.0
image: victoriametrics/victoria-metrics:v1.118.0
volumes:
- vmdata:/storage
- ./prometheus-vl-cluster.yml:/etc/prometheus/prometheus.yml
@@ -81,7 +81,7 @@ services:
# It proxies query requests from vmalert to either VictoriaMetrics or VictoriaLogs,
# depending on the requested path.
vmauth:
image: victoriametrics/vmauth:v1.119.0
image: victoriametrics/vmauth:v1.118.0
depends_on:
- "victoriametrics"
- "vlselect-1"
@@ -97,7 +97,7 @@ services:
# vmalert executes alerting and recording rules according to given rule type.
vmalert:
image: victoriametrics/vmalert:v1.119.0
image: victoriametrics/vmalert:v1.118.0
depends_on:
- "vmauth"
- "alertmanager"

View File

@@ -49,7 +49,7 @@ services:
# VictoriaMetrics instance, a single process responsible for
# scraping, storing metrics and serve read requests.
victoriametrics:
image: victoriametrics/victoria-metrics:v1.119.0
image: victoriametrics/victoria-metrics:v1.118.0
ports:
- "8428:8428"
volumes:
@@ -64,7 +64,7 @@ services:
# It proxies query requests from vmalert to either VictoriaMetrics or VictoriaLogs,
# depending on the requested path.
vmauth:
image: victoriametrics/vmauth:v1.119.0
image: victoriametrics/vmauth:v1.118.0
depends_on:
- "victoriametrics"
- "victorialogs"
@@ -78,7 +78,7 @@ services:
# vmalert executes alerting and recording rules according to the given rule type.
vmalert:
image: victoriametrics/vmalert:v1.119.0
image: victoriametrics/vmalert:v1.118.0
depends_on:
- "vmauth"
- "alertmanager"

View File

@@ -3,7 +3,7 @@ services:
# It scrapes targets defined in --promscrape.config
# And forward them to --remoteWrite.url
vmagent:
image: victoriametrics/vmagent:v1.119.0
image: victoriametrics/vmagent:v1.118.0
depends_on:
- "vmauth"
ports:
@@ -35,14 +35,14 @@ services:
# vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
# where N is number of vmstorages (2 in this case).
vmstorage-1:
image: victoriametrics/vmstorage:v1.119.0-cluster
image: victoriametrics/vmstorage:v1.118.0-cluster
volumes:
- strgdata-1:/storage
command:
- "--storageDataPath=/storage"
restart: always
vmstorage-2:
image: victoriametrics/vmstorage:v1.119.0-cluster
image: victoriametrics/vmstorage:v1.118.0-cluster
volumes:
- strgdata-2:/storage
command:
@@ -52,7 +52,7 @@ services:
# vminsert is ingestion frontend. It receives metrics pushed by vmagent,
# pre-process them and distributes across configured vmstorage shards.
vminsert-1:
image: victoriametrics/vminsert:v1.119.0-cluster
image: victoriametrics/vminsert:v1.118.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -61,7 +61,7 @@ services:
- "--storageNode=vmstorage-2:8400"
restart: always
vminsert-2:
image: victoriametrics/vminsert:v1.119.0-cluster
image: victoriametrics/vminsert:v1.118.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -73,7 +73,7 @@ services:
# vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
# vmselect collects results from configured `--storageNode` shards.
vmselect-1:
image: victoriametrics/vmselect:v1.119.0-cluster
image: victoriametrics/vmselect:v1.118.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -83,7 +83,7 @@ services:
- "--vmalert.proxyURL=http://vmalert:8880"
restart: always
vmselect-2:
image: victoriametrics/vmselect:v1.119.0-cluster
image: victoriametrics/vmselect:v1.118.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -98,7 +98,7 @@ services:
# read requests from Grafana, vmui, vmalert among vmselects.
# It can be used as an authentication proxy.
vmauth:
image: victoriametrics/vmauth:v1.119.0
image: victoriametrics/vmauth:v1.118.0
depends_on:
- "vmselect-1"
- "vmselect-2"
@@ -112,7 +112,7 @@ services:
# vmalert executes alerting and recording rules
vmalert:
image: victoriametrics/vmalert:v1.119.0
image: victoriametrics/vmalert:v1.118.0
depends_on:
- "vmauth"
ports:

View File

@@ -3,7 +3,7 @@ services:
# It scrapes targets defined in --promscrape.config
# And forward them to --remoteWrite.url
vmagent:
image: victoriametrics/vmagent:v1.119.0
image: victoriametrics/vmagent:v1.118.0
depends_on:
- "victoriametrics"
ports:
@@ -18,7 +18,7 @@ services:
# VictoriaMetrics instance, a single process responsible for
# storing metrics and serve read requests.
victoriametrics:
image: victoriametrics/victoria-metrics:v1.119.0
image: victoriametrics/victoria-metrics:v1.118.0
ports:
- 8428:8428
- 8089:8089
@@ -54,7 +54,7 @@ services:
# vmalert executes alerting and recording rules
vmalert:
image: victoriametrics/vmalert:v1.119.0
image: victoriametrics/vmalert:v1.118.0
depends_on:
- "victoriametrics"
- "alertmanager"

View File

@@ -19,7 +19,7 @@ services:
retries: 10
dd-proxy:
image: docker.io/victoriametrics/vmauth:v1.119.0
image: docker.io/victoriametrics/vmauth:v1.118.0
restart: on-failure
volumes:
- ./:/etc/vmauth

View File

@@ -1,6 +1,6 @@
services:
vmagent:
image: victoriametrics/vmagent:v1.119.0
image: victoriametrics/vmagent:v1.118.0
depends_on:
- "victoriametrics"
ports:
@@ -14,7 +14,7 @@ services:
restart: always
victoriametrics:
image: victoriametrics/victoria-metrics:v1.119.0
image: victoriametrics/victoria-metrics:v1.118.0
ports:
- 8428:8428
volumes:
@@ -40,7 +40,7 @@ services:
restart: always
vmalert:
image: victoriametrics/vmalert:v1.119.0
image: victoriametrics/vmalert:v1.118.0
depends_on:
- "victoriametrics"
ports:
@@ -59,7 +59,7 @@ services:
- '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
restart: always
vmanomaly:
image: victoriametrics/vmanomaly:v1.23.2
image: victoriametrics/vmanomaly:v1.23.0
depends_on:
- "victoriametrics"
ports:

View File

@@ -18,7 +18,7 @@ services:
- vlogs
generator:
image: golang:1.24.4-alpine
image: golang:1.24.3-alpine
restart: always
working_dir: /go/src/app
volumes:

View File

@@ -2,7 +2,7 @@ version: "3"
services:
generator:
image: golang:1.24.4-alpine
image: golang:1.24.3-alpine
restart: always
working_dir: /go/src/app
volumes:

View File

@@ -14,20 +14,6 @@ aliases:
---
Please find the changelog for VictoriaMetrics Anomaly Detection below.
## v1.23.2
Released: 2025-06-09
- IMPROVEMENT: Increased convergence speed for [OnlineZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score), [ZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#z-score), [MADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#mad), and [OnlineMADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) models. Now it works better for tight optimization budgets (n_trials < 10, timeout < 1s)
- BUGFIX: Now mean and variance of [OnlineZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) with exponential `decay` < 1 [arg](https://docs.victoriametrics.com/anomaly-detection/components/models/#decay) are properly calculated for unbiased predictions.
## v1.23.1
Released: 2025-06-08
- BUGFIX: In [sharding mode](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#horizontal-scalability) the corner case when shard number (`VMANOMALY_MEMBER_NUM`) is greater than the number of configured shards (`VMANOMALY_MEMBERS_COUNT`) is now properly handled.
- BUGFIX: In [sharding mode](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#horizontal-scalability), the corner case when the number of produced [sub-configurations](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#sub-configuration) is less than the number of configured shards (`VMANOMALY_MEMBERS_COUNT`) is now properly handled. Until config hot-reload is supported, such "idle" shards will be turned off with exit code 1 and respective critical message logged.
## v1.23.0
Released: 2025-06-05

View File

@@ -224,7 +224,7 @@ services:
# ...
vmanomaly:
container_name: vmanomaly
image: victoriametrics/vmanomaly:v1.23.2
image: victoriametrics/vmanomaly:v1.23.0
# ...
ports:
- "8490:8490"
@@ -432,7 +432,7 @@ options:
Heres an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
```sh
docker pull victoriametrics/vmanomaly:v1.23.2 && docker image tag victoriametrics/vmanomaly:v1.23.2 vmanomaly
docker pull victoriametrics/vmanomaly:v1.23.0 && docker image tag victoriametrics/vmanomaly:v1.23.0 vmanomaly
```
```sh

View File

@@ -118,13 +118,13 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
1. Pull Docker image:
```sh
docker pull victoriametrics/vmanomaly:v1.23.2
docker pull victoriametrics/vmanomaly:v1.23.0
```
2. (Optional step) tag the `vmanomaly` Docker image:
```sh
docker image tag victoriametrics/vmanomaly:v1.23.2 vmanomaly
docker image tag victoriametrics/vmanomaly:v1.23.0 vmanomaly
```
3. Start the `vmanomaly` Docker container with a *license file*, use the command below.
@@ -158,7 +158,7 @@ docker run -it --user 1000:1000 \
services:
# ...
vmanomaly:
image: victoriametrics/vmanomaly:v1.23.2
image: victoriametrics/vmanomaly:v1.23.0
volumes:
$YOUR_LICENSE_FILE_PATH:/license
$YOUR_CONFIG_FILE_PATH:/config.yml

View File

@@ -1276,7 +1276,7 @@ monitoring:
Let's pull the docker image for `vmanomaly`:
```sh
docker pull victoriametrics/vmanomaly:v1.23.2
docker pull victoriametrics/vmanomaly:v1.23.0
```
Now we can run the docker container putting as volumes both config and model file:
@@ -1290,7 +1290,7 @@ docker run -it \
-v $(PWD)/license:/license \
-v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
-v $(PWD)/custom.yaml:/config.yaml \
victoriametrics/vmanomaly:v1.23.2 /config.yaml \
victoriametrics/vmanomaly:v1.23.0 /config.yaml \
--licenseFile=/license
```

View File

@@ -2,9 +2,9 @@
- To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
- In the tutorial, we'll be using the following VictoriaMetrics components:
- [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.119.0)
- [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.119.0)
- [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.119.0)
- [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.118.0)
- [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.118.0)
- [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.118.0)
- [Grafana](https://grafana.com/) (v.10.2.1)
- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.27.0)
@@ -315,7 +315,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
services:
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.119.0
image: victoriametrics/vmagent:v1.118.0
depends_on:
- "victoriametrics"
ports:
@@ -332,7 +332,7 @@ services:
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.119.0
image: victoriametrics/victoria-metrics:v1.118.0
ports:
- 8428:8428
volumes:
@@ -365,7 +365,7 @@ services:
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.119.0
image: victoriametrics/vmalert:v1.118.0
depends_on:
- "victoriametrics"
ports:
@@ -387,7 +387,7 @@ services:
restart: always
vmanomaly:
container_name: vmanomaly
image: victoriametrics/vmanomaly:v1.23.2
image: victoriametrics/vmanomaly:v1.23.0
depends_on:
- "victoriametrics"
ports:

View File

@@ -241,27 +241,27 @@ services:
- grafana_data:/var/lib/grafana/
vmsingle:
image: victoriametrics/victoria-metrics:v1.119.0
image: victoriametrics/victoria-metrics:v1.118.0
command:
- -httpListenAddr=0.0.0.0:8429
vmstorage:
image: victoriametrics/vmstorage:v1.119.0-cluster
image: victoriametrics/vmstorage:v1.118.0-cluster
vminsert:
image: victoriametrics/vminsert:v1.119.0-cluster
image: victoriametrics/vminsert:v1.118.0-cluster
command:
- -storageNode=vmstorage:8400
- -httpListenAddr=0.0.0.0:8480
vmselect:
image: victoriametrics/vmselect:v1.119.0-cluster
image: victoriametrics/vmselect:v1.118.0-cluster
command:
- -storageNode=vmstorage:8401
- -httpListenAddr=0.0.0.0:8481
vmagent:
image: victoriametrics/vmagent:v1.119.0
image: victoriametrics/vmagent:v1.118.0
volumes:
- ./scrape.yaml:/etc/vmagent/config.yaml
command:
@@ -270,7 +270,7 @@ services:
- -remoteWrite.url=http://vmsingle:8429/api/v1/write
vmgateway-cluster:
image: victoriametrics/vmgateway:v1.119.0-enterprise
image: victoriametrics/vmgateway:v1.118.0-enterprise
ports:
- 8431:8431
volumes:
@@ -286,7 +286,7 @@ services:
- -auth.oidcDiscoveryEndpoints=http://keycloak:8080/realms/master/.well-known/openid-configuration
vmgateway-single:
image: victoriametrics/vmgateway:v1.119.0-enterprise
image: victoriametrics/vmgateway:v1.118.0-enterprise
ports:
- 8432:8431
volumes:
@@ -397,7 +397,7 @@ Once iDP configuration is done, vmagent configuration needs to be updated to use
```yaml
vmagent:
image: victoriametrics/vmagent:v1.119.0
image: victoriametrics/vmagent:v1.118.0
volumes:
- ./scrape.yaml:/etc/vmagent/config.yaml
- ./vmagent-client-secret:/etc/vmagent/oauth2-client-secret

View File

@@ -954,7 +954,6 @@ There are three modes of displaying query results:
- `Group` - results are displayed as a table with rows grouped by [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
- `Table` - displays query results as a table.
- `JSON` - displays raw JSON response from [`/select/logsql/query` HTTP API](#querying-logs).
- `Live` - displays [live tailing](#live-tailing) results for the given query.
See also [command line interface](#command-line).

View File

@@ -57,9 +57,9 @@ and performing [regular upgrades](https://docs.victoriametrics.com/victoriametri
Download the newest available [VictoriaMetrics release](https://docs.victoriametrics.com/victoriametrics/changelog/)
from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
```sh
docker pull victoriametrics/victoria-metrics:v1.119.0
docker pull victoriametrics/victoria-metrics:v1.118.0
docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
victoriametrics/victoria-metrics:v1.119.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
victoriametrics/victoria-metrics:v1.118.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
```
_For Enterprise images see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._

View File

@@ -127,7 +127,7 @@ Bumping the limits may significantly improve build speed.
```
1. Bump VictoriaMetrics version mentioned in [docs](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7388).
1. Follow the instructions in [release follow-up](https://github.com/VictoriaMetrics/VictoriaMetrics-enterprise/blob/enterprise-single-node/Release-Guide.md).
1. Follow the instructions in [release follow-up](https://github.com/VictoriaMetrics/VictoriaMetrics-enterprise/blob/master/Release-Guide.md).
## Operator

View File

@@ -18,15 +18,6 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
## tip
* SECURITY: upgrade Go builder from Go1.24.3 to Go1.24.4. See [the list of issues addressed in Go1.24.4](https://github.com/golang/go/issues?q=milestone%3AGo1.24.4+label%3ACherryPickApproved).
* SECURITY: upgrade base docker image (Alpine) from 3.21.3 to 3.22.0. See [Alpine 3.22.0 release notes](https://alpinelinux.org/posts/Alpine-3.22.0-released.html).
* FEATURE: [dashboards/cluster](https://grafana.com/grafana/dashboards/11176): add panel `Partitions scheduled for re-processing` to `Troubleshooting` row. It shows the amount of data scheduled for [downsampling](https://docs.victoriametrics.com/#downsampling) or [retention filters](https://docs.victoriametrics.com/#retention-filters). The new panel should help to correlate resource usage with background re-processing of partitions.
## [v1.119.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.119.0)
Released at 2025-06-06
* FEATURE: improve performance on systems with many CPU cores by removing the top sources of [false sharing](https://en.wikipedia.org/wiki/False_sharing) at global variables. See [#8682](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8682). Thanks to @tIGO for raising this issue and for the initial attempt to fix it at the [PR #8683](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8683).
* FEATURE: [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/): add an option to use mTLS for connections to `-write.url` and `-read.url`. See [#8841](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8841).
* FEATURE: [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): dynamically adjusts the concurrent dial limit between 8 and 64 based on `-search.maxConcurrentRequests`. Additionally, goroutines now have the opportunity to access available connections while awaiting the dial limit token. This enables faster connection establishment when sudden requests arrive and reduces the blocking time during the availability check for connections. See [#8922](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8922)
@@ -47,7 +38,6 @@ Released at 2025-06-06
* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): enable dual-stack network mode for `vmctl`connections by default. This allows connecting to IPv6 endpoints as it was not possible previously. See [#9116](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9116).
* BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): fix access conflicts for the temporary test folder when multiple users run tests on the same host. Thanks to @evkuzin for the [PR 9015](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9015).
* BUGFIX: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): fix the alerting rule `ScrapePoolHasNoTargets`. Previously, it may cause false positive in [sharding mode](https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets).
* BUGFIX: [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): include full list of query filters to build a cache key for [multitenant read](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels) queries. Previously, cache key did not include tenancy related labels so cache entries could be shared between tenants breaking tenant isolation. See [#9002](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9002) for details.
## [v1.118.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.118.0)
@@ -288,23 +278,6 @@ Released at 2025-02-10
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix discrepancies when using `or` binary operator. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7759) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7640) issues for details.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly update number of unique series for [cardinality limiter](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter) on ingestion. Previously, limit could undercount the real number of the ingested unique series.
## [v1.110.10](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.10)
Released at 2025-06-06
**v1.110.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.110.x line will be supported for at least 12 months since [v1.110.0](https://docs.victoriametrics.com/changelog/#v11100) release**
* BUGFIX: [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/): add missing vmselect `vmui` related routes to the authorized requests routing. See [9003](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9003) issue for details.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): fixed a regression in downsampling logic introduced in [#7440](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7440) and released in [v1.106.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.0), where downsampling rules with filters `filter:offset:interval` could be incorrectly skipped in favor of unfiltered rules `offset:interval`. See [#8969](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8969).
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `rententionFilter` on flag value changes. Previously, it ignored any `filter` value changes for historical data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8885) for details.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): schedule a single background merge thread for merging historical data. See [4592](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4592) issue for details.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): prevent panic caused by invalid label name in metric relabeling debugging interface. Error is now properly propagated and displayed in the interface. See [#8661](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8661).
* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): enable dual-stack network mode for `vmctl`connections by default. This allows connecting to IPv6 endpoints as it was not possible previously. See [#9116](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9116).
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmrestore](https://docs.victoriametrics.com/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): improve resilience to network issues by retrying requests failing due to `IncompleteBody`. Previously, such requests were not retried and leaded to restore/backup process failure. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8639) and [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8547) for details.
* BUGFIX: [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): include full list of query filters to build a cache key for [multitenant read](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels) queries. Previously, cache key did not include tenancy related labels so cache entries could be shared between tenants breaking tenant isolation. See [#9002](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9002) for details.
## [v1.110.9](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.9)
Released at 2025-05-23
@@ -541,20 +514,6 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2024/#v11030)
## [v1.102.23](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.23)
Released at 2025-06-06
**v1.102.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.102.x line will be supported for at least 12 months since [v1.102.0](https://docs.victoriametrics.com/changelog/#v11020) release**
* BUGFIX: [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/): add missing vmselect `vmui` related routes to the authorized requests routing. See [9003](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9003) issue for details.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): prevent panic caused by invalid label name in metric relabeling debugging interface. Error is now properly propagated and displayed in the interface. See [#8661](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8661).
* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): enable dual-stack network mode for `vmctl`connections by default. This allows connecting to IPv6 endpoints as it was not possible previously. See [#9116](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9116).
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmrestore](https://docs.victoriametrics.com/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): use exponential backoff for retries when uploading or downloading data from S3. This should reduce the number of failed uploads and downloads when S3 is temporarily unavailable. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8639).
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmrestore](https://docs.victoriametrics.com/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): improve resilience to network issues by retrying requests failing due to `IncompleteBody`. Previously, such requests were not retried and leaded to restore/backup process failure. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8639) and [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8547) for details.
## [v1.102.22](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.22)
Released at 2025-05-23

View File

@@ -89,7 +89,7 @@ VictoriaMetrics Enterprise components are available in the following forms:
It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
Binary releases of VictoriaMetrics Enterprise are available [at the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz`.
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.118.0-enterprise.tar.gz`.
In order to run binary release of VictoriaMetrics Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
from the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) and unpack it. Then run the unpacked binary.
@@ -107,8 +107,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
```sh
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.119.0/victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz
tar -xzf victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.118.0/victoria-metrics-linux-amd64-v1.118.0-enterprise.tar.gz
tar -xzf victoria-metrics-linux-amd64-v1.118.0-enterprise.tar.gz
./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
```
@@ -123,7 +123,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
Docker images for VictoriaMetrics Enterprise are available at VictoriaMetrics [Docker Hub](https://hub.docker.com/u/victoriametrics) and [Quay](https://quay.io/organization/victoriametrics).
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.119.0-enterprise`.
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.118.0-enterprise`.
In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via command-line
flag as described [here](#binary-releases).
@@ -133,13 +133,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
```sh
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.119.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.118.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
```
Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
```sh
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.119.0-enterprise -licenseFile=/path/to/vm-license
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.118.0-enterprise -licenseFile=/path/to/vm-license
```
Example docker-compose configuration:
@@ -148,7 +148,7 @@ version: "3.5"
services:
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.119.0
image: victoriametrics/victoria-metrics:v1.118.0
ports:
- 8428:8428
volumes:
@@ -180,7 +180,7 @@ is used to provide key in plain-text:
```yaml
server:
image:
tag: v1.119.0-enterprise
tag: v1.118.0-enterprise
license:
key: {BASE64_ENCODED_LICENSE_KEY}
@@ -191,7 +191,7 @@ In order to provide key via existing secret, the following values file is used:
```yaml
server:
image:
tag: v1.119.0-enterprise
tag: v1.118.0-enterprise
license:
secret:
@@ -240,7 +240,7 @@ spec:
license:
key: {BASE64_ENCODED_LICENSE_KEY}
image:
tag: v1.119.0-enterprise
tag: v1.118.0-enterprise
```
In order to provide key via existing secret, the following custom resource is used:
@@ -257,7 +257,7 @@ spec:
name: vm-license
key: license
image:
tag: v1.119.0-enterprise
tag: v1.118.0-enterprise
```
Example secret with license key:
@@ -283,10 +283,10 @@ See full list of CRD specifications [here](https://docs.victoriametrics.com/oper
### FIPS compliance
VictoriaMetrics Enterprise components can be run in FIPS compliant mode {{% available_from "v1.118.0" %}}. Binary releases and Docker images
of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz`
of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.115.0-enterprise.tar.gz`
archive includes `victoria-metrics-prod` and `victoria-metrics-fips` binaries. The latter binary is FIPS compliant.
Docker images of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.119.0-enterprise-fips`
Docker images of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.118.0-enterprise-fips`
image uses FIPS compliant binary version.
## Monitoring license expiration

View File

@@ -12,7 +12,8 @@ aliases:
---
## Goals
1. The main goal - **to help users and [clients](https://docs.victoriametrics.com/victoriametrics/enterprise/) using VictoriaMetrics and VictoriaLogs component in the most efficient way**.
1. The main goal - **to help users and [clients](https://docs.victoriametrics.com/victoriametrics/enterprise/) resolving issues with VictoriaMetrics components,
so they could use these components in the most efficient way**.
1. Fixing bugs in the essential functionality of VictoriaMetrics components. Small usability bugs are usually the most annoying,
so they **must be fixed first**.
1. Improving [docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs) for VictoriaMetrics components,

View File

@@ -36,8 +36,8 @@ scrape_configs:
After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/) to the same directory:
```
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.119.0/victoria-metrics-linux-amd64-v1.119.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.119.0.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.118.0/victoria-metrics-linux-amd64-v1.118.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.118.0.tar.gz
```
Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -152,8 +152,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/) acco
```yaml
# Download and unpack single-node VictoriaMetrics
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.119.0/victoria-metrics-linux-amd64-v1.119.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.119.0.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.118.0/victoria-metrics-linux-amd64-v1.118.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.118.0.tar.gz
# Run single-node VictoriaMetrics with the given scrape.yaml
./victoria-metrics-prod -promscrape.config=scrape.yaml

6
go.mod
View File

@@ -1,6 +1,6 @@
module github.com/VictoriaMetrics/VictoriaMetrics
go 1.24.4
go 1.24.3
// This is needed in order to avoid vmbackup and vmrestore binary size increase by 20MB
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8008
@@ -135,7 +135,9 @@ require (
github.com/prometheus/sigv4 v0.2.0 // indirect
github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/stretchr/testify v1.10.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
@@ -180,3 +182,5 @@ require (
k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
tool github.com/valyala/quicktemplate/qtc

8
go.sum
View File

@@ -332,14 +332,14 @@ github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.32 h1:4+LP7qmsLSGbmc66m1s5dKRMBwztRppfxFKlYqYte/c=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.32/go.mod h1:kzh+BSAvpoyHHdHBCDhmSWtBc1NbLMZ2lWHqnBoxFks=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=

View File

@@ -126,24 +126,10 @@ func isASCII(s string) bool {
}
func isTokenChar(c byte) bool {
return tokenCharTable[c] != 0
return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_'
}
var tokenCharTable = func() *[256]byte {
var a [256]byte
for c := uint(0); c < 256; c++ {
if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_' {
a[c] = 1
}
}
return &a
}()
func isTokenRune(c rune) bool {
if c < utf8.RuneSelf {
// Fast path - the char is ASCII
return isTokenChar(byte(c))
}
return unicode.IsLetter(c) || unicode.IsDigit(c) || c == '_'
}

View File

@@ -3,7 +3,6 @@ package logstorage
import (
"strings"
"testing"
"unicode/utf8"
)
func BenchmarkTokenizeStrings(b *testing.B) {
@@ -18,55 +17,3 @@ func BenchmarkTokenizeStrings(b *testing.B) {
}
})
}
func BenchmarkIsTokenChar(b *testing.B) {
b.ReportAllocs()
b.SetBytes(int64(len(benchLogs)))
b.RunParallel(func(pb *testing.PB) {
n := 0
for pb.Next() {
for i := range benchLogs {
ch := benchLogs[i]
if isTokenChar(ch) {
n++
}
}
}
GlobalSink.Add(uint64(n))
})
}
func BenchmarkIsTokenRune(b *testing.B) {
b.Run("ascii", func(b *testing.B) {
benchmarkIsTokenRune(b, benchLogs)
})
var buf []byte
for i, ch := range benchLogs {
if i%10 == 0 {
ch += 1024
}
buf = utf8.AppendRune(buf, ch)
}
benchLogsUnicode := string(buf)
b.Run("unicode", func(b *testing.B) {
benchmarkIsTokenRune(b, benchLogsUnicode)
})
}
func benchmarkIsTokenRune(b *testing.B, s string) {
b.ReportAllocs()
b.SetBytes(int64(len(s)))
b.RunParallel(func(pb *testing.PB) {
n := 0
for pb.Next() {
for _, ch := range s {
if isTokenRune(ch) {
n++
}
}
}
GlobalSink.Add(uint64(n))
})
}

View File

@@ -11,10 +11,10 @@ type WriteRequest struct {
Timeseries []TimeSeries
}
func (m *WriteRequest) marshalToSizedBuffer(dst []byte) (int, error) {
func (m *WriteRequest) MarshalToSizedBuffer(dst []byte) (int, error) {
i := len(dst)
for j := len(m.Timeseries) - 1; j >= 0; j-- {
size, err := m.Timeseries[j].marshalToSizedBuffer(dst[:i])
size, err := m.Timeseries[j].MarshalToSizedBuffer(dst[:i])
if err != nil {
return 0, err
}
@@ -37,12 +37,12 @@ func encodeVarint(dst []byte, offset int, v uint64) int {
dst[offset] = uint8(v)
return base
}
func (m *WriteRequest) size() (n int) {
func (m *WriteRequest) Size() (n int) {
if m == nil {
return 0
}
for _, e := range m.Timeseries {
l := e.size()
l := e.Size()
n += 1 + l + sov(uint64(l))
}
return n

View File

@@ -26,7 +26,7 @@ type Label struct {
Value string
}
func (m *Sample) marshalToSizedBuffer(dst []byte) (int, error) {
func (m *Sample) MarshalToSizedBuffer(dst []byte) (int, error) {
i := len(dst)
if m.Timestamp != 0 {
i = encodeVarint(dst, i, uint64(m.Timestamp))
@@ -42,10 +42,10 @@ func (m *Sample) marshalToSizedBuffer(dst []byte) (int, error) {
return len(dst) - i, nil
}
func (m *TimeSeries) marshalToSizedBuffer(dst []byte) (int, error) {
func (m *TimeSeries) MarshalToSizedBuffer(dst []byte) (int, error) {
i := len(dst)
for j := len(m.Samples) - 1; j >= 0; j-- {
size, err := m.Samples[j].marshalToSizedBuffer(dst[:i])
size, err := m.Samples[j].MarshalToSizedBuffer(dst[:i])
if err != nil {
return 0, err
}
@@ -55,7 +55,7 @@ func (m *TimeSeries) marshalToSizedBuffer(dst []byte) (int, error) {
dst[i] = 0x12
}
for j := len(m.Labels) - 1; j >= 0; j-- {
size, err := m.Labels[j].marshalToSizedBuffer(dst[:i])
size, err := m.Labels[j].MarshalToSizedBuffer(dst[:i])
if err != nil {
return 0, err
}
@@ -67,7 +67,7 @@ func (m *TimeSeries) marshalToSizedBuffer(dst []byte) (int, error) {
return len(dst) - i, nil
}
func (m *Label) marshalToSizedBuffer(dst []byte) (int, error) {
func (m *Label) MarshalToSizedBuffer(dst []byte) (int, error) {
i := len(dst)
if len(m.Value) > 0 {
i -= len(m.Value)
@@ -86,7 +86,7 @@ func (m *Label) marshalToSizedBuffer(dst []byte) (int, error) {
return len(dst) - i, nil
}
func (m *Sample) size() (n int) {
func (m *Sample) Size() (n int) {
if m == nil {
return 0
}
@@ -99,22 +99,22 @@ func (m *Sample) size() (n int) {
return n
}
func (m *TimeSeries) size() (n int) {
func (m *TimeSeries) Size() (n int) {
if m == nil {
return 0
}
for _, e := range m.Labels {
l := e.size()
l := e.Size()
n += 1 + l + sov(uint64(l))
}
for _, e := range m.Samples {
l := e.size()
l := e.Size()
n += 1 + l + sov(uint64(l))
}
return n
}
func (m *Label) size() (n int) {
func (m *Label) Size() (n int) {
if m == nil {
return 0
}

View File

@@ -8,10 +8,10 @@ import (
// MarshalProtobuf marshals wr to dst and returns the result.
func (wr *WriteRequest) MarshalProtobuf(dst []byte) []byte {
size := wr.size()
size := wr.Size()
dstLen := len(dst)
dst = slicesutil.SetLength(dst, dstLen+size)
n, err := wr.marshalToSizedBuffer(dst[dstLen:])
n, err := wr.MarshalToSizedBuffer(dst[dstLen:])
if err != nil {
panic(fmt.Errorf("BUG: unexpected error when marshaling WriteRequest: %w", err))
}

View File

@@ -0,0 +1,205 @@
package parser
import (
"fmt"
"go/ast"
goparser "go/parser"
"strings"
)
type funcType struct {
name string
defPrefix string
callPrefix string
argNames string
args string
}
func parseFuncDef(b []byte) (*funcType, error) {
defStr := string(b)
// extract func name
n := strings.Index(defStr, "(")
if n < 0 {
return nil, fmt.Errorf("cannot find '(' in function definition")
}
name := defStr[:n]
defStr = defStr[n+1:]
defPrefix := ""
callPrefix := ""
if len(name) == 0 {
// Either empty func name or valid method definition. Let's check.
// parse method receiver
n = strings.Index(defStr, ")")
if n < 0 {
return nil, fmt.Errorf("cannot find ')' in func")
}
recvStr := defStr[:n]
defStr = defStr[n+1:]
exprStr := fmt.Sprintf("func (%s)", recvStr)
expr, err := goparser.ParseExpr(exprStr)
if err != nil {
return nil, fmt.Errorf("invalid method definition: %s", err)
}
ft := expr.(*ast.FuncType)
if len(ft.Params.List) != 1 || len(ft.Params.List[0].Names) != 1 {
// method receiver must contain only one param
return nil, fmt.Errorf("missing func or method name")
}
recvName := ft.Params.List[0].Names[0].Name
defPrefix = fmt.Sprintf("(%s) ", recvStr)
callPrefix = recvName + "."
// extract method name
n = strings.Index(defStr, "(")
if n < 0 {
return nil, fmt.Errorf("missing func name")
}
name = string(stripLeadingSpace([]byte(defStr[:n])))
if len(name) == 0 {
return nil, fmt.Errorf("missing method name")
}
defStr = defStr[n+1:]
}
// validate and collect func args
if len(defStr) == 0 || defStr[len(defStr)-1] != ')' {
return nil, fmt.Errorf("missing ')' at the end of func")
}
args := defStr[:len(defStr)-1]
exprStr := fmt.Sprintf("func (%s)", args)
expr, err := goparser.ParseExpr(exprStr)
if err != nil {
return nil, fmt.Errorf("invalid func args: %s", err)
}
ft := expr.(*ast.FuncType)
if ft.Results != nil {
return nil, fmt.Errorf("func mustn't return any results")
}
// extract arg names
var tmp []string
for _, f := range ft.Params.List {
if len(f.Names) == 0 {
return nil, fmt.Errorf("func cannot contain untyped arguments")
}
for _, n := range f.Names {
if n == nil {
return nil, fmt.Errorf("func cannot contain untyped arguments")
}
if _, isVariadic := f.Type.(*ast.Ellipsis); isVariadic {
tmp = append(tmp, n.Name+"...")
} else {
tmp = append(tmp, n.Name)
}
}
}
argNames := strings.Join(tmp, ", ")
if len(args) > 0 {
args = ", " + args
}
if len(argNames) > 0 {
argNames = ", " + argNames
}
return &funcType{
name: name,
defPrefix: defPrefix,
callPrefix: callPrefix,
argNames: argNames,
args: args,
}, nil
}
func parseFuncCall(b []byte) (*funcType, error) {
exprStr := string(b)
expr, err := goparser.ParseExpr(exprStr)
if err != nil {
return nil, err
}
ce, ok := expr.(*ast.CallExpr)
if !ok {
return nil, fmt.Errorf("missing function call")
}
callPrefix, name, err := getCallName(ce)
if err != nil {
return nil, err
}
argNames := exprStr[ce.Lparen : ce.Rparen-1]
if len(argNames) > 0 {
argNames = ", " + argNames
}
return &funcType{
name: name,
callPrefix: callPrefix,
argNames: argNames,
}, nil
}
func (f *funcType) DefStream(dst string) string {
return fmt.Sprintf("%s%s%s(%s *qt%s.Writer%s)", f.defPrefix, f.prefixStream(), f.name, dst, mangleSuffix, f.args)
}
func (f *funcType) CallStream(dst string) string {
return fmt.Sprintf("%s%s%s(%s%s)", f.callPrefix, f.prefixStream(), f.name, dst, f.argNames)
}
func (f *funcType) DefWrite(dst string) string {
return fmt.Sprintf("%s%s%s(%s qtio%s.Writer%s)", f.defPrefix, f.prefixWrite(), f.name, dst, mangleSuffix, f.args)
}
func (f *funcType) CallWrite(dst string) string {
return fmt.Sprintf("%s%s%s(%s%s)", f.callPrefix, f.prefixWrite(), f.name, dst, f.argNames)
}
func (f *funcType) DefString() string {
args := f.args
if len(args) > 0 {
// skip the first ', '
args = args[2:]
}
return fmt.Sprintf("%s%s(%s) string", f.defPrefix, f.name, args)
}
func (f *funcType) prefixWrite() string {
s := "write"
if isUpper(f.name[0]) {
s = "Write"
}
return s
}
func (f *funcType) prefixStream() string {
s := "stream"
if isUpper(f.name[0]) {
s = "Stream"
}
return s
}
func getCallName(ce *ast.CallExpr) (string, string, error) {
callPrefix := ""
name := ""
expr := ce.Fun
for {
switch x := expr.(type) {
case *ast.Ident:
if len(callPrefix) == 0 && len(name) == 0 {
return "", x.Name, nil
}
callPrefix = x.Name + "." + callPrefix
return callPrefix, name, nil
case *ast.SelectorExpr:
if len(name) == 0 {
name = x.Sel.Name
} else {
callPrefix = x.Sel.Name + "." + callPrefix
}
expr = x.X
default:
return "", "", fmt.Errorf("unexpected function name")
}
}
}

View File

@@ -0,0 +1,921 @@
package parser
import (
"bytes"
"fmt"
"go/ast"
goparser "go/parser"
gotoken "go/token"
"io"
"path/filepath"
"strconv"
"strings"
)
type parser struct {
s *scanner
w io.Writer
packageName string
skipLineComments bool
prefix string
forDepth int
switchDepth int
skipOutputDepth int
importsUseEmitted bool
packageNameEmitted bool
}
// Parse parses the contents of the supplied reader, writing generated code to
// the supplied writer. Uses filename as the source file for line comments, and
// pkg as the Go package name.
func Parse(w io.Writer, r io.Reader, filename, pkg string) error {
return parse(w, r, filename, pkg, false)
}
// ParseNoLineComments is the same as Parse, but does not write line comments.
func ParseNoLineComments(w io.Writer, r io.Reader, filename, pkg string) error {
return parse(w, r, filename, pkg, true)
}
func parse(w io.Writer, r io.Reader, filename, pkg string, skipLineComments bool) error {
p := &parser{
s: newScanner(r, filename),
w: w,
packageName: pkg,
skipLineComments: skipLineComments,
}
return p.parseTemplate()
}
func (p *parser) parseTemplate() error {
s := p.s
fmt.Fprintf(p.w, `// Code generated by qtc from %q. DO NOT EDIT.
// See https://github.com/valyala/quicktemplate for details.
`,
filepath.Base(s.filePath))
for s.Next() {
t := s.Token()
switch t.ID {
case text:
p.emitComment(t.Value)
case tagName:
switch string(t.Value) {
case "package":
if p.packageNameEmitted {
return fmt.Errorf("package name must be at the top of the template. Found at %s", s.Context())
}
if err := p.parsePackageName(); err != nil {
return err
}
case "import":
p.emitPackageName()
if p.importsUseEmitted {
return fmt.Errorf("imports must be at the top of the template. Found at %s", s.Context())
}
if err := p.parseImport(); err != nil {
return err
}
default:
p.emitPackageName()
p.emitImportsUse()
switch string(t.Value) {
case "interface", "iface":
if err := p.parseInterface(); err != nil {
return err
}
case "code":
if err := p.parseTemplateCode(); err != nil {
return err
}
case "func":
if err := p.parseFunc(); err != nil {
return err
}
default:
return fmt.Errorf("unexpected tag found outside func: %q at %s", t.Value, s.Context())
}
}
default:
return fmt.Errorf("unexpected token found %s outside func at %s", t, s.Context())
}
}
p.emitImportsUse()
if err := s.LastError(); err != nil {
return fmt.Errorf("cannot parse template: %s", err)
}
return nil
}
func (p *parser) emitPackageName() {
if !p.packageNameEmitted {
p.Printf("package %s\n", p.packageName)
p.packageNameEmitted = true
}
}
func (p *parser) emitComment(comment []byte) {
isFirstNonemptyLine := false
for len(comment) > 0 {
n := bytes.IndexByte(comment, '\n')
if n < 0 {
n = len(comment)
}
line := stripTrailingSpace(comment[:n])
if bytes.HasPrefix(line, []byte("//")) {
line = line[2:]
if len(line) > 0 && isSpace(line[0]) {
line = line[1:]
}
}
if len(line) == 0 {
if isFirstNonemptyLine {
fmt.Fprintf(p.w, "//\n")
}
} else {
fmt.Fprintf(p.w, "// %s\n", line)
isFirstNonemptyLine = true
}
if n < len(comment) {
comment = comment[n+1:]
} else {
comment = comment[n:]
}
}
fmt.Fprintf(p.w, "\n")
}
func (p *parser) emitImportsUse() {
if p.importsUseEmitted {
return
}
p.Printf(`import (
qtio%s "io"
qt%s "github.com/valyala/quicktemplate"
)
`, mangleSuffix, mangleSuffix)
p.Printf(`var (
_ = qtio%s.Copy
_ = qt%s.AcquireByteBuffer
)
`, mangleSuffix, mangleSuffix)
p.importsUseEmitted = true
}
func (p *parser) parseFunc() error {
s := p.s
t, err := expectTagContents(s)
if err != nil {
return err
}
funcStr := "func " + string(t.Value)
f, err := parseFuncDef(t.Value)
if err != nil {
return fmt.Errorf("error in %q at %s: %s", funcStr, s.Context(), err)
}
p.emitFuncStart(f)
for s.Next() {
t := s.Token()
switch t.ID {
case text:
p.emitText(t.Value)
case tagName:
ok, err := p.tryParseCommonTags(t.Value)
if err != nil {
return fmt.Errorf("error in %q: %s", funcStr, err)
}
if ok {
continue
}
switch string(t.Value) {
case "endfunc":
if err = skipTagContents(s); err != nil {
return err
}
p.emitFuncEnd(f)
return nil
default:
return fmt.Errorf("unexpected tag found in %q: %q at %s", funcStr, t.Value, s.Context())
}
default:
return fmt.Errorf("unexpected token found when parsing %q: %s at %s", funcStr, t, s.Context())
}
}
if err := s.LastError(); err != nil {
return fmt.Errorf("cannot parse %q: %s", funcStr, err)
}
return fmt.Errorf("cannot find endfunc tag for %q at %s", funcStr, s.Context())
}
func (p *parser) parseFor() error {
s := p.s
t, err := expectTagContents(s)
if err != nil {
return err
}
forStr := "for " + string(t.Value)
if err = validateForStmt(t.Value); err != nil {
return fmt.Errorf("invalid statement %q at %s: %s", forStr, s.Context(), err)
}
p.Printf("for %s {", t.Value)
p.prefix += "\t"
p.forDepth++
for s.Next() {
t := s.Token()
switch t.ID {
case text:
p.emitText(t.Value)
case tagName:
ok, err := p.tryParseCommonTags(t.Value)
if err != nil {
return fmt.Errorf("error in %q: %s", forStr, err)
}
if ok {
continue
}
switch string(t.Value) {
case "endfor":
if err = skipTagContents(s); err != nil {
return err
}
p.forDepth--
p.prefix = p.prefix[1:]
p.Printf("}")
return nil
default:
return fmt.Errorf("unexpected tag found in %q: %q at %s", forStr, t.Value, s.Context())
}
default:
return fmt.Errorf("unexpected token found when parsing %q: %s at %s", forStr, t, s.Context())
}
}
if err := s.LastError(); err != nil {
return fmt.Errorf("cannot parse %q: %s", forStr, err)
}
return fmt.Errorf("cannot find endfor tag for %q at %s", forStr, s.Context())
}
func (p *parser) parseDefault() error {
s := p.s
if err := skipTagContents(s); err != nil {
return err
}
stmtStr := "default"
p.Printf("default:")
p.prefix += "\t"
for s.Next() {
t := s.Token()
switch t.ID {
case text:
p.emitText(t.Value)
case tagName:
ok, err := p.tryParseCommonTags(t.Value)
if err != nil {
return fmt.Errorf("error in %q: %s", stmtStr, err)
}
if !ok {
s.Rewind()
p.prefix = p.prefix[1:]
return nil
}
default:
return fmt.Errorf("unexpected token found when parsing %q: %s at %s", stmtStr, t, s.Context())
}
}
if err := s.LastError(); err != nil {
return fmt.Errorf("cannot parse %q: %s", stmtStr, err)
}
return fmt.Errorf("cannot find end of %q at %s", stmtStr, s.Context())
}
func (p *parser) parseCase(switchValue string) error {
s := p.s
t, err := expectTagContents(s)
if err != nil {
return err
}
caseStr := "case " + string(t.Value)
if err = validateCaseStmt(switchValue, t.Value); err != nil {
return fmt.Errorf("invalid statement %q at %s: %s", caseStr, s.Context(), err)
}
p.Printf("case %s:", t.Value)
p.prefix += "\t"
for s.Next() {
t := s.Token()
switch t.ID {
case text:
p.emitText(t.Value)
case tagName:
ok, err := p.tryParseCommonTags(t.Value)
if err != nil {
return fmt.Errorf("error in %q: %s", caseStr, err)
}
if !ok {
s.Rewind()
p.prefix = p.prefix[1:]
return nil
}
default:
return fmt.Errorf("unexpected token found when parsing %q: %s at %s", caseStr, t, s.Context())
}
}
if err := s.LastError(); err != nil {
return fmt.Errorf("cannot parse %q: %s", caseStr, err)
}
return fmt.Errorf("cannot find end of %q at %s", caseStr, s.Context())
}
func (p *parser) parseCat() error {
s := p.s
t, err := expectTagContents(s)
if err != nil {
return err
}
filename, err := strconv.Unquote(string(t.Value))
if err != nil {
return fmt.Errorf("invalid cat value %q at %s: %s", t.Value, s.Context(), err)
}
data, err := readFile(s.filePath, filename)
if err != nil {
return fmt.Errorf("cannot cat file %q at %s: %s", filename, s.Context(), err)
}
p.emitText(data)
return nil
}
func (p *parser) parseSwitch() error {
s := p.s
t, err := expectTagContents(s)
if err != nil {
return err
}
switchStr := "switch " + string(t.Value)
if err = validateSwitchStmt(t.Value); err != nil {
return fmt.Errorf("invalid statement %q at %s: %s", switchStr, s.Context(), err)
}
p.Printf("switch %s {", t.Value)
switchValue := string(t.Value)
caseNum := 0
defaultFound := false
p.switchDepth++
for s.Next() {
t := s.Token()
switch t.ID {
case text:
if caseNum == 0 {
comment := stripLeadingSpace(t.Value)
if len(comment) > 0 {
p.emitComment(comment)
}
} else {
p.emitText(t.Value)
}
case tagName:
switch string(t.Value) {
case "endswitch":
if caseNum == 0 {
return fmt.Errorf("empty statement %q found at %s", switchStr, s.Context())
}
if err = skipTagContents(s); err != nil {
return err
}
p.switchDepth--
p.Printf("}")
return nil
case "case":
caseNum++
if err = p.parseCase(switchValue); err != nil {
return err
}
case "default":
if defaultFound {
return fmt.Errorf("duplicate default tag found in %q at %s", switchStr, s.Context())
}
defaultFound = true
caseNum++
if err = p.parseDefault(); err != nil {
return err
}
default:
return fmt.Errorf("unexpected tag found in %q: %q at %s", switchStr, t.Value, s.Context())
}
default:
return fmt.Errorf("unexpected token found when parsing %q: %s at %s", switchStr, t, s.Context())
}
}
if err := s.LastError(); err != nil {
return fmt.Errorf("cannot parse %q: %s", switchStr, err)
}
return fmt.Errorf("cannot find endswitch tag for %q at %s", switchStr, s.Context())
}
func (p *parser) parseIf() error {
s := p.s
t, err := expectTagContents(s)
if err != nil {
return err
}
if len(t.Value) == 0 {
return fmt.Errorf("empty if condition at %s", s.Context())
}
ifStr := "if " + string(t.Value)
if err = validateIfStmt(t.Value); err != nil {
return fmt.Errorf("invalid statement %q at %s: %s", ifStr, s.Context(), err)
}
p.Printf("if %s {", t.Value)
p.prefix += "\t"
elseUsed := false
for s.Next() {
t := s.Token()
switch t.ID {
case text:
p.emitText(t.Value)
case tagName:
ok, err := p.tryParseCommonTags(t.Value)
if err != nil {
return fmt.Errorf("error in %q: %s", ifStr, err)
}
if ok {
continue
}
switch string(t.Value) {
case "endif":
if err = skipTagContents(s); err != nil {
return err
}
p.prefix = p.prefix[1:]
p.Printf("}")
return nil
case "else":
if elseUsed {
return fmt.Errorf("duplicate else branch found for %q at %s", ifStr, s.Context())
}
if err = skipTagContents(s); err != nil {
return err
}
p.prefix = p.prefix[1:]
p.Printf("} else {")
p.prefix += "\t"
elseUsed = true
case "elseif":
if elseUsed {
return fmt.Errorf("unexpected elseif branch found after else branch for %q at %s",
ifStr, s.Context())
}
t, err = expectTagContents(s)
if err != nil {
return err
}
p.prefix = p.prefix[1:]
p.Printf("} else if %s {", t.Value)
p.prefix += "\t"
default:
return fmt.Errorf("unexpected tag found in %q: %q at %s", ifStr, t.Value, s.Context())
}
default:
return fmt.Errorf("unexpected token found when parsing %q: %s at %s", ifStr, t, s.Context())
}
}
if err := s.LastError(); err != nil {
return fmt.Errorf("cannot parse %q: %s", ifStr, err)
}
return fmt.Errorf("cannot find endif tag for %q at %s", ifStr, s.Context())
}
func (p *parser) tryParseCommonTags(tagBytes []byte) (bool, error) {
tagNameStr, prec := splitTagNamePrec(string(tagBytes))
switch tagNameStr {
case "s", "v", "d", "dl", "dul", "f", "q", "z", "j", "u",
"s=", "v=", "d=", "dl=", "dul=", "f=", "q=", "z=", "j=", "u=",
"sz", "qz", "jz", "uz",
"sz=", "qz=", "jz=", "uz=":
if err := p.parseOutputTag(tagNameStr, prec); err != nil {
return false, err
}
case "=", "=h", "=u", "=uh", "=q", "=qh", "=j", "=jh":
if err := p.parseOutputFunc(tagNameStr); err != nil {
return false, err
}
case "return":
if err := p.skipAfterTag(tagNameStr); err != nil {
return false, err
}
case "break":
if p.forDepth <= 0 && p.switchDepth <= 0 {
return false, fmt.Errorf("found break tag outside for loop and switch block")
}
if err := p.skipAfterTag(tagNameStr); err != nil {
return false, err
}
case "continue":
if p.forDepth <= 0 {
return false, fmt.Errorf("found continue tag outside for loop")
}
if err := p.skipAfterTag(tagNameStr); err != nil {
return false, err
}
case "code":
if err := p.parseFuncCode(); err != nil {
return false, err
}
case "for":
if err := p.parseFor(); err != nil {
return false, err
}
case "if":
if err := p.parseIf(); err != nil {
return false, err
}
case "switch":
if err := p.parseSwitch(); err != nil {
return false, err
}
case "cat":
if err := p.parseCat(); err != nil {
return false, err
}
default:
return false, nil
}
return true, nil
}
func splitTagNamePrec(tagName string) (string, int) {
parts := strings.Split(tagName, ".")
if len(parts) == 2 && parts[0] == "f" {
p := parts[1]
if strings.HasSuffix(p, "=") {
p = p[:len(p)-1]
}
if len(p) == 0 {
return "f", 0
}
prec, err := strconv.Atoi(p)
if err == nil && prec >= 0 {
return "f", prec
}
}
return tagName, -1
}
func (p *parser) skipAfterTag(tagStr string) error {
s := p.s
if err := skipTagContents(s); err != nil {
return err
}
p.Printf("%s", tagStr)
p.skipOutputDepth++
defer func() {
p.skipOutputDepth--
}()
for s.Next() {
t := s.Token()
switch t.ID {
case text:
// skip text
case tagName:
ok, err := p.tryParseCommonTags(t.Value)
if err != nil {
return fmt.Errorf("error when parsing contents after %q: %s", tagStr, err)
}
if ok {
continue
}
switch string(t.Value) {
case "endfunc", "endfor", "endif", "else", "elseif", "case", "default", "endswitch":
s.Rewind()
return nil
default:
return fmt.Errorf("unexpected tag found after %q: %q at %s", tagStr, t.Value, s.Context())
}
default:
return fmt.Errorf("unexpected token found when parsing contents after %q: %s at %s", tagStr, t, s.Context())
}
}
if err := s.LastError(); err != nil {
return fmt.Errorf("cannot parse contents after %q: %s", tagStr, err)
}
return fmt.Errorf("cannot find closing tag after %q at %s", tagStr, s.Context())
}
func (p *parser) parseInterface() error {
s := p.s
t, err := expectTagContents(s)
if err != nil {
return err
}
n := bytes.IndexByte(t.Value, '{')
if n < 0 {
return fmt.Errorf("missing '{' in interface at %s", s.Context())
}
ifname := string(stripTrailingSpace(t.Value[:n]))
if len(ifname) == 0 {
return fmt.Errorf("missing interface name at %s", s.Context())
}
p.Printf("type %s interface {", ifname)
p.prefix = "\t"
tail := t.Value[n:]
exprStr := fmt.Sprintf("interface %s", tail)
expr, err := goparser.ParseExpr(exprStr)
if err != nil {
return fmt.Errorf("error when parsing interface at %s: %s", s.Context(), err)
}
it, ok := expr.(*ast.InterfaceType)
if !ok {
return fmt.Errorf("unexpected interface type at %s: %T", s.Context(), expr)
}
methods := it.Methods.List
if len(methods) == 0 {
return fmt.Errorf("interface must contain at least one method at %s", s.Context())
}
for _, m := range it.Methods.List {
methodStr := exprStr[m.Pos()-1 : m.End()-1]
f, err := parseFuncDef([]byte(methodStr))
if err != nil {
return fmt.Errorf("when when parsing %q at %s: %s", methodStr, s.Context(), err)
}
p.Printf("%s string", methodStr)
p.Printf("%s", f.DefStream("qw"+mangleSuffix))
p.Printf("%s", f.DefWrite("qq"+mangleSuffix))
}
p.prefix = ""
p.Printf("}")
return nil
}
func (p *parser) parsePackageName() error {
t, err := expectTagContents(p.s)
if err != nil {
return err
}
if len(t.Value) == 0 {
return fmt.Errorf("empty package name found at %s", p.s.Context())
}
if err = validatePackageName(t.Value); err != nil {
return fmt.Errorf("invalid package name found at %s: %s", p.s.Context(), err)
}
p.packageName = string(t.Value)
p.emitPackageName()
return nil
}
func (p *parser) parseImport() error {
t, err := expectTagContents(p.s)
if err != nil {
return err
}
if len(t.Value) == 0 {
return fmt.Errorf("empty import found at %s", p.s.Context())
}
if err = validateImport(t.Value); err != nil {
return fmt.Errorf("invalid import found at %s: %s", p.s.Context(), err)
}
p.Printf("import %s\n", t.Value)
return nil
}
func (p *parser) parseTemplateCode() error {
t, err := expectTagContents(p.s)
if err != nil {
return err
}
if err = validateTemplateCode(t.Value); err != nil {
return fmt.Errorf("invalid code at %s: %s", p.s.Context(), err)
}
p.Printf("%s\n", t.Value)
return nil
}
func (p *parser) parseFuncCode() error {
t, err := expectTagContents(p.s)
if err != nil {
return err
}
if err = validateFuncCode(t.Value); err != nil {
return fmt.Errorf("invalid code at %s: %s", p.s.Context(), err)
}
p.Printf("%s\n", t.Value)
return nil
}
func (p *parser) parseOutputTag(tagNameStr string, prec int) error {
s := p.s
t, err := expectTagContents(s)
if err != nil {
return err
}
if err = validateOutputTagValue(t.Value); err != nil {
return fmt.Errorf("invalid output tag value at %s: %s", s.Context(), err)
}
filter := "N"
switch tagNameStr {
case "s", "v", "q", "z", "j", "sz", "qz", "jz":
filter = "E"
}
if strings.HasSuffix(tagNameStr, "=") {
tagNameStr = tagNameStr[:len(tagNameStr)-1]
}
if tagNameStr == "f" && prec >= 0 {
p.Printf("qw%s.N().FPrec(%s, %d)", mangleSuffix, t.Value, prec)
} else {
tagNameStr = strings.ToUpper(tagNameStr)
p.Printf("qw%s.%s().%s(%s)", mangleSuffix, filter, tagNameStr, t.Value)
}
return nil
}
func (p *parser) parseOutputFunc(tagNameStr string) error {
s := p.s
t, err := expectTagContents(s)
if err != nil {
return err
}
f, err := parseFuncCall(t.Value)
if err != nil {
return fmt.Errorf("error at %s: %s", s.Context(), err)
}
filter := "N"
tagNameStr = tagNameStr[1:]
if strings.HasSuffix(tagNameStr, "h") {
tagNameStr = tagNameStr[:len(tagNameStr)-1]
switch tagNameStr {
case "", "q", "j":
filter = "E"
}
}
if len(tagNameStr) > 0 || filter == "E" {
tagNameStr = strings.ToUpper(tagNameStr)
p.Printf("{")
p.Printf("qb%s := qt%s.AcquireByteBuffer()", mangleSuffix, mangleSuffix)
p.Printf("%s", f.CallWrite("qb"+mangleSuffix))
p.Printf("qw%s.%s().%sZ(qb%s.B)", mangleSuffix, filter, tagNameStr, mangleSuffix)
p.Printf("qt%s.ReleaseByteBuffer(qb%s)", mangleSuffix, mangleSuffix)
p.Printf("}")
} else {
p.Printf("%s", f.CallStream("qw"+mangleSuffix))
}
return nil
}
func (p *parser) emitText(text []byte) {
for len(text) > 0 {
n := bytes.IndexByte(text, '`')
if n < 0 {
p.Printf("qw%s.N().S(`%s`)", mangleSuffix, text)
return
}
p.Printf("qw%s.N().S(`%s`)", mangleSuffix, text[:n])
p.Printf("qw%s.N().S(\"`\")", mangleSuffix)
text = text[n+1:]
}
}
func (p *parser) emitFuncStart(f *funcType) {
p.Printf("func %s {", f.DefStream("qw"+mangleSuffix))
p.prefix = "\t"
}
func (p *parser) emitFuncEnd(f *funcType) {
p.prefix = ""
p.Printf("}\n")
p.Printf("func %s {", f.DefWrite("qq"+mangleSuffix))
p.prefix = "\t"
p.Printf("qw%s := qt%s.AcquireWriter(qq%s)", mangleSuffix, mangleSuffix, mangleSuffix)
p.Printf("%s", f.CallStream("qw"+mangleSuffix))
p.Printf("qt%s.ReleaseWriter(qw%s)", mangleSuffix, mangleSuffix)
p.prefix = ""
p.Printf("}\n")
p.Printf("func %s {", f.DefString())
p.prefix = "\t"
p.Printf("qb%s := qt%s.AcquireByteBuffer()", mangleSuffix, mangleSuffix)
p.Printf("%s", f.CallWrite("qb"+mangleSuffix))
p.Printf("qs%s := string(qb%s.B)", mangleSuffix, mangleSuffix)
p.Printf("qt%s.ReleaseByteBuffer(qb%s)", mangleSuffix, mangleSuffix)
p.Printf("return qs%s", mangleSuffix)
p.prefix = ""
p.Printf("}\n")
}
func (p *parser) Printf(format string, args ...interface{}) {
if p.skipOutputDepth > 0 {
return
}
w := p.w
if !p.skipLineComments {
// line comments are required to start at the beginning of the line
p.s.WriteLineComment(w)
}
fmt.Fprintf(w, "%s", p.prefix)
fmt.Fprintf(w, format, args...)
fmt.Fprintf(w, "\n")
}
func skipTagContents(s *scanner) error {
tagName := string(s.Token().Value)
t, err := expectTagContents(s)
if err != nil {
return err
}
if len(t.Value) > 0 {
return fmt.Errorf("unexpected extra value after %s: %q at %s", tagName, t.Value, s.Context())
}
return err
}
func expectTagContents(s *scanner) (*token, error) {
return expectToken(s, tagContents)
}
func expectToken(s *scanner, id int) (*token, error) {
if !s.Next() {
return nil, fmt.Errorf("cannot find token %s: %v", tokenIDToStr(id), s.LastError())
}
t := s.Token()
if t.ID != id {
return nil, fmt.Errorf("unexpected token found %s. Expecting %s at %s", t, tokenIDToStr(id), s.Context())
}
return t, nil
}
func validateOutputTagValue(stmt []byte) error {
exprStr := string(stmt)
_, err := goparser.ParseExpr(exprStr)
return err
}
func validateForStmt(stmt []byte) error {
exprStr := fmt.Sprintf("func () { for %s {} }", stmt)
_, err := goparser.ParseExpr(exprStr)
return err
}
func validateIfStmt(stmt []byte) error {
exprStr := fmt.Sprintf("func () { if %s {} }", stmt)
_, err := goparser.ParseExpr(exprStr)
return err
}
func validateSwitchStmt(stmt []byte) error {
exprStr := fmt.Sprintf("func () { switch %s {} }", stmt)
_, err := goparser.ParseExpr(exprStr)
return err
}
func validateCaseStmt(switchValue string, stmt []byte) error {
exprStr := fmt.Sprintf("func () { switch %s {case %s:} }", switchValue, stmt)
_, err := goparser.ParseExpr(exprStr)
return err
}
func validateFuncCode(code []byte) error {
exprStr := fmt.Sprintf("func () { for { %s\n } }", code)
_, err := goparser.ParseExpr(exprStr)
return err
}
func validateTemplateCode(code []byte) error {
codeStr := fmt.Sprintf("package foo\nvar _ = a\n%s", code)
fset := gotoken.NewFileSet()
_, err := goparser.ParseFile(fset, "", codeStr, 0)
return err
}
func validatePackageName(code []byte) error {
codeStr := fmt.Sprintf("package %s", code)
fset := gotoken.NewFileSet()
_, err := goparser.ParseFile(fset, "", codeStr, 0)
return err
}
func validateImport(code []byte) error {
codeStr := fmt.Sprintf("package foo\nimport %s", code)
fset := gotoken.NewFileSet()
f, err := goparser.ParseFile(fset, "", codeStr, 0)
if err != nil {
return err
}
for _, d := range f.Decls {
gd, ok := d.(*ast.GenDecl)
if !ok {
return fmt.Errorf("unexpected code found: %T. Expecting ast.GenDecl", d)
}
for _, s := range gd.Specs {
if _, ok := s.(*ast.ImportSpec); !ok {
return fmt.Errorf("unexpected code found: %T. Expecting ast.ImportSpec", s)
}
}
}
return nil
}

View File

@@ -0,0 +1,467 @@
package parser
import (
"bufio"
"bytes"
"fmt"
"io"
"regexp"
"strings"
)
// token ids
const (
text = iota
tagName
tagContents
)
var tokenStrMap = map[int]string{
text: "text",
tagName: "tagName",
tagContents: "tagContents",
}
func tokenIDToStr(id int) string {
str := tokenStrMap[id]
if str == "" {
panic(fmt.Sprintf("unknown tokenID=%d", id))
}
return str
}
type token struct {
ID int
Value []byte
line int
pos int
}
func (t *token) init(id, line, pos int) {
t.ID = id
t.Value = t.Value[:0]
t.line = line
t.pos = pos
}
func (t *token) String() string {
return fmt.Sprintf("Token %q, value %q", tokenIDToStr(t.ID), t.Value)
}
type scanner struct {
r *bufio.Reader
t token
c byte
err error
filePath string
line int
lineStr []byte
nextTokenID int
capture bool
capturedValue []byte
collapseSpaceDepth int
stripSpaceDepth int
stripToNewLine bool
rewind bool
}
var tailOfLine = regexp.MustCompile(`^[[:blank:]]*(?:\r*\n)?`)
var prevBlank = regexp.MustCompile(`[[:blank:]]+$`)
func newScanner(r io.Reader, filePath string) *scanner {
// Substitute backslashes with forward slashes in filePath
// for the sake of consistency on different platforms (windows, linux).
// See https://github.com/valyala/quicktemplate/issues/62.
filePath = strings.Replace(filePath, "\\", "/", -1)
return &scanner{
r: bufio.NewReader(r),
filePath: filePath,
}
}
func (s *scanner) Rewind() {
if s.rewind {
panic("BUG: duplicate Rewind call")
}
s.rewind = true
}
func (s *scanner) Next() bool {
if s.rewind {
s.rewind = false
return true
}
for {
if !s.scanToken() {
return false
}
switch s.t.ID {
case text:
if s.stripToNewLine {
s.t.Value = tailOfLine.ReplaceAll(s.t.Value, nil)
s.stripToNewLine = false
}
if len(s.t.Value) == 0 {
// skip empty text
continue
}
case tagName:
switch string(s.t.Value) {
case "comment":
if !s.skipComment() {
return false
}
continue
case "plain":
if !s.readPlain() {
return false
}
if len(s.t.Value) == 0 {
// skip empty text
continue
}
case "collapsespace":
if !s.readTagContents() {
return false
}
s.collapseSpaceDepth++
continue
case "stripspace":
if !s.readTagContents() {
return false
}
s.stripSpaceDepth++
continue
case "endcollapsespace":
if s.collapseSpaceDepth == 0 {
s.err = fmt.Errorf("endcollapsespace tag found without the corresponding collapsespace tag")
return false
}
if !s.readTagContents() {
return false
}
s.collapseSpaceDepth--
continue
case "endstripspace":
if s.stripSpaceDepth == 0 {
s.err = fmt.Errorf("endstripspace tag found without the corresponding stripspace tag")
return false
}
if !s.readTagContents() {
return false
}
s.stripSpaceDepth--
continue
case "space":
if !s.readTagContents() {
return false
}
s.t.init(text, s.t.line, s.t.pos)
s.t.Value = append(s.t.Value[:0], ' ')
return true
case "newline":
if !s.readTagContents() {
return false
}
s.t.init(text, s.t.line, s.t.pos)
s.t.Value = append(s.t.Value[:0], '\n')
return true
}
}
return true
}
}
func (s *scanner) readPlain() bool {
if !s.readTagContents() {
return false
}
startLine := s.line
startPos := s.pos()
s.startCapture()
ok := s.skipUntilTag("endplain")
v := s.stopCapture()
s.t.init(text, startLine, startPos)
if ok {
n := bytes.LastIndex(v, strTagOpen)
v = v[:n]
s.t.Value = append(s.t.Value[:0], v...)
}
return ok
}
var strTagOpen = []byte("{%")
func (s *scanner) skipComment() bool {
if !s.readTagContents() {
return false
}
return s.skipUntilTag("endcomment")
}
func (s *scanner) skipUntilTag(tagName string) bool {
ok := false
for {
if !s.nextByte() {
break
}
if s.c != '{' {
continue
}
if !s.nextByte() {
break
}
if s.c != '%' {
s.unreadByte('~')
continue
}
ok = s.readTagName()
s.nextTokenID = text
if !ok {
s.err = nil
continue
}
if string(s.t.Value) == tagName {
ok = s.readTagContents()
break
}
}
if !ok {
s.err = fmt.Errorf("cannot find %q tag: %s", tagName, s.err)
}
return ok
}
func (s *scanner) scanToken() bool {
switch s.nextTokenID {
case text:
return s.readText()
case tagName:
return s.readTagName()
case tagContents:
return s.readTagContents()
default:
panic(fmt.Sprintf("BUG: unknown nextTokenID %d", s.nextTokenID))
}
}
func (s *scanner) readText() bool {
s.t.init(text, s.line, s.pos())
ok := false
for {
if !s.nextByte() {
ok = (len(s.t.Value) > 0)
break
}
if s.c != '{' {
s.appendByte()
continue
}
if !s.nextByte() {
s.appendByte()
ok = true
break
}
if s.c == '%' {
s.nextTokenID = tagName
ok = true
if !s.nextByte() {
s.appendByte()
break
}
if s.c != '-' {
s.unreadByte(s.c)
break
}
s.t.Value = prevBlank.ReplaceAll(s.t.Value, nil)
break
}
s.unreadByte('{')
s.appendByte()
}
if s.stripSpaceDepth > 0 {
s.t.Value = stripSpace(s.t.Value)
} else if s.collapseSpaceDepth > 0 {
s.t.Value = collapseSpace(s.t.Value)
}
return ok
}
func (s *scanner) readTagName() bool {
s.skipSpace()
s.t.init(tagName, s.line, s.pos())
for {
if s.isSpace() || s.c == '%' {
if s.c == '%' {
s.unreadByte('~')
}
s.nextTokenID = tagContents
return true
}
if (s.c >= 'a' && s.c <= 'z') || (s.c >= 'A' && s.c <= 'Z') || (s.c >= '0' && s.c <= '9') || s.c == '=' || s.c == '.' {
s.appendByte()
if !s.nextByte() {
return false
}
continue
}
if s.c == '-' {
s.unreadByte(s.c)
s.nextTokenID = tagContents
return true
}
s.err = fmt.Errorf("unexpected character: '%c'", s.c)
s.unreadByte('~')
return false
}
}
func (s *scanner) readTagContents() bool {
s.skipSpace()
s.t.init(tagContents, s.line, s.pos())
for {
if s.c != '%' {
s.appendByte()
if !s.nextByte() {
return false
}
continue
}
if !s.nextByte() {
s.appendByte()
return false
}
if s.c == '}' {
if bytes.HasSuffix(s.t.Value, []byte("-")) {
s.t.Value = s.t.Value[:len(s.t.Value)-1]
s.stripToNewLine = true
}
s.nextTokenID = text
s.t.Value = stripTrailingSpace(s.t.Value)
return true
}
s.unreadByte('%')
s.appendByte()
if !s.nextByte() {
return false
}
}
}
func (s *scanner) skipSpace() {
for s.nextByte() && s.isSpace() {
}
}
func (s *scanner) isSpace() bool {
return isSpace(s.c)
}
func (s *scanner) nextByte() bool {
if s.err != nil {
return false
}
c, err := s.r.ReadByte()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
s.err = err
return false
}
if c == '\n' {
s.line++
s.lineStr = s.lineStr[:0]
} else {
s.lineStr = append(s.lineStr, c)
}
s.c = c
if s.capture {
s.capturedValue = append(s.capturedValue, c)
}
return true
}
func (s *scanner) startCapture() {
s.capture = true
s.capturedValue = s.capturedValue[:0]
}
func (s *scanner) stopCapture() []byte {
s.capture = false
v := s.capturedValue
s.capturedValue = s.capturedValue[:0]
return v
}
func (s *scanner) Token() *token {
return &s.t
}
func (s *scanner) LastError() error {
if s.err == nil {
return nil
}
if s.err == io.ErrUnexpectedEOF && s.t.ID == text {
if s.collapseSpaceDepth > 0 {
return fmt.Errorf("missing endcollapsespace tag at %s", s.Context())
}
if s.stripSpaceDepth > 0 {
return fmt.Errorf("missing endstripspace tag at %s", s.Context())
}
return nil
}
return fmt.Errorf("error when reading %s at %s: %s",
tokenIDToStr(s.t.ID), s.Context(), s.err)
}
func (s *scanner) appendByte() {
s.t.Value = append(s.t.Value, s.c)
}
func (s *scanner) unreadByte(c byte) {
if err := s.r.UnreadByte(); err != nil {
panic(fmt.Sprintf("BUG: bufio.Reader.UnreadByte returned non-nil error: %s", err))
}
if s.capture {
s.capturedValue = s.capturedValue[:len(s.capturedValue)-1]
}
if s.c == '\n' {
s.line--
s.lineStr = s.lineStr[:0] // TODO: use correct line
} else {
s.lineStr = s.lineStr[:len(s.lineStr)-1]
}
s.c = c
}
func (s *scanner) pos() int {
return len(s.lineStr)
}
func (s *scanner) Context() string {
t := s.Token()
return fmt.Sprintf("file %q, line %d, pos %d, token %s, last line %s",
s.filePath, t.line+1, t.pos, snippet(t.Value), snippet(s.lineStr))
}
func (s *scanner) WriteLineComment(w io.Writer) {
fmt.Fprintf(w, "//line %s:%d\n", s.filePath, s.t.line+1)
}
func snippet(s []byte) string {
if len(s) <= 40 {
return fmt.Sprintf("%q", s)
}
return fmt.Sprintf("%q ... %q", s[:20], s[len(s)-20:])
}

96
vendor/github.com/valyala/quicktemplate/parser/util.go generated vendored Normal file
View File

@@ -0,0 +1,96 @@
package parser
import (
"bytes"
"errors"
"io/ioutil"
"path/filepath"
"unicode"
)
// mangleSuffix is used for mangling quicktemplate-specific names
// in the generated code, so they don't clash with user-provided names.
const mangleSuffix = "422016"
func stripLeadingSpace(b []byte) []byte {
for len(b) > 0 && isSpace(b[0]) {
b = b[1:]
}
return b
}
func stripTrailingSpace(b []byte) []byte {
for len(b) > 0 && isSpace(b[len(b)-1]) {
b = b[:len(b)-1]
}
return b
}
func collapseSpace(b []byte) []byte {
return stripSpaceExt(b, true)
}
func stripSpace(b []byte) []byte {
return stripSpaceExt(b, false)
}
func stripSpaceExt(b []byte, isCollapse bool) []byte {
if len(b) == 0 {
return b
}
var dst []byte
if isCollapse && isSpace(b[0]) {
dst = append(dst, ' ')
}
isLastSpace := isSpace(b[len(b)-1])
for len(b) > 0 {
n := bytes.IndexByte(b, '\n')
if n < 0 {
n = len(b)
}
z := b[:n]
if n == len(b) {
b = b[n:]
} else {
b = b[n+1:]
}
z = stripLeadingSpace(z)
z = stripTrailingSpace(z)
if len(z) == 0 {
continue
}
dst = append(dst, z...)
if isCollapse {
dst = append(dst, ' ')
}
}
if isCollapse && !isLastSpace && len(dst) > 0 {
dst = dst[:len(dst)-1]
}
return dst
}
func isSpace(c byte) bool {
return unicode.IsSpace(rune(c))
}
func isUpper(c byte) bool {
return unicode.IsUpper(rune(c))
}
func readFile(cwd, filename string) ([]byte, error) {
if len(filename) == 0 {
return nil, errors.New("filename cannot be empty")
}
if filename[0] != '/' {
cwdAbs, err := filepath.Abs(cwd)
if err != nil {
return nil, err
}
dir, _ := filepath.Split(cwdAbs)
filename = filepath.Join(dir, filename)
}
return ioutil.ReadFile(filename)
}

22
vendor/github.com/valyala/quicktemplate/qtc/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2016 Aliaksandr Valialkin, VertaMedia
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

33
vendor/github.com/valyala/quicktemplate/qtc/README.md generated vendored Normal file
View File

@@ -0,0 +1,33 @@
# qtc
Template compiler (converter) for [quicktemplate](https://github.com/valyala/quicktemplate).
Converts quicktemplate files into Go code. By default these files
have `.qtpl` extension.
# Usage
```
$ go get -u github.com/valyala/quicktemplate/qtc
$ qtc -h
```
`qtc` may be called either directly or via [go generate](https://blog.golang.org/generate).
The latter case is preffered. Just put the following line near the `main` function:
```go
package main
//go:generate qtc -dir=path/to/directory/with/templates
func main() {
// main code here
}
```
Then run `go generate` whenever you need re-generating template code.
Directory with templates may contain arbirary number of subdirectories -
`qtc` generates template code recursively for each subdirectory.
Directories with templates may also contain arbitrary `.go` files - contents
of these files may be used inside templates. Such Go files usually contain
various helper functions and structs.

174
vendor/github.com/valyala/quicktemplate/qtc/main.go generated vendored Normal file
View File

@@ -0,0 +1,174 @@
// Command qtc is a compiler for quicktemplate files.
//
// See https://github.com/valyala/quicktemplate/qtc for details.
package main
import (
"flag"
"go/format"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
"strings"
"github.com/valyala/quicktemplate/parser"
)
var (
dir = flag.String("dir", ".", "Path to directory with template files to compile. "+
"Only files with ext extension are compiled. See ext flag for details.\n"+
"The compiler recursively processes all the subdirectories.\n"+
"Compiled template files are placed near the original file with .go extension added.")
file = flag.String("file", "", "Path to template file to compile.\n"+
"Flags -dir and -ext are ignored if file is set.\n"+
"The compiled file will be placed near the original file with .go extension added.")
ext = flag.String("ext", "qtpl", "Only files with this extension are compiled")
skipLineComments = flag.Bool("skipLineComments", false, "Don't write line comments")
)
var logger = log.New(os.Stderr, "qtc: ", log.LstdFlags)
var filesCompiled int
func main() {
flag.Parse()
if len(*file) > 0 {
compileSingleFile(*file)
return
}
if len(*ext) == 0 {
logger.Fatalf("ext cannot be empty")
}
if len(*dir) == 0 {
*dir = "."
}
if (*ext)[0] != '.' {
*ext = "." + *ext
}
logger.Printf("Compiling *%s template files in directory %q", *ext, *dir)
compileDir(*dir)
logger.Printf("Total files compiled: %d", filesCompiled)
}
func compileSingleFile(filename string) {
fi, err := os.Stat(filename)
if err != nil {
logger.Fatalf("cannot stat file %q: %s", filename, err)
}
if fi.IsDir() {
logger.Fatalf("cannot compile directory %q. Use -dir flag", filename)
}
compileFile(filename)
}
func compileDir(path string) {
fi, err := os.Stat(path)
if err != nil {
logger.Fatalf("cannot compile files in %q: %s", path, err)
}
if !fi.IsDir() {
logger.Fatalf("cannot compile files in %q: it is not directory", path)
}
d, err := os.Open(path)
if err != nil {
logger.Fatalf("cannot compile files in %q: %s", path, err)
}
defer d.Close()
fis, err := d.Readdir(-1)
if err != nil {
logger.Fatalf("cannot read files in %q: %s", path, err)
}
var names []string
for _, fi = range fis {
name := fi.Name()
if name == "." || name == ".." {
continue
}
if !fi.IsDir() {
names = append(names, name)
} else {
subPath := filepath.Join(path, name)
compileDir(subPath)
}
}
sort.Strings(names)
for _, name := range names {
if strings.HasSuffix(name, *ext) {
filename := filepath.Join(path, name)
compileFile(filename)
}
}
}
func compileFile(infile string) {
outfile := infile + ".go"
logger.Printf("Compiling %q to %q...", infile, outfile)
inf, err := os.Open(infile)
if err != nil {
logger.Fatalf("cannot open file %q: %s", infile, err)
}
tmpfile := outfile + ".tmp"
outf, err := os.Create(tmpfile)
if err != nil {
logger.Fatalf("cannot create file %q: %s", tmpfile, err)
}
packageName, err := getPackageName(infile)
if err != nil {
logger.Fatalf("cannot determine package name for %q: %s", infile, err)
}
parseFunc := parser.Parse
if *skipLineComments {
parseFunc = parser.ParseNoLineComments
}
if err = parseFunc(outf, inf, infile, packageName); err != nil {
logger.Fatalf("error when parsing file %q: %s", infile, err)
}
if err = outf.Close(); err != nil {
logger.Fatalf("error when closing file %q: %s", tmpfile, err)
}
if err = inf.Close(); err != nil {
logger.Fatalf("error when closing file %q: %s", infile, err)
}
// prettify the output file
uglyCode, err := ioutil.ReadFile(tmpfile)
if err != nil {
logger.Fatalf("cannot read file %q: %s", tmpfile, err)
}
prettyCode, err := format.Source(uglyCode)
if err != nil {
logger.Fatalf("error when formatting compiled code for %q: %s. See %q for details", infile, err, tmpfile)
}
if err = ioutil.WriteFile(outfile, prettyCode, 0666); err != nil {
logger.Fatalf("error when writing file %q: %s", outfile, err)
}
if err = os.Remove(tmpfile); err != nil {
logger.Fatalf("error when removing file %q: %s", tmpfile, err)
}
filesCompiled++
}
func getPackageName(filename string) (string, error) {
filenameAbs, err := filepath.Abs(filename)
if err != nil {
return "", err
}
dir, _ := filepath.Split(filenameAbs)
return filepath.Base(dir), nil
}

6
vendor/modules.txt vendored
View File

@@ -560,9 +560,13 @@ github.com/puzpuzpuz/xsync/v3
# github.com/rivo/uniseg v0.4.7
## explicit; go 1.18
github.com/rivo/uniseg
# github.com/rogpeppe/go-internal v1.14.1
## explicit; go 1.23
# github.com/russross/blackfriday/v2 v2.1.0
## explicit
github.com/russross/blackfriday/v2
# github.com/spf13/pflag v1.0.6
## explicit; go 1.12
# github.com/stretchr/testify v1.10.0
## explicit; go 1.17
github.com/stretchr/testify/assert
@@ -593,6 +597,8 @@ github.com/valyala/histogram
# github.com/valyala/quicktemplate v1.8.0
## explicit; go 1.17
github.com/valyala/quicktemplate
github.com/valyala/quicktemplate/parser
github.com/valyala/quicktemplate/qtc
# github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1
## explicit; go 1.15
github.com/xrash/smetrics