mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-06-09 11:54:31 +03:00
Compare commits
111 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4cb6bcd2d7 | ||
|
|
6b1317b6a4 | ||
|
|
b7fcdb528d | ||
|
|
dabbf930d8 | ||
|
|
1c669a69a8 | ||
|
|
7119f294f3 | ||
|
|
8a057e705a | ||
|
|
b65236530c | ||
|
|
ae04378424 | ||
|
|
bf95fbfc1d | ||
|
|
78d2715d04 | ||
|
|
d0ffb49ee2 | ||
|
|
b7f4fc6e0d | ||
|
|
d48363534a | ||
|
|
0acdab3ab9 | ||
|
|
7e8dcf9ddc | ||
|
|
aa90b93778 | ||
|
|
de523c81b9 | ||
|
|
a724dde90a | ||
|
|
fb8e56d8a2 | ||
|
|
f0c207fae2 | ||
|
|
d3794eb994 | ||
|
|
f765985947 | ||
|
|
e614a14b21 | ||
|
|
9d160f9048 | ||
|
|
d7932775cc | ||
|
|
eec76718e9 | ||
|
|
093a891762 | ||
|
|
c03e4ef9d6 | ||
|
|
de7f315231 | ||
|
|
97a0c80904 | ||
|
|
09105ff49c | ||
|
|
2859a452d4 | ||
|
|
170e2f54ab | ||
|
|
8b116b619a | ||
|
|
6e6d62284c | ||
|
|
a02a12f639 | ||
|
|
f818ab497b | ||
|
|
b73802372a | ||
|
|
2f05f90888 | ||
|
|
7e4bcbd853 | ||
|
|
a11659013f | ||
|
|
a6b2b2c005 | ||
|
|
c2afa3fdd7 | ||
|
|
d4cc934c77 | ||
|
|
870270c75e | ||
|
|
7addbfc831 | ||
|
|
1c477bc2fc | ||
|
|
d57214244d | ||
|
|
84b986b2fc | ||
|
|
1052effb6d | ||
|
|
266788be14 | ||
|
|
cf18df367d | ||
|
|
72ab3f7230 | ||
|
|
30a922f383 | ||
|
|
2c67232565 | ||
|
|
86f99c6b55 | ||
|
|
3c1434118e | ||
|
|
27a417bcd3 | ||
|
|
6fa806f1ca | ||
|
|
f5500251d9 | ||
|
|
5d6d2ef3a6 | ||
|
|
0208d8c103 | ||
|
|
465923b181 | ||
|
|
a1f3795b78 | ||
|
|
414cd39659 | ||
|
|
d100341394 | ||
|
|
6251762787 | ||
|
|
48d033a198 | ||
|
|
4aaee33860 | ||
|
|
6c0d36e4a9 | ||
|
|
ef9a8989fd | ||
|
|
5d27642106 | ||
|
|
0deabbbb4a | ||
|
|
67b41c080d | ||
|
|
6fcbd17bdd | ||
|
|
9ce5c0c33f | ||
|
|
c5daf8a27b | ||
|
|
d9d01f976b | ||
|
|
1f19c167a4 | ||
|
|
cdf1e6684b | ||
|
|
28ea993872 | ||
|
|
149c0c4a6d | ||
|
|
4f8a3af061 | ||
|
|
57a4af98fa | ||
|
|
3fa9ab4a49 | ||
|
|
47a038401b | ||
|
|
077f8cbe1c | ||
|
|
4057305148 | ||
|
|
bb06b98202 | ||
|
|
4adb96161a | ||
|
|
4c8e01b312 | ||
|
|
51c529a2b6 | ||
|
|
1437d6db0c | ||
|
|
e60c0d0bae | ||
|
|
462913ed2f | ||
|
|
1e69c151eb | ||
|
|
348edd92fe | ||
|
|
352485b0de | ||
|
|
9e40eec7d8 | ||
|
|
e205975716 | ||
|
|
6e668fd480 | ||
|
|
47390d8947 | ||
|
|
ba4a2c8bca | ||
|
|
0d7a3f4eb3 | ||
|
|
fc499ab501 | ||
|
|
3adf8c5a6f | ||
|
|
0d1855f661 | ||
|
|
bcd139362b | ||
|
|
6c24c5caa3 | ||
|
|
ef6ab3d2c9 |
169
CHANGELOG.md
169
CHANGELOG.md
@@ -1,169 +0,0 @@
|
||||
# CHANGELOG
|
||||
|
||||
# tip
|
||||
|
||||
|
||||
# [v1.46.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.46.0)
|
||||
|
||||
* FEATURE: optimize requests to `/api/v1/labels` and `/api/v1/label/<name>/values` when `start` and `end` args are set.
|
||||
* FEATURE: reduce memory usage when query touches big number of time series.
|
||||
* FEATURE: vmagent: reduce memory usage when `kubernetes_sd_config` discovers big number of scrape targets (e.g. hundreds of thouthands) and the majority of these targets (99%)
|
||||
are dropped during relabeling. Previously labels for all the dropped targets were displayed at `/api/v1/targets` page. Now only up to `-promscrape.maxDroppedTargets` such
|
||||
targets are displayed. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/878 for details.
|
||||
* FEATURE: vmagent: reduce memory usage when scraping big number of targets with big number of temporary labels starting with `__`.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825
|
||||
* FEATURE: vmagent: add `/ready` HTTP endpoint, which returns 200 OK status code when all the service discovery has been initialized.
|
||||
This may be useful during rolling upgrades. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/875
|
||||
|
||||
* BUGFIX: vmagent: eliminate data race when `-promscrape.streamParse` command-line is set. Previously this mode could result in scraped metrics with garbage labels.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825#issuecomment-723198247 for details.
|
||||
* BUGFIX: properly calculate `topk_*` and `bottomk_*` functions from [MetricsQL](https://victoriametrics.github.io/MetricsQL.html) for time series with gaps.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/883
|
||||
|
||||
|
||||
# [v1.45.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.45.0)
|
||||
|
||||
* FEATURE: allow setting `-retentionPeriod` smaller than one month. I.e. `-retentionPeriod=3d`, `-retentionPeriod=2w`, etc. is supported now.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/173
|
||||
* FEATURE: optimize more cases according to https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization . Now the following cases are optimized too:
|
||||
* `rollup_func(foo{filters}[d]) op bar` -> `rollup_func(foo{filters}[d]) op bar{filters}`
|
||||
* `transform_func(foo{filters}) op bar` -> `transform_func(foo{filters}) op bar{filters}`
|
||||
* `num_or_scalar op foo{filters} op bar` -> `num_or_scalar op foo{filters} op bar{filters}`
|
||||
* FEATURE: improve time series search for queries with multiple label filters. I.e. `foo{label1="value", label2=~"regexp"}`.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/781
|
||||
* FEATURE: vmagent: add `stream parse` mode. This mode allows reducing memory usage when individual scrape targets expose tens of millions of metrics.
|
||||
For example, during scraping Prometheus in [federation](https://prometheus.io/docs/prometheus/latest/federation/) mode.
|
||||
See `-promscrape.streamParse` command-line option and `stream_parse: true` config option for `scrape_config` section in `-promscrape.config`.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825 and [troubleshooting docs for vmagent](https://victoriametrics.github.io/vmagent.html#troubleshooting).
|
||||
* FEATURE: vmalert: add `-dryRun` command-line option for validating the provided config files without the need to start `vmalert` service.
|
||||
* FEATURE: accept optional third argument of string type at `topk_*` and `bottomk_*` functions. This is label name for additional time series to return with the sum of time series outside top/bottom K. See [MetricsQL docs](https://victoriametrics.github.io/MetricsQL.html) for more details.
|
||||
* FEATURE: vmagent: expose `/api/v1/targets` page according to [the corresponding Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/#targets).
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/643
|
||||
|
||||
* BUGFIX: vmagent: properly handle OpenStack endpoint ending with `v3.0` such as `https://ostack.example.com:5000/v3.0`
|
||||
in the same way as Prometheus does. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/728#issuecomment-709914803
|
||||
* BUGFIX: drop trailing data points for time series with a single raw sample. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748
|
||||
* BUGFIX: do not drop trailing data points for instant queries to `/api/v1/query`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/845
|
||||
* BUGFIX: vmbackup: fix panic when `-origin` isn't specified. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/856
|
||||
* BUGFIX: vmalert: skip automatically added labels on alerts restore. Label `alertgroup` was introduced in [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/611)
|
||||
and automatically added to generated time series. By mistake, this new label wasn't correctly purged on restore event and affected alert's ID uniqueness.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/870
|
||||
* BUGFIX: vmagent: fix panic at scrape error body formating. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/864
|
||||
* BUGFIX: vmagent: add leading missing slash to metrics path like Prometheus does. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/835
|
||||
* BUGFIX: vmagent: drop packet if remote storage returns 4xx status code. This make the behaviour consistent with Prometheus.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
|
||||
* BUGFIX: vmagent: properly handle 301 redirects. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/869
|
||||
|
||||
|
||||
# [v1.44.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.44.0)
|
||||
|
||||
* FEATURE: automatically add missing label filters to binary operands as described at https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization .
|
||||
This should improve performance for queries with missing label filters in binary operands. For example, the following query should work faster now, because it shouldn't
|
||||
fetch and discard time series for `node_filesystem_files_free` metric without matching labels for the left side of the expression:
|
||||
```
|
||||
node_filesystem_files{ host="$host", mountpoint="/" } - node_filesystem_files_free
|
||||
```
|
||||
* FEATURE: vmagent: add Docker Swarm service discovery (aka [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)).
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/656
|
||||
* FEATURE: add ability to export data in CSV format. See [these docs](https://victoriametrics.github.io/#how-to-export-csv-data) for details.
|
||||
* FEATURE: vmagent: add `-promscrape.suppressDuplicateScrapeTargetErrors` command-line flag for suppressing `duplicate scrape target` errors.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/651 and https://victoriametrics.github.io/vmagent.html#troubleshooting .
|
||||
* FEATURE: vmagent: show original labels before relabeling is applied on `duplicate scrape target` errors. This should simplify debugging for incorrect relabeling.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/651
|
||||
* FEATURE: vmagent: `/targets` page now accepts optional `show_original_labels=1` query arg for displaying original labels for each target before relabeling is applied.
|
||||
This should simplify debugging for target relabeling configs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/651
|
||||
* FEATURE: add `-finalMergeDelay` command-line flag for configuring the delay before final merge for per-month partitions.
|
||||
The final merge is started after no new data is ingested into per-month partition during `-finalMergeDelay`.
|
||||
* FEATURE: add `vm_rows_added_to_storage_total` metric, which shows the total number of rows added to storage since app start.
|
||||
The `sum(rate(vm_rows_added_to_storage_total))` can be smaller than `sum(rate(vm_rows_inserted_total))` if certain metrics are dropped
|
||||
due to [relabeling](https://victoriametrics.github.io/#relabeling). The `sum(rate(vm_rows_added_to_storage_total))` can be bigger
|
||||
than `sum(rate(vm_rows_inserted_total))` if [replication](https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#replication-and-data-safety) is enabled.
|
||||
* FEATURE: keep metric name after applying [MetricsQL](https://victoriametrics.github.io/MetricsQL.html) functions, which don't change time series meaning.
|
||||
The list of such functions:
|
||||
* `keep_last_value`
|
||||
* `keep_next_value`
|
||||
* `interpolate`
|
||||
* `running_min`
|
||||
* `running_max`
|
||||
* `running_avg`
|
||||
* `range_min`
|
||||
* `range_max`
|
||||
* `range_avg`
|
||||
* `range_first`
|
||||
* `range_last`
|
||||
* `range_quantile`
|
||||
* `smooth_exponential`
|
||||
* `ceil`
|
||||
* `floor`
|
||||
* `round`
|
||||
* `clamp_min`
|
||||
* `clamp_max`
|
||||
* `max_over_time`
|
||||
* `min_over_time`
|
||||
* `avg_over_time`
|
||||
* `quantile_over_time`
|
||||
* `mode_over_time`
|
||||
* `geomean_over_time`
|
||||
* `holt_winters`
|
||||
* `predict_linear`
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/674
|
||||
|
||||
* BUGFIX: properly handle stale time series after K8S deployment. Previously such time series could be double-counted.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748
|
||||
* BUGFIX: return a single time series at max from `absent()` function like Prometheus does.
|
||||
* BUGFIX: vmalert: accept days, weeks and years in `for: ` part of config like Prometheus does. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/817
|
||||
* BUGFIX: fix `mode_over_time(m[d])` calculations. Previously the function could return incorrect results.
|
||||
|
||||
|
||||
# [v1.43.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.43.0)
|
||||
|
||||
* FEATURE: reduce CPU usage for repeated queries over sliding time window when no new time series are added to the database.
|
||||
Typical use cases: repeated evaluation of alerting rules in [vmalert](https://victoriametrics.github.io/vmalert.html) or dashboard auto-refresh in Grafana.
|
||||
* FEATURE: vmagent: add OpenStack service discovery aka [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config).
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/728 .
|
||||
* FEATURE: vmalert: make `-maxIdleConnections` configurable for datasource HTTP client. This option can be used for minimizing connection churn.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/795 .
|
||||
* FEATURE: add `-influx.maxLineSize` command-line flag for configuring the maximum size for a single Influx line during parsing.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/807
|
||||
|
||||
* BUGFIX: properly handle `inf` values during [background merge of LSM parts](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
|
||||
Previously `Inf` values could result in `NaN` values for adjancent samples in time series. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/805 .
|
||||
* BUGFIX: fill gaps on graphs for `range_*` and `running_*` functions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/806 .
|
||||
* BUGFIX: make a copy of label with new name during relabeling with `action: labelmap` in the same way as Prometheus does.
|
||||
Previously the original label name has been replaced. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/812 .
|
||||
* BUGFIX: support parsing floating-point timestamp like Graphite Carbon does. Such timestmaps are truncated to seconds.
|
||||
|
||||
|
||||
# [v1.42.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.42.0)
|
||||
|
||||
* FEATURE: use all the available CPU cores when accepting data via a single TCP connection
|
||||
for [all the supported protocols](https://victoriametrics.github.io/#how-to-import-time-series-data).
|
||||
Previously data ingested via a single TCP connection could use only a single CPU core. This could limit data ingestion performance.
|
||||
The main benefit of this feature is that data can be imported at max speed via a single connection - there is no need to open multiple concurrent
|
||||
connections to VictoriaMetrics or [vmagent](https://victoriametrics.github.io/vmagent.html) in order to achieve the maximum data ingestion speed.
|
||||
* FEATURE: cluster: improve performance for data ingestion path from `vminsert` to `vmstorage` nodes. The maximum data ingestion performance
|
||||
for a single connection between `vminsert` and `vmstorage` node scales with the number of available CPU cores on `vmstorage` side.
|
||||
This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/791 .
|
||||
* FEATURE: add ability to export / import data in native format via `/api/v1/export/native` and `/api/v1/import/native`.
|
||||
This is the most optimized approach for data migration between VictoriaMetrics instances. Both single-node and cluster instances are supported.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/787#issuecomment-700632551 .
|
||||
* FEATURE: add `reduce_mem_usage` query option to `/api/v1/export` in order to reduce memory usage during data export / import.
|
||||
See [these docs](https://victoriametrics.github.io/#how-to-export-data-in-json-line-format) for details.
|
||||
* FEATURE: improve performance for `/api/v1/series` handler when it returns big number of time series.
|
||||
* FEATURE: add `vm_merge_need_free_disk_space` metric, which can be used for estimating the number of deferred background data merges due to the lack of free disk space.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/686 .
|
||||
* FEATURE: add OpenBSD support. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/785 .
|
||||
|
||||
* BUGFIX: properly apply `-search.maxStalenessInterval` command-line flag value. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784 .
|
||||
* BUGFIX: fix displaying data in Grafana tables. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720 .
|
||||
* BUGFIX: do not adjust the number of detected CPU cores found at `/sys/devices/system/cpu/online`.
|
||||
The adjustement was increasing the resulting GOMAXPROC by 1, which looked confusing to users.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/685#issuecomment-698595309 .
|
||||
* BUGFIX: vmagent: do not show `-remoteWrite.url` in initial logs if `-remoteWrite.showURL` isn't set. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/773 .
|
||||
* BUGFIX: properly handle case when [/metrics/find](https://victoriametrics.github.io/#graphite-metrics-api-usage) finds both a leaf and a node for the given `query=prefix.*`.
|
||||
In this case only the node must be returned with stripped dot in the end of id as carbonapi does.
|
||||
|
||||
|
||||
# Previous releases
|
||||
|
||||
See [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
|
||||
5
Makefile
5
Makefile
@@ -122,8 +122,8 @@ benchmark-pure:
|
||||
GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./app/...
|
||||
|
||||
vendor-update:
|
||||
GO111MODULE=on go get -u ./lib/...
|
||||
GO111MODULE=on go get -u ./app/...
|
||||
GO111MODULE=on go get -u -d ./lib/...
|
||||
GO111MODULE=on go get -u -d ./app/...
|
||||
GO111MODULE=on go mod tidy
|
||||
GO111MODULE=on go mod vendor
|
||||
|
||||
@@ -156,4 +156,3 @@ docs-sync:
|
||||
cp app/vmbackup/README.md docs/vmbackup.md
|
||||
cp app/vmrestore/README.md docs/vmrestore.md
|
||||
cp README.md docs/Single-server-VictoriaMetrics.md
|
||||
cp CHANGELOG.md docs/
|
||||
|
||||
210
README.md
210
README.md
@@ -10,23 +10,27 @@
|
||||
|
||||
## VictoriaMetrics
|
||||
|
||||
VictoriaMetrics is fast, cost-effective and scalable time-series database.
|
||||
VictoriaMetrics is fast, cost-effective and scalable monitoring solution and time series database.
|
||||
|
||||
It is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
|
||||
[docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and
|
||||
in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics and see [how to start it](#how-to-start-victoriametrics).
|
||||
[docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Snap package](https://snapcraft.io/victoriametrics)
|
||||
and in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics and see [how to start it](#how-to-start-victoriametrics).
|
||||
If you use Ubuntu, then just run `snap install victoriametrics` in order to install and run it.
|
||||
Then read [Prometheus setup](#prometheus-setup) and [Grafana setup](#grafana-setup) docs.
|
||||
|
||||
Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
|
||||
|
||||
See our [Wiki](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki) for additional documentation.
|
||||
|
||||
[Contact us](mailto:info@victoriametrics.com) if you need paid enterprise support for VictoriaMetrics.
|
||||
See [features available for enterprise customers](https://github.com/VictoriaMetrics/VictoriaMetrics/issues?q=is%3Aissue+label%3Aenterprise).
|
||||
See [features available for enterprise customers](https://victoriametrics.com/enterprise.html).
|
||||
|
||||
|
||||
## Case studies and talks
|
||||
|
||||
* [Adidas](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#adidas)
|
||||
Click on a link in order to read the corresponding case study
|
||||
|
||||
* [adidas](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#adidas)
|
||||
* [CERN](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#cern)
|
||||
* [COLOPL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#colopl)
|
||||
* [Zerodha](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#zerodha)
|
||||
@@ -46,8 +50,8 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
|
||||
* VictoriaMetrics can be used as long-term storage for Prometheus or for [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md).
|
||||
See [these docs](#prometheus-setup) for details.
|
||||
* Supports [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/), so it can be used as Prometheus drop-in replacement in Grafana.
|
||||
VictoriaMetrics implements [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL) query language, which is inspired by PromQL.
|
||||
* Supports global query view. Multiple Prometheus instances may write data into VictoriaMetrics. Later this data may be used in a single query.
|
||||
VictoriaMetrics implements [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL) query language, which inspired by PromQL. MetricsQL is backwards-compatible with PromQL.
|
||||
* Supports global query view. Multiple Prometheus instances or any other data sources may write data into VictoriaMetrics. Later this data may be queried in a single query.
|
||||
* High performance and good scalability for both [inserts](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
|
||||
and [selects](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4).
|
||||
[Outperforms InfluxDB and TimescaleDB by up to 20x](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
|
||||
@@ -104,7 +108,9 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
|
||||
* [How to send data from OpenTSDB-compatible agents](#how-to-send-data-from-opentsdb-compatible-agents)
|
||||
* [Prometheus querying API usage](#prometheus-querying-api-usage)
|
||||
* [Prometheus querying API enhancements](#prometheus-querying-api-enhancements)
|
||||
* [Graphite Metrics API usage](#graphite-metrics-api-usage)
|
||||
* [Graphite API usage](#graphite-api-usage)
|
||||
* [Graphite Metrics API usage](#graphite-metrics-api-usage)
|
||||
* [Graphite Tags API usage](#graphite-tags-api-usage)
|
||||
* [How to build from sources](#how-to-build-from-sources)
|
||||
* [Development build](#development-build)
|
||||
* [Production build](#production-build)
|
||||
@@ -157,7 +163,7 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
|
||||
* [We kindly ask](#we-kindly-ask)
|
||||
|
||||
|
||||
### How to start VictoriaMetrics
|
||||
## How to start VictoriaMetrics
|
||||
|
||||
Start VictoriaMetrics [executable](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
|
||||
or [docker image](https://hub.docker.com/r/victoriametrics/victoria-metrics/) with the desired command-line flags.
|
||||
@@ -174,7 +180,7 @@ VictoriaMetrics accepts [Prometheus querying API requests](#prometheus-querying-
|
||||
|
||||
It is recommended setting up [monitoring](#monitoring) for VictoriaMetrics.
|
||||
|
||||
#### Environment variables
|
||||
### Environment variables
|
||||
|
||||
Each flag value can be set via environment variables according to these rules:
|
||||
|
||||
@@ -184,7 +190,7 @@ Each flag value can be set via environment variables according to these rules:
|
||||
* It is possible setting prefix for environment vars with `-envflag.prefix`. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_`
|
||||
|
||||
|
||||
### Prometheus setup
|
||||
## Prometheus setup
|
||||
|
||||
Prometheus must be configured with [remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
|
||||
in order to send data to VictoriaMetrics. Add the following lines
|
||||
@@ -240,7 +246,7 @@ Take a look also at [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics
|
||||
which can be used as faster and less resource-hungry alternative to Prometheus in certain cases.
|
||||
|
||||
|
||||
### Grafana setup
|
||||
## Grafana setup
|
||||
|
||||
Create [Prometheus datasource](http://docs.grafana.org/features/datasources/prometheus/) in Grafana with the following url:
|
||||
|
||||
@@ -255,7 +261,7 @@ or [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL
|
||||
which is used by Grafana.
|
||||
|
||||
|
||||
### How to upgrade VictoriaMetrics
|
||||
## How to upgrade VictoriaMetrics
|
||||
|
||||
It is safe upgrading VictoriaMetrics to new versions unless [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
|
||||
say otherwise. It is safe skipping multiple versions during the upgrade unless [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) say otherwise.
|
||||
@@ -273,7 +279,7 @@ Prometheus doesn't drop data during VictoriaMetrics restart.
|
||||
See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details.
|
||||
|
||||
|
||||
### How to apply new config to VictoriaMetrics
|
||||
## How to apply new config to VictoriaMetrics
|
||||
|
||||
VictoriaMetrics is configured via command-line flags, so it must be restarted when new command-line flags should be applied:
|
||||
|
||||
@@ -285,7 +291,7 @@ Prometheus doesn't drop data during VictoriaMetrics restart.
|
||||
See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details.
|
||||
|
||||
|
||||
### How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter)
|
||||
## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter)
|
||||
|
||||
VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file).
|
||||
Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets.
|
||||
@@ -300,6 +306,8 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la
|
||||
* [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
|
||||
* [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config)
|
||||
* [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)
|
||||
* [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)
|
||||
|
||||
|
||||
Other `*_sd_config` types will be supported in the future.
|
||||
|
||||
@@ -310,7 +318,7 @@ VictoriaMetrics also supports [importing data in Prometheus exposition format](#
|
||||
See also [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md), which can be used as drop-in replacement for Prometheus.
|
||||
|
||||
|
||||
### How to send data from InfluxDB-compatible agents such as [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)
|
||||
## How to send data from InfluxDB-compatible agents such as [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)
|
||||
|
||||
Use `http://<victoriametric-addr>:8428` url instead of InfluxDB url in agents' configs.
|
||||
For instance, put the following lines into `Telegraf` config, so it sends data to VictoriaMetrics instead of InfluxDB:
|
||||
@@ -372,7 +380,7 @@ Note that Influx line protocol expects [timestamps in *nanoseconds* by default](
|
||||
while VictoriaMetrics stores them with *milliseconds* precision.
|
||||
|
||||
|
||||
### How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd)
|
||||
## How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd)
|
||||
|
||||
Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command line flag. For instance,
|
||||
the following command will enable Graphite receiver in VictoriaMetrics on TCP and UDP port `2003`:
|
||||
@@ -404,21 +412,22 @@ The `/api/v1/export` endpoint should return the following response:
|
||||
{"metric":{"__name__":"foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560277406000]}
|
||||
```
|
||||
|
||||
### Querying Graphite data
|
||||
## Querying Graphite data
|
||||
|
||||
Data sent to VictoriaMetrics via `Graphite plaintext protocol` may be read via the following APIs:
|
||||
|
||||
* [Prometheus querying API](#prometheus-querying-api-usage)
|
||||
* Metric names can be explored via [Graphite metrics API](#graphite-metrics-api-usage)
|
||||
* Tags can be explored via [Graphite tags API](#graphite-tags-api-usage)
|
||||
* [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml)
|
||||
|
||||
### How to send data from OpenTSDB-compatible agents
|
||||
## How to send data from OpenTSDB-compatible agents
|
||||
|
||||
VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html)
|
||||
and [HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html) for ingesting OpenTSDB data.
|
||||
The same protocol is used for [ingesting data in KairosDB](https://kairosdb.github.io/docs/build/html/PushingData.html).
|
||||
|
||||
#### Sending data via `telnet put` protocol
|
||||
### Sending data via `telnet put` protocol
|
||||
|
||||
Enable OpenTSDB receiver in VictoriaMetrics by setting `-opentsdbListenAddr` command line flag. For instance,
|
||||
the following command enables OpenTSDB receiver in VictoriaMetrics on TCP and UDP port `4242`:
|
||||
@@ -448,7 +457,7 @@ The `/api/v1/export` endpoint should return the following response:
|
||||
{"metric":{"__name__":"foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560277292000]}
|
||||
```
|
||||
|
||||
#### Sending OpenTSDB data via HTTP `/api/put` requests
|
||||
### Sending OpenTSDB data via HTTP `/api/put` requests
|
||||
|
||||
Enable HTTP server for OpenTSDB `/api/put` requests by setting `-opentsdbHTTPListenAddr` command line flag. For instance,
|
||||
the following command enables OpenTSDB HTTP server on port `4242`:
|
||||
@@ -486,7 +495,7 @@ The `/api/v1/export` endpoint should return the following response:
|
||||
```
|
||||
|
||||
|
||||
### Prometheus querying API usage
|
||||
## Prometheus querying API usage
|
||||
|
||||
VictoriaMetrics supports the following handlers from [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/):
|
||||
|
||||
@@ -495,12 +504,14 @@ VictoriaMetrics supports the following handlers from [Prometheus querying API](h
|
||||
* [/api/v1/series](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers)
|
||||
* [/api/v1/labels](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names)
|
||||
* [/api/v1/label/.../values](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values)
|
||||
* [/api/v1/status/tsdb](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats)
|
||||
* [/api/v1/status/tsdb](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats). VictoriaMetrics accepts optional `topN=N` and `date=YYYY-MM-DD`
|
||||
query args for this handler, where `N` is the number of top entries to return in the response and `YYYY-MM-DD` is the date for collecting the stats.
|
||||
By default top 10 entries are returned and the stats is collected for the current day.
|
||||
* [/api/v1/targets](https://prometheus.io/docs/prometheus/latest/querying/api/#targets) - see [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter) for more details.
|
||||
|
||||
These handlers can be queried from Prometheus-compatible clients such as Grafana or curl.
|
||||
|
||||
#### Prometheus querying API enhancements
|
||||
### Prometheus querying API enhancements
|
||||
|
||||
Additionally to unix timestamps and [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) VictoriaMetrics accepts relative times in `time`, `start` and `end` query args.
|
||||
For example, the following query would return data for the last 30 minutes: `/api/v1/query_range?start=-30m&query=...`.
|
||||
@@ -522,6 +533,14 @@ Additionally VictoriaMetrics provides the following handlers:
|
||||
* `/api/v1/status/active_queries` - it returns a list of currently running queries.
|
||||
|
||||
|
||||
## Graphite API usage
|
||||
|
||||
VictoriaMetrics supports the following Graphite APIs:
|
||||
|
||||
* Metrics API - see [these docs](#graphite-metrics-api-usage).
|
||||
* Tags API - see [these docs](#graphite-tags-api-usage).
|
||||
|
||||
|
||||
### Graphite Metrics API usage
|
||||
|
||||
VictoriaMetrics supports the following handlers from [Graphite Metrics API](https://graphite-api.readthedocs.io/en/latest/api.html#the-metrics-api):
|
||||
@@ -536,42 +555,56 @@ VictoriaMetrics accepts the following additional query args at `/metrics/find` a
|
||||
that start with `node_`. By default `delimiter=.`.
|
||||
|
||||
|
||||
### How to build from sources
|
||||
### Graphite Tags API usage
|
||||
|
||||
VictoriaMetrics supports the following handlers from [Graphite Tags API](https://graphite.readthedocs.io/en/stable/tags.html):
|
||||
|
||||
* [/tags/tagSeries](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb)
|
||||
* [/tags/tagMultiSeries](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb)
|
||||
* [/tags](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags)
|
||||
* [/tags/{tag_name}](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags)
|
||||
* [/tags/findSeries](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags)
|
||||
* [/tags/autoComplete/tags](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support)
|
||||
* [/tags/autoComplete/values](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support)
|
||||
* [/tags/delSeries](https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb)
|
||||
|
||||
|
||||
## How to build from sources
|
||||
|
||||
We recommend using either [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or
|
||||
[docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/) instead of building VictoriaMetrics
|
||||
from sources. Building from sources is reasonable when developing additional features specific
|
||||
to your needs or when testing bugfixes.
|
||||
|
||||
#### Development build
|
||||
### Development build
|
||||
|
||||
1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
|
||||
2. Run `make victoria-metrics` from the root folder of the repository.
|
||||
It builds `victoria-metrics` binary and puts it into the `bin` folder.
|
||||
|
||||
#### Production build
|
||||
### Production build
|
||||
|
||||
1. [Install docker](https://docs.docker.com/install/).
|
||||
2. Run `make victoria-metrics-prod` from the root folder of the repository.
|
||||
It builds `victoria-metrics-prod` binary and puts it into the `bin` folder.
|
||||
|
||||
#### ARM build
|
||||
### ARM build
|
||||
|
||||
ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://blog.cloudflare.com/arm-takes-wing/).
|
||||
|
||||
#### Development ARM build
|
||||
### Development ARM build
|
||||
|
||||
1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
|
||||
2. Run `make victoria-metrics-arm` or `make victoria-metrics-arm64` from the root folder of the repository.
|
||||
It builds `victoria-metrics-arm` or `victoria-metrics-arm64` binary respectively and puts it into the `bin` folder.
|
||||
|
||||
#### Production ARM build
|
||||
### Production ARM build
|
||||
|
||||
1. [Install docker](https://docs.docker.com/install/).
|
||||
2. Run `make victoria-metrics-arm-prod` or `make victoria-metrics-arm64-prod` from the root folder of the repository.
|
||||
It builds `victoria-metrics-arm-prod` or `victoria-metrics-arm64-prod` binary respectively and puts it into the `bin` folder.
|
||||
|
||||
#### Pure Go build (CGO_ENABLED=0)
|
||||
### Pure Go build (CGO_ENABLED=0)
|
||||
|
||||
`Pure Go` mode builds only Go code without [cgo](https://golang.org/cmd/cgo/) dependencies.
|
||||
This is an experimental mode, which may result in a lower compression ratio and slower decompression performance.
|
||||
@@ -581,7 +614,7 @@ Use it with caution!
|
||||
2. Run `make victoria-metrics-pure` from the root folder of the repository.
|
||||
It builds `victoria-metrics-pure` binary and puts it into the `bin` folder.
|
||||
|
||||
#### Building docker images
|
||||
### Building docker images
|
||||
|
||||
Run `make package-victoria-metrics`. It builds `victoriametrics/victoria-metrics:<PKG_TAG>` docker image locally.
|
||||
`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
|
||||
@@ -595,17 +628,20 @@ For example, the following command builds the image on top of [scratch](https://
|
||||
ROOT_IMAGE=scratch make package-victoria-metrics
|
||||
```
|
||||
|
||||
### Start with docker-compose
|
||||
## Start with docker-compose
|
||||
|
||||
[Docker-compose](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/docker-compose.yml)
|
||||
helps to spin up VictoriaMetrics, [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md) and Grafana with one command.
|
||||
More details may be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#folder-contains-basic-images-and-tools-for-building-and-running-victoria-metrics-in-docker).
|
||||
|
||||
### Setting up service
|
||||
|
||||
## Setting up service
|
||||
|
||||
Read [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/43) on how to set up VictoriaMetrics as a service in your OS.
|
||||
There is also [snap package for Ubuntu](https://snapcraft.io/victoriametrics).
|
||||
|
||||
### How to work with snapshots
|
||||
|
||||
## How to work with snapshots
|
||||
|
||||
VictoriaMetrics can create [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
|
||||
for all the data stored under `-storageDataPath` directory.
|
||||
@@ -634,7 +670,7 @@ Steps for restoring from a snapshot:
|
||||
to the directory pointed by `-storageDataPath`.
|
||||
3. Start VictoriaMetrics.
|
||||
|
||||
### How to delete time series
|
||||
## How to delete time series
|
||||
|
||||
Send a request to `http://<victoriametrics-addr>:8428/api/v1/admin/tsdb/delete_series?match[]=<timeseries_selector_for_delete>`,
|
||||
where `<timeseries_selector_for_delete>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
|
||||
@@ -666,7 +702,7 @@ It isn't recommended using delete API for the following cases, since it brings n
|
||||
It is better using `-retentionPeriod` command-line flag for efficient pruning of old data.
|
||||
|
||||
|
||||
### Forced merge
|
||||
## Forced merge
|
||||
|
||||
VictoriaMetrics performs [data compactions in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
|
||||
in order to keep good performance characteristics when accepting new data. These compactions (merges) are performed independently on per-month partitions.
|
||||
@@ -681,7 +717,7 @@ since VictoriaMetrics automatically performs [optimal merges in background](http
|
||||
when new data is ingested into it.
|
||||
|
||||
|
||||
### How to export time series
|
||||
## How to export time series
|
||||
|
||||
VictoriaMetrics provides the following handlers for exporting data:
|
||||
|
||||
@@ -691,11 +727,20 @@ VictoriaMetrics provides the following handlers for exporting data:
|
||||
* `/api/v1/export/csv` for exporting data in CSV. See [these docs](#how-to-export-csv-data) for details.
|
||||
|
||||
|
||||
#### How to export data in native format
|
||||
### How to export data in native format
|
||||
|
||||
Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/native?match[]=<timeseries_selector_for_export>`,
|
||||
where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
|
||||
for metrics to export. Use `{__name__!=""}` selector for fetching all the time series.
|
||||
for metrics to export. Use `{__name__=~".*"}` selector for fetching all the time series.
|
||||
|
||||
On large databases you may experience problems with limit on unique timeseries (default value is 300000). In this case you need to adjust `-search.maxUniqueTimeseries` parameter:
|
||||
|
||||
```bash
|
||||
# count unique timeseries in database
|
||||
wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq '.data[0]'
|
||||
|
||||
# relaunch victoriametrics with search.maxUniqueTimeseries more than value from previous command
|
||||
```
|
||||
|
||||
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
|
||||
unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
|
||||
@@ -703,7 +748,7 @@ unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) val
|
||||
The exported data can be imported to VictoriaMetrics via [/api/v1/import/native](#how-to-import-data-in-native-format).
|
||||
|
||||
|
||||
#### How to export data in JSON line format
|
||||
### How to export data in JSON line format
|
||||
|
||||
Consider [exporting data in native format](#how-to-export-data-in-native-format) if big amounts of data must be migrated between VictoriaMetrics instances,
|
||||
since exporting in native format usually consumes lower amounts of CPU and memory resources, while the resulting exported data occupies lower amounts of disk space.
|
||||
@@ -738,7 +783,7 @@ The maximum duration for each request to `/api/v1/export` is limited by `-search
|
||||
Exported data can be imported via POST'ing it to [/api/v1/import](#how-to-import-data-in-json-line-format).
|
||||
|
||||
|
||||
#### How to export CSV data
|
||||
### How to export CSV data
|
||||
|
||||
Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/csv?format=<format>&match=<timeseries_selector_for_export>`,
|
||||
where:
|
||||
@@ -762,7 +807,7 @@ unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) val
|
||||
The exported CSV data can be imported to VictoriaMetrics via [/api/v1/import/csv](#how-to-import-csv-data).
|
||||
|
||||
|
||||
### How to import time series data
|
||||
## How to import time series data
|
||||
|
||||
Time series data can be imported via any supported ingestion protocol:
|
||||
|
||||
@@ -772,14 +817,14 @@ Time series data can be imported via any supported ingestion protocol:
|
||||
* OpenTSDB telnet put protocol. See [these docs](#sending-data-via-telnet-put-protocol) for details.
|
||||
* OpenTSDB http `/api/put` protocol. See [these docs](#sending-opentsdb-data-via-http-apiput-requests) for details.
|
||||
* `/api/v1/import` for importing data obtained from [/api/v1/export](#how-to-export-data-in-json-line-format).
|
||||
See [these docs](##how-to-import-data-in-json-line-format) for details.
|
||||
See [these docs](#how-to-import-data-in-json-line-format) for details.
|
||||
* `/api/v1/import/native` for importing data obtained from [/api/v1/export/native](#how-to-export-data-in-native-format).
|
||||
See [these docs](#how-to-import-data-in-native-format) for details.
|
||||
* `/api/v1/import/csv` for importing arbitrary CSV data. See [these docs](#how-to-import-csv-data) for details.
|
||||
* `/api/v1/import/prometheus` for importing data in Prometheus exposition format. See [these docs](#how-to-import-data-in-prometheus-exposition-format) for details.
|
||||
|
||||
|
||||
#### How to import data in native format
|
||||
### How to import data in native format
|
||||
|
||||
The most efficient protocol for importing data into VictoriaMetrics is `/api/v1/import/native`.
|
||||
Example for importing data obtained via [/api/v1/export/native](#how-to-export-data-in-native-format):
|
||||
@@ -808,7 +853,7 @@ For example, `/api/v1/import/native?extra_label=foo=bar` would add `"foo":"bar"`
|
||||
Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
|
||||
|
||||
|
||||
#### How to import data in JSON line format
|
||||
### How to import data in JSON line format
|
||||
|
||||
Example for importing data obtained via [/api/v1/export](#how-to-export-data-in-json-line-format):
|
||||
|
||||
@@ -836,7 +881,7 @@ For example, `/api/v1/import?extra_label=foo=bar` would add `"foo":"bar"` label
|
||||
Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
|
||||
|
||||
|
||||
#### How to import CSV data
|
||||
### How to import CSV data
|
||||
|
||||
Arbitrary CSV data can be imported via `/api/v1/import/csv`. The CSV data is imported according to the provided `format` query arg.
|
||||
The `format` query arg must contain comma-separated list of parsing rules for CSV fields. Each rule consists of three parts delimited by a colon:
|
||||
@@ -889,7 +934,7 @@ For example, `/api/v1/import/csv?extra_label=foo=bar` would add `"foo":"bar"` la
|
||||
Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
|
||||
|
||||
|
||||
#### How to import data in Prometheus exposition format
|
||||
### How to import data in Prometheus exposition format
|
||||
|
||||
VictoriaMetrics accepts data in [Prometheus exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format)
|
||||
via `/api/v1/import/prometheus` path. For example, the following line imports a single line in Prometheus exposition format into VictoriaMetrics:
|
||||
@@ -924,7 +969,7 @@ VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-sc
|
||||
|
||||
|
||||
|
||||
### Relabeling
|
||||
## Relabeling
|
||||
|
||||
VictoriaMetrics supports Prometheus-compatible relabeling for all the ingested metrics if `-relabelConfig` command-line flag points
|
||||
to a file containing a list of [relabel_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) entries.
|
||||
@@ -948,7 +993,7 @@ VictoriaMetrics provides the following extra actions for relabeling rules:
|
||||
See also [relabeling in vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md#relabeling).
|
||||
|
||||
|
||||
### Federation
|
||||
## Federation
|
||||
|
||||
VictoriaMetrics exports [Prometheus-compatible federation data](https://prometheus.io/docs/prometheus/latest/federation/)
|
||||
at `http://<victoriametrics-addr>:8428/federate?match[]=<timeseries_selector_for_federation>`.
|
||||
@@ -959,7 +1004,7 @@ on the interval `[now - max_lookback ... now]` is scraped for each time series.
|
||||
For instance, `/federate?match[]=up&max_lookback=1h` would return last points on the `[now - 1h ... now]` interval. This may be useful for time series federation
|
||||
with scrape intervals exceeding `5m`.
|
||||
|
||||
### Capacity planning
|
||||
## Capacity planning
|
||||
|
||||
A rough estimation of the required resources for ingestion path:
|
||||
|
||||
@@ -999,7 +1044,8 @@ The required resources for query path:
|
||||
* Network usage: depends on the frequency and the type of incoming requests. Typical Grafana dashboards usually
|
||||
require negligible network bandwidth.
|
||||
|
||||
### High availability
|
||||
|
||||
## High availability
|
||||
|
||||
* Install multiple VictoriaMetrics instances in distinct datacenters (availability zones).
|
||||
* Pass addresses of these instances to [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md) via `-remoteWrite.url` command-line flag:
|
||||
@@ -1040,7 +1086,7 @@ Another option is to write data simultaneously from Prometheus HA pair to a pair
|
||||
with the enabled de-duplication. See [this section](#deduplication) for details.
|
||||
|
||||
|
||||
### Deduplication
|
||||
## Deduplication
|
||||
|
||||
VictoriaMetrics de-duplicates data points if `-dedup.minScrapeInterval` command-line flag
|
||||
is set to positive duration. For example, `-dedup.minScrapeInterval=60s` would de-duplicate data points
|
||||
@@ -1051,7 +1097,7 @@ write data to the same VictoriaMetrics instance. Note that these Prometheus inst
|
||||
`external_labels` section in their configs, so they write data to the same time series.
|
||||
|
||||
|
||||
### Retention
|
||||
## Retention
|
||||
|
||||
Retention is configured with `-retentionPeriod` command-line flag. For instance, `-retentionPeriod=3` means
|
||||
that the data will be stored for 3 months and then deleted.
|
||||
@@ -1066,7 +1112,7 @@ VictoriaMetrics supports retention smaller than 1 month. For example, `-retentio
|
||||
Older data is eventually deleted during [background merge](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
|
||||
|
||||
|
||||
### Multiple retentions
|
||||
## Multiple retentions
|
||||
|
||||
Just start multiple VictoriaMetrics instances with distinct values for the following flags:
|
||||
|
||||
@@ -1079,7 +1125,7 @@ so it could route requests from particular user to VictoriaMetrics with the desi
|
||||
The same scheme could be implemented for multiple tenants in [VictoriaMetrics cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md).
|
||||
|
||||
|
||||
### Downsampling
|
||||
## Downsampling
|
||||
|
||||
There is no downsampling support at the moment, but:
|
||||
|
||||
@@ -1095,11 +1141,13 @@ It is possible to (ab)use [-dedup.minScrapeInterval](#deduplication) for basic d
|
||||
For instance, if interval between the ingested data points is 15s, then `-dedup.minScrapeInterval=5m` will leave
|
||||
only a single data point out of 20 initial data points per each 5m interval.
|
||||
|
||||
### Multi-tenancy
|
||||
|
||||
## Multi-tenancy
|
||||
|
||||
Single-node VictoriaMetrics doesn't support multi-tenancy. Use [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster) instead.
|
||||
|
||||
### Scalability and cluster version
|
||||
|
||||
## Scalability and cluster version
|
||||
|
||||
Though single-node VictoriaMetrics cannot scale to multiple nodes, it is optimized for resource usage - storage size / bandwidth / IOPS, RAM, CPU.
|
||||
This means that a single-node VictoriaMetrics may scale vertically and substitute a moderately sized cluster built with competing solutions
|
||||
@@ -1109,7 +1157,8 @@ So try single-node VictoriaMetrics at first and then [switch to cluster version]
|
||||
horizontally scalable long-term remote storage for really large Prometheus deployments.
|
||||
[Contact us](mailto:info@victoriametrics.com) for paid support.
|
||||
|
||||
### Alerting
|
||||
|
||||
## Alerting
|
||||
|
||||
It is recommended using [vmalert](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md) for alerting.
|
||||
|
||||
@@ -1120,7 +1169,7 @@ Additionally, alerting can be set up with the following tools:
|
||||
* With Grafana - see [the corresponding docs](https://grafana.com/docs/alerting/rules/).
|
||||
|
||||
|
||||
### Security
|
||||
## Security
|
||||
|
||||
Do not forget protecting sensitive endpoints in VictoriaMetrics when exposing it to untrusted networks such as the internet.
|
||||
Consider setting the following command-line flags:
|
||||
@@ -1140,7 +1189,7 @@ Prefer authorizing all the incoming requests from untrusted networks with [vmaut
|
||||
or similar auth proxy.
|
||||
|
||||
|
||||
### Tuning
|
||||
## Tuning
|
||||
|
||||
* There is no need for VictoriaMetrics tuning since it uses reasonable defaults for command-line flags,
|
||||
which are automatically adjusted for the available CPU and RAM resources.
|
||||
@@ -1156,7 +1205,7 @@ or similar auth proxy.
|
||||
mkfs.ext4 ... -O 64bit,huge_file,extent -T huge
|
||||
```
|
||||
|
||||
### Monitoring
|
||||
## Monitoring
|
||||
|
||||
VictoriaMetrics exports internal metrics in Prometheus format at `/metrics` page.
|
||||
These metrics may be collected by [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md)
|
||||
@@ -1186,7 +1235,7 @@ The most interesting metrics are:
|
||||
VictoriaMetrics also exposes currently running queries with their execution times at `/api/v1/status/active_queries` page.
|
||||
|
||||
|
||||
### Troubleshooting
|
||||
## Troubleshooting
|
||||
|
||||
* It is recommended to use default command-line flag values (i.e. don't set them explicitly) until the need
|
||||
of tweaking these flag values arises.
|
||||
@@ -1196,13 +1245,18 @@ VictoriaMetrics also exposes currently running queries with their execution time
|
||||
|
||||
* It is recommended inspecting logs during troubleshooting, since they may contain useful information.
|
||||
|
||||
* VictoriaMetrics buffers incoming data in memory for up to a few seconds before flushing it to persistent storage.
|
||||
This may lead to the following "issues":
|
||||
* Data becomes available for querying in a few seconds after inserting. It is possible to flush in-memory buffers to persistent storage
|
||||
by requesting `/internal/force_flush` http handler. This handler is mostly needed for testing and debugging purposes.
|
||||
* The last few seconds of inserted data may be lost on unclean shutdown (i.e. OOM, `kill -9` or hardware reset).
|
||||
See [this article for technical details](https://valyala.medium.com/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704).
|
||||
|
||||
* If VictoriaMetrics works slowly and eats more than a CPU core per 100K ingested data points per second,
|
||||
then it is likely you have too many active time series for the current amount of RAM.
|
||||
VictoriaMetrics [exposes](#monitoring) `vm_slow_*` metrics, which could be used as an indicator of low amounts of RAM.
|
||||
It is recommended increasing the amount of RAM on the node with VictoriaMetrics in order to improve
|
||||
ingestion and query performance in this case.
|
||||
Another option is to increase `-memory.allowedPercent` command-line flag value. Be careful with this
|
||||
option, since too big value for `-memory.allowedPercent` may result in high I/O usage.
|
||||
|
||||
* VictoriaMetrics prioritizes data ingestion over data querying. So if it has no enough resources for data ingestion,
|
||||
then data querying may slow down significantly.
|
||||
@@ -1217,9 +1271,9 @@ VictoriaMetrics also exposes currently running queries with their execution time
|
||||
which would start background merge if they had more free disk space.
|
||||
|
||||
* If VictoriaMetrics doesn't work because of certain parts are corrupted due to disk errors,
|
||||
then just remove directories with broken parts. This will recover VictoriaMetrics at the cost
|
||||
of data loss stored in the broken parts. In the future, `vmrecover` tool will be created
|
||||
for automatic recovering from such errors.
|
||||
then just remove directories with broken parts. It is safe removing subdirectories under `<-storageDataPath>/data/{big,small}/YYYY_MM` directories
|
||||
when VictoriaMetrics isn't running. This recovers VictoriaMetrics at the cost of data loss stored in the deleted broken parts.
|
||||
In the future, `vmrecover` tool will be created for automatic recovering from such errors.
|
||||
|
||||
* If you see gaps on the graphs, try resetting the cache by sending request to `/internal/resetRollupResultCache`.
|
||||
If this removes gaps on the graphs, then it is likely data with timestamps older than `-search.cacheTimestampOffset`
|
||||
@@ -1241,10 +1295,15 @@ VictoriaMetrics also exposes currently running queries with their execution time
|
||||
This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
|
||||
metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
|
||||
|
||||
* If you store Graphite metrics like `foo.bar.baz` in VictoriaMetrics, then `-search.treatDotsAsIsInRegexps` command-line flag could be useful.
|
||||
By default `.` chars in regexps match any char. If you need matching only dots, then the `\\.` must be used in regexp filters.
|
||||
When `-search.treatDotsAsIsInRegexps` option is enabled, then dots in regexps are automatically escaped in order to match only dots instead of arbitrary chars.
|
||||
This may significantly increase performance when locating time series for the given label filters.
|
||||
|
||||
* VictoriaMetrics ignores `NaN` values during data ingestion.
|
||||
|
||||
|
||||
### Backfilling
|
||||
## Backfilling
|
||||
|
||||
VictoriaMetrics accepts historical data in arbitrary order of time via [any supported ingestion method](#how-to-import-time-series-data).
|
||||
Make sure that configured `-retentionPeriod` covers timestamps for the backfilled data.
|
||||
@@ -1260,7 +1319,7 @@ Yet another solution is to increase `-search.cacheTimestampOffset` flag value in
|
||||
for data with timestamps close to the current time.
|
||||
|
||||
|
||||
### Data updates
|
||||
## Data updates
|
||||
|
||||
VictoriaMetrics doesn't support updating already existing sample values to new ones. It stores all the ingested data points
|
||||
for the same time series with identical timestamps. While is possible substituting old time series with new time series via
|
||||
@@ -1268,7 +1327,7 @@ for the same time series with identical timestamps. While is possible substituti
|
||||
should be used only for one-off updates. It shouldn't be used for frequent updates because of non-zero overhead related to data removal.
|
||||
|
||||
|
||||
### Replication
|
||||
## Replication
|
||||
|
||||
Single-node VictoriaMetrics doesn't support application-level replication. Use cluster version instead.
|
||||
See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#replication-and-data-safety) for details.
|
||||
@@ -1278,14 +1337,14 @@ Storage-level replication may be offloaded to durable persistent storage such as
|
||||
See also [high availability docs](#high-availability) and [backup docs](#backups).
|
||||
|
||||
|
||||
### Backups
|
||||
## Backups
|
||||
|
||||
VictoriaMetrics supports backups via [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md)
|
||||
and [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md) tools.
|
||||
We also provide provide `vmbackuper` tool for paid enterprise subscribers - see [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/466) for details.
|
||||
|
||||
|
||||
### Profiling
|
||||
## Profiling
|
||||
|
||||
VictoriaMetrics provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
|
||||
|
||||
@@ -1305,6 +1364,7 @@ The command for collecting CPU profile waits for 30 seconds before returning.
|
||||
|
||||
The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
|
||||
|
||||
|
||||
## Integrations
|
||||
|
||||
* [Helm charts for single-node and cluster versions of VictoriaMetrics](https://github.com/VictoriaMetrics/helm-charts).
|
||||
@@ -1316,6 +1376,8 @@ The collected profiles may be analyzed with [go tool pprof](https://github.com/g
|
||||
See [this example](https://github.com/go-graphite/carbonapi/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml).
|
||||
* [Ansible role for installing single-node VictoriaMetrics](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
|
||||
* [Ansible role for installing cluster VictoriaMetrics](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
|
||||
* [Snap package for VictoriaMetrics](https://snapcraft.io/victoriametrics).
|
||||
|
||||
|
||||
## Third-party contributions
|
||||
|
||||
@@ -1324,10 +1386,12 @@ The collected profiles may be analyzed with [go tool pprof](https://github.com/g
|
||||
* [Prometheus -> VictoriaMetrics exporter #2](https://github.com/AnchorFree/tsdb-remote-write)
|
||||
* [Prometheus Oauth proxy](https://gitlab.com/optima_public/prometheus_oauth_proxy) - see [this article](https://medium.com/@richard.holly/powerful-saas-solution-for-detection-metrics-c67b9208d362) for details.
|
||||
|
||||
|
||||
## Contacts
|
||||
|
||||
Contact us with any questions regarding VictoriaMetrics at [info@victoriametrics.com](mailto:info@victoriametrics.com).
|
||||
|
||||
|
||||
## Community and contributions
|
||||
|
||||
Feel free asking any questions regarding VictoriaMetrics:
|
||||
|
||||
@@ -108,3 +108,10 @@ victoria-metrics-package-deb-rpm-all: \
|
||||
victoria-metrics-package-deb-arm64 \
|
||||
victoria-metrics-package-rpm \
|
||||
victoria-metrics-package-rpm-arm64
|
||||
|
||||
### Packaging as snap
|
||||
victoria-metrics-package-snap:
|
||||
which snapcraft || snap install snapcraft
|
||||
which multipass || snap install multipass
|
||||
snapcraft
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
)
|
||||
|
||||
@@ -25,6 +26,8 @@ var (
|
||||
minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Remove superflouos samples from time series if they are located closer to each other than this duration. "+
|
||||
"This may be useful for reducing overhead when multiple identically configured Prometheus instances write data to the same VictoriaMetrics. "+
|
||||
"Deduplication is disabled if the -dedup.minScrapeInterval is 0")
|
||||
dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
|
||||
"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -34,6 +37,18 @@ func main() {
|
||||
buildinfo.Init()
|
||||
logger.Init()
|
||||
cgroup.UpdateGOMAXPROCSToCPUQuota()
|
||||
|
||||
if promscrape.IsDryRun() {
|
||||
*dryRun = true
|
||||
}
|
||||
if *dryRun {
|
||||
if err := promscrape.CheckConfig(); err != nil {
|
||||
logger.Fatalf("error when checking -promscrape.config: %s", err)
|
||||
}
|
||||
logger.Infof("-promscrape.config is ok; exitting with 0 status code")
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
|
||||
startTime := time.Now()
|
||||
storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
|
||||
|
||||
@@ -63,6 +63,22 @@ Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://gi
|
||||
Pass `-help` to `vmagent` in order to see the full list of supported command-line flags with their descriptions.
|
||||
|
||||
|
||||
### Configuration update
|
||||
|
||||
`vmagent` should be restarted in order to update config options set via command-line args.
|
||||
|
||||
`vmagent` supports multiple approaches for reloading configs from updated config files such as `-promscrape.config`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`:
|
||||
|
||||
* Sending `SUGHUP` signal to `vmagent` process:
|
||||
```bash
|
||||
kill -SIGHUP `pidof vmagent`
|
||||
```
|
||||
|
||||
* Sending HTTP request to `http://vmagent:8429/-/reload` endpoint.
|
||||
|
||||
There is also `-promscrape.configCheckInterval` command-line option, which can be used for automatic reloading configs from updated `-promscrape.config` file.
|
||||
|
||||
|
||||
### Use cases
|
||||
|
||||
|
||||
@@ -153,6 +169,8 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
|
||||
[OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
|
||||
* `dockerswarm_sd_configs` - for scraping Docker Swarm targets.
|
||||
See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
|
||||
* `eureka_sd_configs` - for scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka).
|
||||
See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details.
|
||||
|
||||
File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
|
||||
|
||||
@@ -197,6 +215,7 @@ The relabeling can be defined in the following places:
|
||||
|
||||
Read more about relabeling in the following articles:
|
||||
|
||||
* [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
|
||||
* [Life of a label](https://www.robustperception.io/life-of-a-label)
|
||||
* [Discarding targets and timeseries with relabeling](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts)
|
||||
* [Dropping labels at scrape time](https://www.robustperception.io/dropping-metrics-at-scrape-time-with-prometheus)
|
||||
@@ -237,6 +256,9 @@ It may be useful for performing `vmagent` rolling update without scrape loss.
|
||||
* The `/api/v1/targets` page could be useful for debugging relabeling process for scrape targets.
|
||||
This page contains original labels for targets dropped during relabeling (see "droppedTargets" section in the page output). By default up to `-promscrape.maxDroppedTargets` targets are shown here. If your setup drops more targets during relabeling, then increase `-promscrape.maxDroppedTargets` command-line flag value in order to see all the dropped targets. Note that tracking each dropped target requires up to 10Kb of RAM, so big values for `-promscrape.maxDroppedTargets` may result in increased memory usage if big number of scrape targets are dropped during relabeling.
|
||||
|
||||
* If `vmagent` scrapes big number of targets, then `-promscrape.dropOriginalLabels` command-line option may be passed to `vmagent` in order to reduce memory usage.
|
||||
This option drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page, which may result in reduced debuggability for improperly configured per-target relabeling.
|
||||
|
||||
* If `vmagent` scrapes targets with millions of metrics per each target (for instance, when scraping [federation endpoints](https://prometheus.io/docs/prometheus/latest/federation/)),
|
||||
then it is recommended enabling `stream parsing mode` in order to reduce memory usage during scraping. This mode may be enabled either globally for all the scrape targets
|
||||
by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example:
|
||||
|
||||
@@ -48,7 +48,8 @@ var (
|
||||
"Usually :4242 must be set. Doesn't work if empty")
|
||||
opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
|
||||
dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
|
||||
"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . See also -promscrape.config.dryRun")
|
||||
"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
|
||||
"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -68,15 +69,19 @@ func main() {
|
||||
logger.Init()
|
||||
cgroup.UpdateGOMAXPROCSToCPUQuota()
|
||||
|
||||
if *dryRun {
|
||||
if err := flag.Set("promscrape.config.strictParse", "true"); err != nil {
|
||||
logger.Panicf("BUG: cannot set promscrape.config.strictParse=true: %s", err)
|
||||
if promscrape.IsDryRun() {
|
||||
if err := promscrape.CheckConfig(); err != nil {
|
||||
logger.Fatalf("error when checking -promscrape.config: %s", err)
|
||||
}
|
||||
logger.Infof("-promscrape.config is ok; exitting with 0 status code")
|
||||
return
|
||||
}
|
||||
if *dryRun {
|
||||
if err := remotewrite.CheckRelabelConfigs(); err != nil {
|
||||
logger.Fatalf("error when checking relabel configs: %s", err)
|
||||
}
|
||||
if err := promscrape.CheckConfig(); err != nil {
|
||||
logger.Fatalf("error when checking Prometheus config: %s", err)
|
||||
logger.Fatalf("error when checking -promscrape.config: %s", err)
|
||||
}
|
||||
logger.Infof("all the configs are ok; exitting with 0 status code")
|
||||
return
|
||||
@@ -208,13 +213,13 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
return true
|
||||
case "/targets":
|
||||
promscrapeTargetsRequests.Inc()
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
|
||||
promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
|
||||
return true
|
||||
case "/api/v1/targets":
|
||||
promscrapeAPIV1TargetsRequests.Inc()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
state := r.FormValue("state")
|
||||
promscrape.WriteAPIV1Targets(w, state)
|
||||
return true
|
||||
@@ -228,7 +233,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
|
||||
http.Error(w, errMsg, http.StatusTooEarly)
|
||||
} else {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("OK"))
|
||||
}
|
||||
|
||||
@@ -140,7 +140,17 @@ func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series b
|
||||
updated := make(map[uint64]struct{})
|
||||
// update list of active alerts
|
||||
for _, m := range qMetrics {
|
||||
for k, v := range ar.Labels {
|
||||
// apply extra labels
|
||||
m.SetLabel(k, v)
|
||||
}
|
||||
|
||||
h := hash(m)
|
||||
if _, ok := updated[h]; ok {
|
||||
// duplicate may be caused by extra labels
|
||||
// conflicting with the metric labels
|
||||
return nil, fmt.Errorf("labels %v: %w", m.Labels, errDuplicate)
|
||||
}
|
||||
updated[h] = struct{}{}
|
||||
if a, ok := ar.alerts[h]; ok {
|
||||
if a.Value != m.Value {
|
||||
@@ -258,25 +268,11 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time) (*notifie
|
||||
}
|
||||
|
||||
func (ar *AlertingRule) template(a *notifier.Alert) error {
|
||||
// 1. template rule labels with data labels
|
||||
rLabels, err := a.ExecTemplate(ar.Labels)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 2. merge data labels and rule labels
|
||||
// metric labels may be overridden by
|
||||
// rule labels
|
||||
for k, v := range rLabels {
|
||||
a.Labels[k] = v
|
||||
}
|
||||
|
||||
// 3. template merged labels
|
||||
var err error
|
||||
a.Labels, err = a.ExecTemplate(a.Labels)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
a.Annotations, err = a.ExecTemplate(ar.Annotations)
|
||||
return err
|
||||
}
|
||||
@@ -419,14 +415,7 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
|
||||
// drop all extra labels, so hash key will
|
||||
// be identical to time series received in Exec
|
||||
for _, l := range labels {
|
||||
if l.Name == alertNameLabel {
|
||||
continue
|
||||
}
|
||||
if l.Name == alertGroupNameLabel {
|
||||
continue
|
||||
}
|
||||
// drop all overridden labels
|
||||
if _, ok := ar.Labels[l.Name]; ok {
|
||||
if l.Name == alertNameLabel || l.Name == alertGroupNameLabel {
|
||||
continue
|
||||
}
|
||||
m.Labels = append(m.Labels, l)
|
||||
|
||||
@@ -2,6 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -218,19 +220,6 @@ func TestAlertingRule_Exec(t *testing.T) {
|
||||
hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
|
||||
},
|
||||
},
|
||||
{
|
||||
newTestAlertingRule("duplicate", 0),
|
||||
[][]datasource.Metric{
|
||||
{
|
||||
// metrics with the same labelset should result in one alert
|
||||
metricWithLabels(t, "name", "foo", "type", "bar"),
|
||||
metricWithLabels(t, "type", "bar", "name", "foo"),
|
||||
},
|
||||
},
|
||||
map[uint64]*notifier.Alert{
|
||||
hash(metricWithLabels(t, "name", "foo", "type", "bar")): {State: notifier.StateFiring},
|
||||
},
|
||||
},
|
||||
{
|
||||
newTestAlertingRule("for-pending", time.Minute),
|
||||
[][]datasource.Metric{
|
||||
@@ -376,7 +365,7 @@ func TestAlertingRule_Restore(t *testing.T) {
|
||||
alertNameLabel, "",
|
||||
"foo", "bar",
|
||||
"namespace", "baz",
|
||||
// following pair supposed to be dropped
|
||||
// extra labels set by rule
|
||||
"source", "vm",
|
||||
),
|
||||
},
|
||||
@@ -384,6 +373,7 @@ func TestAlertingRule_Restore(t *testing.T) {
|
||||
hash(metricWithLabels(t,
|
||||
"foo", "bar",
|
||||
"namespace", "baz",
|
||||
"source", "vm",
|
||||
)): {State: notifier.StatePending,
|
||||
Start: time.Now().Truncate(time.Hour)},
|
||||
},
|
||||
@@ -442,6 +432,38 @@ func TestAlertingRule_Restore(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertingRule_Exec_Negative(t *testing.T) {
|
||||
fq := &fakeQuerier{}
|
||||
ar := newTestAlertingRule("test", 0)
|
||||
ar.Labels = map[string]string{"job": "test"}
|
||||
|
||||
// successful attempt
|
||||
fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
|
||||
_, err := ar.Exec(context.TODO(), fq, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// label `job` will collide with rule extra label and will make both time series equal
|
||||
fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
|
||||
_, err = ar.Exec(context.TODO(), fq, false)
|
||||
if !errors.Is(err, errDuplicate) {
|
||||
t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
|
||||
}
|
||||
|
||||
fq.reset()
|
||||
|
||||
expErr := "connection reset by peer"
|
||||
fq.setErr(errors.New(expErr))
|
||||
_, err = ar.Exec(context.TODO(), fq, false)
|
||||
if err == nil {
|
||||
t.Fatalf("expected to get err; got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), expErr) {
|
||||
t.Fatalf("expected to get err %q; got %q insterad", expErr, err)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
|
||||
r := newTestAlertingRule(name, 0)
|
||||
r.Labels = make(map[string]string)
|
||||
|
||||
@@ -17,6 +17,23 @@ type Metric struct {
|
||||
Value float64
|
||||
}
|
||||
|
||||
// SetLabel adds or updates existing one label
|
||||
// by the given key and label
|
||||
func (m *Metric) SetLabel(key, value string) {
|
||||
for i, l := range m.Labels {
|
||||
if l.Name == key {
|
||||
m.Labels[i].Value = value
|
||||
return
|
||||
}
|
||||
}
|
||||
m.AddLabel(key, value)
|
||||
}
|
||||
|
||||
// AddLabel appends the given label to the label set
|
||||
func (m *Metric) AddLabel(key, value string) {
|
||||
m.Labels = append(m.Labels, Label{Name: key, Value: value})
|
||||
}
|
||||
|
||||
// Label represents metric's label
|
||||
type Label struct {
|
||||
Name string
|
||||
|
||||
@@ -37,7 +37,7 @@ func (r response) metrics() ([]Metric, error) {
|
||||
}
|
||||
m.Labels = nil
|
||||
for k, v := range r.Data.Result[i].Labels {
|
||||
m.Labels = append(m.Labels, Label{Name: k, Value: v})
|
||||
m.AddLabel(k, v)
|
||||
}
|
||||
m.Timestamp = int64(res.TV[0].(float64))
|
||||
m.Value = f
|
||||
@@ -82,7 +82,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
if s.basicAuthPass != "" {
|
||||
req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass)
|
||||
}
|
||||
|
||||
@@ -172,6 +172,11 @@ func TestGroupStart(t *testing.T) {
|
||||
t.Fatalf("faield to create alert: %s", err)
|
||||
}
|
||||
alert1.State = notifier.StateFiring
|
||||
// add external label
|
||||
alert1.Labels["cluster"] = "east-1"
|
||||
// add rule labels - see config/testdata/rules1-good.rules
|
||||
alert1.Labels["label"] = "bar"
|
||||
alert1.Labels["host"] = inst1
|
||||
alert1.ID = hash(m1)
|
||||
|
||||
alert2, err := r.newAlert(m2, time.Now())
|
||||
@@ -179,6 +184,11 @@ func TestGroupStart(t *testing.T) {
|
||||
t.Fatalf("faield to create alert: %s", err)
|
||||
}
|
||||
alert2.State = notifier.StateFiring
|
||||
// add external label
|
||||
alert2.Labels["cluster"] = "east-1"
|
||||
// add rule labels - see config/testdata/rules1-good.rules
|
||||
alert2.Labels["label"] = "bar"
|
||||
alert2.Labels["host"] = inst2
|
||||
alert2.ID = hash(m2)
|
||||
|
||||
finished := make(chan struct{})
|
||||
|
||||
@@ -28,7 +28,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
req = req.WithContext(ctx)
|
||||
if am.basicAuthPass != "" {
|
||||
req.SetBasicAuth(am.basicAuthUser, am.basicAuthPass)
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"sort"
|
||||
@@ -79,8 +78,6 @@ func (rr *RecordingRule) Close() {
|
||||
metrics.UnregisterMetric(rr.metrics.errors.name)
|
||||
}
|
||||
|
||||
var errDuplicate = errors.New("result contains metrics with the same labelset after applying rule labels")
|
||||
|
||||
// Exec executes RecordingRule expression via the given Querier.
|
||||
func (rr *RecordingRule) Exec(ctx context.Context, q datasource.Querier, series bool) ([]prompbmarshal.TimeSeries, error) {
|
||||
if !series {
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
@@ -25,3 +26,5 @@ type Rule interface {
|
||||
// such as metrics unregister
|
||||
Close()
|
||||
}
|
||||
|
||||
var errDuplicate = errors.New("result contains metrics with the same labelset after applying rule labels")
|
||||
|
||||
@@ -40,7 +40,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
|
||||
httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
|
||||
return true
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Write(data)
|
||||
return true
|
||||
case "/api/v1/alerts":
|
||||
@@ -49,7 +49,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
|
||||
httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
|
||||
return true
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Write(data)
|
||||
return true
|
||||
case "/-/reload":
|
||||
@@ -67,7 +67,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
|
||||
httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
|
||||
return true
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Write(data)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ func main() {
|
||||
cgroup.UpdateGOMAXPROCSToCPUQuota()
|
||||
|
||||
if len(*snapshotCreateURL) > 0 {
|
||||
logger.Infof("%s", "Snapshots enabled")
|
||||
logger.Infof("Snapshots enabled")
|
||||
logger.Infof("Snapshot create url %s", *snapshotCreateURL)
|
||||
if len(*snapshotDeleteURL) <= 0 {
|
||||
err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1))
|
||||
@@ -55,17 +55,17 @@ func main() {
|
||||
|
||||
name, err := snapshot.Create(*snapshotCreateURL)
|
||||
if err != nil {
|
||||
logger.Fatalf("%s", err)
|
||||
logger.Fatalf("cannot create snapshot: %s", err)
|
||||
}
|
||||
err = flag.Set("snapshotName", name)
|
||||
if err != nil {
|
||||
logger.Fatalf("Failed to set snapshotName flag: %v", err)
|
||||
logger.Fatalf("cannot set snapshotName flag: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := snapshot.Delete(*snapshotDeleteURL, name)
|
||||
if err != nil {
|
||||
logger.Fatalf("%s", err)
|
||||
logger.Fatalf("cannot delete snapshot: %s", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ type snapshot struct {
|
||||
// Create creates a snapshot and the provided api endpoint and returns
|
||||
// the snapshot name
|
||||
func Create(createSnapshotURL string) (string, error) {
|
||||
logger.Infof("%s", "Creating snapshot")
|
||||
logger.Infof("Creating snapshot")
|
||||
u, err := url.Parse(createSnapshotURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -40,7 +40,7 @@ var (
|
||||
"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
|
||||
"Usually :4242 must be set. Doesn't work if empty")
|
||||
opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
|
||||
maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superflouos labels are dropped")
|
||||
maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped")
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -155,13 +155,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
return true
|
||||
case "/targets":
|
||||
promscrapeTargetsRequests.Inc()
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
|
||||
promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
|
||||
return true
|
||||
case "/api/v1/targets":
|
||||
promscrapeAPIV1TargetsRequests.Inc()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
state := r.FormValue("state")
|
||||
promscrape.WriteAPIV1Targets(w, state)
|
||||
return true
|
||||
@@ -175,7 +175,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
errMsg := fmt.Sprintf("waiting for scrape config to init targets, configs left: %d", rdy)
|
||||
http.Error(w, errMsg, http.StatusTooEarly)
|
||||
} else {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("OK"))
|
||||
}
|
||||
|
||||
@@ -84,10 +84,7 @@ func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
paths = deduplicatePaths(paths, delimiter)
|
||||
sortPaths(paths, delimiter)
|
||||
contentType := "application/json"
|
||||
if jsonp != "" {
|
||||
contentType = "text/javascript"
|
||||
}
|
||||
contentType := getContentType(jsonp)
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
@@ -166,10 +163,7 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
m[query] = paths
|
||||
}
|
||||
contentType := "application/json"
|
||||
if jsonp != "" {
|
||||
contentType = "text/javascript"
|
||||
}
|
||||
contentType := getContentType(jsonp)
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
if groupByExpr {
|
||||
for _, paths := range m {
|
||||
@@ -215,10 +209,7 @@ func MetricsIndexHandler(startTime time.Time, w http.ResponseWriter, r *http.Req
|
||||
if err != nil {
|
||||
return fmt.Errorf(`cannot obtain metric names: %w`, err)
|
||||
}
|
||||
contentType := "application/json"
|
||||
if jsonp != "" {
|
||||
contentType = "text/javascript"
|
||||
}
|
||||
contentType := getContentType(jsonp)
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
@@ -417,3 +408,10 @@ var regexpCache = make(map[regexpCacheKey]*regexpCacheEntry)
|
||||
var regexpCacheLock sync.Mutex
|
||||
|
||||
const maxRegexpCacheSize = 10000
|
||||
|
||||
func getContentType(jsonp string) string {
|
||||
if jsonp == "" {
|
||||
return "application/json; charset=utf-8"
|
||||
}
|
||||
return "text/javascript; charset=utf-8"
|
||||
}
|
||||
20
app/vmselect/graphite/tag_values_response.qtpl
Normal file
20
app/vmselect/graphite/tag_values_response.qtpl
Normal file
@@ -0,0 +1,20 @@
|
||||
{% stripspace %}
|
||||
|
||||
Tags generates response for /tags/<tag_name> handler
|
||||
See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
|
||||
{% func TagValuesResponse(tagName string, tagValues []string) %}
|
||||
{
|
||||
"tag":{%q= tagName %},
|
||||
"values":[
|
||||
{% for i, value := range tagValues %}
|
||||
{
|
||||
"count":1,
|
||||
"value":{%q= value %}
|
||||
}
|
||||
{% if i+1 < len(tagValues) %},{% endif %}
|
||||
{% endfor %}
|
||||
]
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% endstripspace %}
|
||||
75
app/vmselect/graphite/tag_values_response.qtpl.go
Normal file
75
app/vmselect/graphite/tag_values_response.qtpl.go
Normal file
@@ -0,0 +1,75 @@
|
||||
// Code generated by qtc from "tag_values_response.qtpl". DO NOT EDIT.
|
||||
// See https://github.com/valyala/quicktemplate for details.
|
||||
|
||||
// Tags generates response for /tags/<tag_name> handlerSee https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
|
||||
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:5
|
||||
package graphite
|
||||
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:5
|
||||
import (
|
||||
qtio422016 "io"
|
||||
|
||||
qt422016 "github.com/valyala/quicktemplate"
|
||||
)
|
||||
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:5
|
||||
var (
|
||||
_ = qtio422016.Copy
|
||||
_ = qt422016.AcquireByteBuffer
|
||||
)
|
||||
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:5
|
||||
func StreamTagValuesResponse(qw422016 *qt422016.Writer, tagName string, tagValues []string) {
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:5
|
||||
qw422016.N().S(`{"tag":`)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:7
|
||||
qw422016.N().Q(tagName)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:7
|
||||
qw422016.N().S(`,"values":[`)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:9
|
||||
for i, value := range tagValues {
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:9
|
||||
qw422016.N().S(`{"count":1,"value":`)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:12
|
||||
qw422016.N().Q(value)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:12
|
||||
qw422016.N().S(`}`)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:14
|
||||
if i+1 < len(tagValues) {
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:14
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:14
|
||||
}
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:15
|
||||
}
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:15
|
||||
qw422016.N().S(`]}`)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
}
|
||||
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
func WriteTagValuesResponse(qq422016 qtio422016.Writer, tagName string, tagValues []string) {
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
StreamTagValuesResponse(qw422016, tagName, tagValues)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
}
|
||||
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
func TagValuesResponse(tagName string, tagValues []string) string {
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
WriteTagValuesResponse(qb422016, tagName, tagValues)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
return qs422016
|
||||
//line app/vmselect/graphite/tag_values_response.qtpl:18
|
||||
}
|
||||
504
app/vmselect/graphite/tags_api.go
Normal file
504
app/vmselect/graphite/tags_api.go
Normal file
@@ -0,0 +1,504 @@
|
||||
package graphite
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
graphiteparser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
// TagsDelSeriesHandler implements /tags/delSeries handler.
|
||||
//
|
||||
// See https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb
|
||||
func TagsDelSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
return fmt.Errorf("cannot parse form values: %w", err)
|
||||
}
|
||||
paths := r.Form["path"]
|
||||
totalDeleted := 0
|
||||
var row graphiteparser.Row
|
||||
var tagsPool []graphiteparser.Tag
|
||||
ct := time.Now().UnixNano() / 1e6
|
||||
for _, path := range paths {
|
||||
var err error
|
||||
tagsPool, err = row.UnmarshalMetricAndTags(path, tagsPool[:0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse path=%q: %w", path, err)
|
||||
}
|
||||
tfs := make([]storage.TagFilter, 0, 1+len(row.Tags))
|
||||
tfs = append(tfs, storage.TagFilter{
|
||||
Key: nil,
|
||||
Value: []byte(row.Metric),
|
||||
})
|
||||
for _, tag := range row.Tags {
|
||||
tfs = append(tfs, storage.TagFilter{
|
||||
Key: []byte(tag.Key),
|
||||
Value: []byte(tag.Value),
|
||||
})
|
||||
}
|
||||
sq := storage.NewSearchQuery(0, ct, [][]storage.TagFilter{tfs})
|
||||
n, err := netstorage.DeleteSeries(sq)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot delete series for %q: %w", sq, err)
|
||||
}
|
||||
totalDeleted += n
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
if totalDeleted > 0 {
|
||||
fmt.Fprintf(w, "true")
|
||||
} else {
|
||||
fmt.Fprintf(w, "false")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TagsTagSeriesHandler implements /tags/tagSeries handler.
|
||||
//
|
||||
// See https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb
|
||||
func TagsTagSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
return registerMetrics(startTime, w, r, false)
|
||||
}
|
||||
|
||||
// TagsTagMultiSeriesHandler implements /tags/tagMultiSeries handler.
|
||||
//
|
||||
// See https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb
|
||||
func TagsTagMultiSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
return registerMetrics(startTime, w, r, true)
|
||||
}
|
||||
|
||||
func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request, isJSONResponse bool) error {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
return fmt.Errorf("cannot parse form values: %w", err)
|
||||
}
|
||||
paths := r.Form["path"]
|
||||
var row graphiteparser.Row
|
||||
var labels []prompb.Label
|
||||
var b []byte
|
||||
var tagsPool []graphiteparser.Tag
|
||||
mrs := make([]storage.MetricRow, len(paths))
|
||||
ct := time.Now().UnixNano() / 1e6
|
||||
canonicalPaths := make([]string, len(paths))
|
||||
for i, path := range paths {
|
||||
var err error
|
||||
tagsPool, err = row.UnmarshalMetricAndTags(path, tagsPool[:0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse path=%q: %w", path, err)
|
||||
}
|
||||
|
||||
// Construct canonical path according to https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb
|
||||
sort.Slice(row.Tags, func(i, j int) bool {
|
||||
return row.Tags[i].Key < row.Tags[j].Key
|
||||
})
|
||||
b = append(b[:0], row.Metric...)
|
||||
for _, tag := range row.Tags {
|
||||
b = append(b, ';')
|
||||
b = append(b, tag.Key...)
|
||||
b = append(b, '=')
|
||||
b = append(b, tag.Value...)
|
||||
}
|
||||
canonicalPaths[i] = string(b)
|
||||
|
||||
// Convert parsed metric and tags to labels.
|
||||
labels = append(labels[:0], prompb.Label{
|
||||
Name: []byte("__name__"),
|
||||
Value: []byte(row.Metric),
|
||||
})
|
||||
for _, tag := range row.Tags {
|
||||
labels = append(labels, prompb.Label{
|
||||
Name: []byte(tag.Key),
|
||||
Value: []byte(tag.Value),
|
||||
})
|
||||
}
|
||||
|
||||
// Put labels with the current timestamp to MetricRow
|
||||
mr := &mrs[i]
|
||||
mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
|
||||
mr.Timestamp = ct
|
||||
}
|
||||
if err := vmstorage.RegisterMetricNames(mrs); err != nil {
|
||||
return fmt.Errorf("cannot register paths: %w", err)
|
||||
}
|
||||
|
||||
// Return response
|
||||
contentType := "text/plain; charset=utf-8"
|
||||
if isJSONResponse {
|
||||
contentType = "application/json; charset=utf-8"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
WriteTagsTagMultiSeriesResponse(w, canonicalPaths, isJSONResponse)
|
||||
if isJSONResponse {
|
||||
tagsTagMultiSeriesDuration.UpdateDuration(startTime)
|
||||
} else {
|
||||
tagsTagSeriesDuration.UpdateDuration(startTime)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
tagsTagSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/tagSeries"}`)
|
||||
tagsTagMultiSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/tagMultiSeries"}`)
|
||||
)
|
||||
|
||||
// TagsAutoCompleteValuesHandler implements /tags/autoComplete/values endpoint from Graphite Tags API.
|
||||
//
|
||||
// See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
|
||||
func TagsAutoCompleteValuesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
deadline := searchutils.GetDeadlineForQuery(r, startTime)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
return fmt.Errorf("cannot parse form values: %w", err)
|
||||
}
|
||||
limit, err := getInt(r, "limit")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if limit <= 0 {
|
||||
// Use limit=100 by default. See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
|
||||
limit = 100
|
||||
}
|
||||
tag := r.FormValue("tag")
|
||||
if len(tag) == 0 {
|
||||
return fmt.Errorf("missing `tag` query arg")
|
||||
}
|
||||
valuePrefix := r.FormValue("valuePrefix")
|
||||
exprs := r.Form["expr"]
|
||||
var tagValues []string
|
||||
if len(exprs) == 0 {
|
||||
// Fast path: there are no `expr` filters, so use netstorage.GetGraphiteTagValues.
|
||||
// Escape special chars in tagPrefix as Graphite does.
|
||||
// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L228
|
||||
filter := regexp.QuoteMeta(valuePrefix)
|
||||
tagValues, err = netstorage.GetGraphiteTagValues(tag, filter, limit, deadline)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// Slow path: use netstorage.SearchMetricNames for applying `expr` filters.
|
||||
sq, err := getSearchQueryForExprs(exprs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mns, err := netstorage.SearchMetricNames(sq, deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
|
||||
}
|
||||
m := make(map[string]struct{})
|
||||
if tag == "name" {
|
||||
tag = "__name__"
|
||||
}
|
||||
for _, mn := range mns {
|
||||
tagValue := mn.GetTagValue(tag)
|
||||
if len(tagValue) == 0 {
|
||||
continue
|
||||
}
|
||||
m[string(tagValue)] = struct{}{}
|
||||
}
|
||||
if len(valuePrefix) > 0 {
|
||||
for tagValue := range m {
|
||||
if !strings.HasPrefix(tagValue, valuePrefix) {
|
||||
delete(m, tagValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
tagValues = make([]string, 0, len(m))
|
||||
for tagValue := range m {
|
||||
tagValues = append(tagValues, tagValue)
|
||||
}
|
||||
sort.Strings(tagValues)
|
||||
if limit > 0 && limit < len(tagValues) {
|
||||
tagValues = tagValues[:limit]
|
||||
}
|
||||
}
|
||||
|
||||
jsonp := r.FormValue("jsonp")
|
||||
contentType := getContentType(jsonp)
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteTagsAutoCompleteResponse(bw, tagValues, jsonp)
|
||||
if err := bw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
tagsAutoCompleteValuesDuration.UpdateDuration(startTime)
|
||||
return nil
|
||||
}
|
||||
|
||||
var tagsAutoCompleteValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/autoComplete/values"}`)
|
||||
|
||||
// TagsAutoCompleteTagsHandler implements /tags/autoComplete/tags endpoint from Graphite Tags API.
|
||||
//
|
||||
// See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
|
||||
func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
deadline := searchutils.GetDeadlineForQuery(r, startTime)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
return fmt.Errorf("cannot parse form values: %w", err)
|
||||
}
|
||||
limit, err := getInt(r, "limit")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if limit <= 0 {
|
||||
// Use limit=100 by default. See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
|
||||
limit = 100
|
||||
}
|
||||
tagPrefix := r.FormValue("tagPrefix")
|
||||
exprs := r.Form["expr"]
|
||||
var labels []string
|
||||
if len(exprs) == 0 {
|
||||
// Fast path: there are no `expr` filters, so use netstorage.GetGraphiteTags.
|
||||
|
||||
// Escape special chars in tagPrefix as Graphite does.
|
||||
// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L181
|
||||
filter := regexp.QuoteMeta(tagPrefix)
|
||||
labels, err = netstorage.GetGraphiteTags(filter, limit, deadline)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// Slow path: use netstorage.SearchMetricNames for applying `expr` filters.
|
||||
sq, err := getSearchQueryForExprs(exprs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mns, err := netstorage.SearchMetricNames(sq, deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
|
||||
}
|
||||
m := make(map[string]struct{})
|
||||
for _, mn := range mns {
|
||||
m["name"] = struct{}{}
|
||||
for _, tag := range mn.Tags {
|
||||
m[string(tag.Key)] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(tagPrefix) > 0 {
|
||||
for label := range m {
|
||||
if !strings.HasPrefix(label, tagPrefix) {
|
||||
delete(m, label)
|
||||
}
|
||||
}
|
||||
}
|
||||
labels = make([]string, 0, len(m))
|
||||
for label := range m {
|
||||
labels = append(labels, label)
|
||||
}
|
||||
sort.Strings(labels)
|
||||
if limit > 0 && limit < len(labels) {
|
||||
labels = labels[:limit]
|
||||
}
|
||||
}
|
||||
|
||||
jsonp := r.FormValue("jsonp")
|
||||
contentType := getContentType(jsonp)
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteTagsAutoCompleteResponse(bw, labels, jsonp)
|
||||
if err := bw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
tagsAutoCompleteTagsDuration.UpdateDuration(startTime)
|
||||
return nil
|
||||
}
|
||||
|
||||
var tagsAutoCompleteTagsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/autoComplete/tags"}`)
|
||||
|
||||
// TagsFindSeriesHandler implements /tags/findSeries endpoint from Graphite Tags API.
|
||||
//
|
||||
// See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
|
||||
func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
deadline := searchutils.GetDeadlineForQuery(r, startTime)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
return fmt.Errorf("cannot parse form values: %w", err)
|
||||
}
|
||||
limit, err := getInt(r, "limit")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
exprs := r.Form["expr"]
|
||||
if len(exprs) == 0 {
|
||||
return fmt.Errorf("expecting at least one `expr` query arg")
|
||||
}
|
||||
sq, err := getSearchQueryForExprs(exprs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mns, err := netstorage.SearchMetricNames(sq, deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
|
||||
}
|
||||
paths := getCanonicalPaths(mns)
|
||||
if limit > 0 && limit < len(paths) {
|
||||
paths = paths[:limit]
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteTagsFindSeriesResponse(bw, paths)
|
||||
if err := bw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
tagsFindSeriesDuration.UpdateDuration(startTime)
|
||||
return nil
|
||||
}
|
||||
|
||||
func getCanonicalPaths(mns []storage.MetricName) []string {
|
||||
paths := make([]string, 0, len(mns))
|
||||
var b []byte
|
||||
var tags []storage.Tag
|
||||
for _, mn := range mns {
|
||||
b = append(b[:0], mn.MetricGroup...)
|
||||
tags = append(tags[:0], mn.Tags...)
|
||||
sort.Slice(tags, func(i, j int) bool {
|
||||
return string(tags[i].Key) < string(tags[j].Key)
|
||||
})
|
||||
for _, tag := range tags {
|
||||
b = append(b, ';')
|
||||
b = append(b, tag.Key...)
|
||||
b = append(b, '=')
|
||||
b = append(b, tag.Value...)
|
||||
}
|
||||
paths = append(paths, string(b))
|
||||
}
|
||||
sort.Strings(paths)
|
||||
return paths
|
||||
}
|
||||
|
||||
var tagsFindSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/findSeries"}`)
|
||||
|
||||
// TagValuesHandler implements /tags/<tag_name> endpoint from Graphite Tags API.
|
||||
//
|
||||
// See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
|
||||
func TagValuesHandler(startTime time.Time, tagName string, w http.ResponseWriter, r *http.Request) error {
|
||||
deadline := searchutils.GetDeadlineForQuery(r, startTime)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
return fmt.Errorf("cannot parse form values: %w", err)
|
||||
}
|
||||
limit, err := getInt(r, "limit")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filter := r.FormValue("filter")
|
||||
tagValues, err := netstorage.GetGraphiteTagValues(tagName, filter, limit, deadline)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteTagValuesResponse(bw, tagName, tagValues)
|
||||
if err := bw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
tagValuesDuration.UpdateDuration(startTime)
|
||||
return nil
|
||||
}
|
||||
|
||||
var tagValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/<tag_name>"}`)
|
||||
|
||||
// TagsHandler implements /tags endpoint from Graphite Tags API.
|
||||
//
|
||||
// See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
|
||||
func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
deadline := searchutils.GetDeadlineForQuery(r, startTime)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
return fmt.Errorf("cannot parse form values: %w", err)
|
||||
}
|
||||
limit, err := getInt(r, "limit")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filter := r.FormValue("filter")
|
||||
labels, err := netstorage.GetGraphiteTags(filter, limit, deadline)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteTagsResponse(bw, labels)
|
||||
if err := bw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
tagsDuration.UpdateDuration(startTime)
|
||||
return nil
|
||||
}
|
||||
|
||||
var tagsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags"}`)
|
||||
|
||||
func getInt(r *http.Request, argName string) (int, error) {
|
||||
argValue := r.FormValue(argName)
|
||||
if len(argValue) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
n, err := strconv.Atoi(argValue)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse %q=%q: %w", argName, argValue, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func getSearchQueryForExprs(exprs []string) (*storage.SearchQuery, error) {
|
||||
tfs, err := exprsToTagFilters(exprs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ct := time.Now().UnixNano() / 1e6
|
||||
sq := storage.NewSearchQuery(0, ct, [][]storage.TagFilter{tfs})
|
||||
return sq, nil
|
||||
}
|
||||
|
||||
func exprsToTagFilters(exprs []string) ([]storage.TagFilter, error) {
|
||||
tfs := make([]storage.TagFilter, 0, len(exprs))
|
||||
for _, expr := range exprs {
|
||||
tf, err := parseFilterExpr(expr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse `expr` query arg: %w", err)
|
||||
}
|
||||
tfs = append(tfs, *tf)
|
||||
}
|
||||
return tfs, nil
|
||||
}
|
||||
|
||||
func parseFilterExpr(s string) (*storage.TagFilter, error) {
|
||||
n := strings.Index(s, "=")
|
||||
if n < 0 {
|
||||
return nil, fmt.Errorf("missing tag value in filter expression %q", s)
|
||||
}
|
||||
tagName := s[:n]
|
||||
tagValue := s[n+1:]
|
||||
isNegative := false
|
||||
if strings.HasSuffix(tagName, "!") {
|
||||
isNegative = true
|
||||
tagName = tagName[:len(tagName)-1]
|
||||
}
|
||||
if tagName == "name" {
|
||||
tagName = ""
|
||||
}
|
||||
isRegexp := false
|
||||
if strings.HasPrefix(tagValue, "~") {
|
||||
isRegexp = true
|
||||
tagValue = "^(?:" + tagValue[1:] + ").*"
|
||||
}
|
||||
return &storage.TagFilter{
|
||||
Key: []byte(tagName),
|
||||
Value: []byte(tagValue),
|
||||
IsNegative: isNegative,
|
||||
IsRegexp: isRegexp,
|
||||
}, nil
|
||||
}
|
||||
16
app/vmselect/graphite/tags_autocomplete_response.qtpl
Normal file
16
app/vmselect/graphite/tags_autocomplete_response.qtpl
Normal file
@@ -0,0 +1,16 @@
|
||||
{% stripspace %}
|
||||
|
||||
TagsAutoCompleteResponse generates responses for /tags/autoComplete/{tags,values} handlers in Graphite Tags API.
|
||||
See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
|
||||
{% func TagsAutoCompleteResponse(ss []string, jsonp string) %}
|
||||
{% if jsonp != "" %}{%s= jsonp %}({% endif %}
|
||||
[
|
||||
{% for i, s := range ss %}
|
||||
{%q= s %}
|
||||
{% if i+1 < len(ss) %},{% endif %}
|
||||
{% endfor %}
|
||||
]
|
||||
{% if jsonp != "" %}){% endif %}
|
||||
{% endfunc %}
|
||||
|
||||
{% endstripspace %}
|
||||
81
app/vmselect/graphite/tags_autocomplete_response.qtpl.go
Normal file
81
app/vmselect/graphite/tags_autocomplete_response.qtpl.go
Normal file
@@ -0,0 +1,81 @@
|
||||
// Code generated by qtc from "tags_autocomplete_response.qtpl". DO NOT EDIT.
|
||||
// See https://github.com/valyala/quicktemplate for details.
|
||||
|
||||
// TagsAutoCompleteResponse generates responses for /tags/autoComplete/{tags,values} handlers in Graphite Tags API.See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
|
||||
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5
|
||||
package graphite
|
||||
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5
|
||||
import (
|
||||
qtio422016 "io"
|
||||
|
||||
qt422016 "github.com/valyala/quicktemplate"
|
||||
)
|
||||
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5
|
||||
var (
|
||||
_ = qtio422016.Copy
|
||||
_ = qt422016.AcquireByteBuffer
|
||||
)
|
||||
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5
|
||||
func StreamTagsAutoCompleteResponse(qw422016 *qt422016.Writer, ss []string, jsonp string) {
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6
|
||||
if jsonp != "" {
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6
|
||||
qw422016.N().S(jsonp)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6
|
||||
qw422016.N().S(`(`)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6
|
||||
}
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6
|
||||
qw422016.N().S(`[`)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:8
|
||||
for i, s := range ss {
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:9
|
||||
qw422016.N().Q(s)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:10
|
||||
if i+1 < len(ss) {
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:10
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:10
|
||||
}
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:11
|
||||
}
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:11
|
||||
qw422016.N().S(`]`)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:13
|
||||
if jsonp != "" {
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:13
|
||||
qw422016.N().S(`)`)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:13
|
||||
}
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
}
|
||||
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
func WriteTagsAutoCompleteResponse(qq422016 qtio422016.Writer, ss []string, jsonp string) {
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
StreamTagsAutoCompleteResponse(qw422016, ss, jsonp)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
}
|
||||
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
func TagsAutoCompleteResponse(ss []string, jsonp string) string {
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
WriteTagsAutoCompleteResponse(qb422016, ss, jsonp)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
return qs422016
|
||||
//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
|
||||
}
|
||||
12
app/vmselect/graphite/tags_find_series_response.qtpl
Normal file
12
app/vmselect/graphite/tags_find_series_response.qtpl
Normal file
@@ -0,0 +1,12 @@
|
||||
{% stripspace %}
|
||||
|
||||
{% func TagsFindSeriesResponse(paths []string) %}
|
||||
[
|
||||
{% for i, path := range paths %}
|
||||
{%q= path %}
|
||||
{% if i+1 < len(paths) %},{% endif %}
|
||||
{% endfor %}
|
||||
]
|
||||
{% endfunc %}
|
||||
|
||||
{% endstripspace %}
|
||||
65
app/vmselect/graphite/tags_find_series_response.qtpl.go
Normal file
65
app/vmselect/graphite/tags_find_series_response.qtpl.go
Normal file
@@ -0,0 +1,65 @@
|
||||
// Code generated by qtc from "tags_find_series_response.qtpl". DO NOT EDIT.
|
||||
// See https://github.com/valyala/quicktemplate for details.
|
||||
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:3
|
||||
package graphite
|
||||
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:3
|
||||
import (
|
||||
qtio422016 "io"
|
||||
|
||||
qt422016 "github.com/valyala/quicktemplate"
|
||||
)
|
||||
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:3
|
||||
var (
|
||||
_ = qtio422016.Copy
|
||||
_ = qt422016.AcquireByteBuffer
|
||||
)
|
||||
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:3
|
||||
func StreamTagsFindSeriesResponse(qw422016 *qt422016.Writer, paths []string) {
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:3
|
||||
qw422016.N().S(`[`)
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:5
|
||||
for i, path := range paths {
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:6
|
||||
qw422016.N().Q(path)
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:7
|
||||
if i+1 < len(paths) {
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:7
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:7
|
||||
}
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:8
|
||||
}
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:8
|
||||
qw422016.N().S(`]`)
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
}
|
||||
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
func WriteTagsFindSeriesResponse(qq422016 qtio422016.Writer, paths []string) {
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
StreamTagsFindSeriesResponse(qw422016, paths)
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
}
|
||||
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
func TagsFindSeriesResponse(paths []string) string {
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
WriteTagsFindSeriesResponse(qb422016, paths)
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
return qs422016
|
||||
//line app/vmselect/graphite/tags_find_series_response.qtpl:10
|
||||
}
|
||||
16
app/vmselect/graphite/tags_response.qtpl
Normal file
16
app/vmselect/graphite/tags_response.qtpl
Normal file
@@ -0,0 +1,16 @@
|
||||
{% stripspace %}
|
||||
|
||||
Tags generates response for /tags handler
|
||||
See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
|
||||
{% func TagsResponse(tags []string) %}
|
||||
[
|
||||
{% for i, tag := range tags %}
|
||||
{
|
||||
"tag":{%q= tag %}
|
||||
}
|
||||
{% if i+1 < len(tags) %},{% endif %}
|
||||
{% endfor %}
|
||||
]
|
||||
{% endfunc %}
|
||||
|
||||
{% endstripspace %}
|
||||
71
app/vmselect/graphite/tags_response.qtpl.go
Normal file
71
app/vmselect/graphite/tags_response.qtpl.go
Normal file
@@ -0,0 +1,71 @@
|
||||
// Code generated by qtc from "tags_response.qtpl". DO NOT EDIT.
|
||||
// See https://github.com/valyala/quicktemplate for details.
|
||||
|
||||
// Tags generates response for /tags handlerSee https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
|
||||
|
||||
//line app/vmselect/graphite/tags_response.qtpl:5
|
||||
package graphite
|
||||
|
||||
//line app/vmselect/graphite/tags_response.qtpl:5
|
||||
import (
|
||||
qtio422016 "io"
|
||||
|
||||
qt422016 "github.com/valyala/quicktemplate"
|
||||
)
|
||||
|
||||
//line app/vmselect/graphite/tags_response.qtpl:5
|
||||
var (
|
||||
_ = qtio422016.Copy
|
||||
_ = qt422016.AcquireByteBuffer
|
||||
)
|
||||
|
||||
//line app/vmselect/graphite/tags_response.qtpl:5
|
||||
func StreamTagsResponse(qw422016 *qt422016.Writer, tags []string) {
|
||||
//line app/vmselect/graphite/tags_response.qtpl:5
|
||||
qw422016.N().S(`[`)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:7
|
||||
for i, tag := range tags {
|
||||
//line app/vmselect/graphite/tags_response.qtpl:7
|
||||
qw422016.N().S(`{"tag":`)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:9
|
||||
qw422016.N().Q(tag)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:9
|
||||
qw422016.N().S(`}`)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:11
|
||||
if i+1 < len(tags) {
|
||||
//line app/vmselect/graphite/tags_response.qtpl:11
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:11
|
||||
}
|
||||
//line app/vmselect/graphite/tags_response.qtpl:12
|
||||
}
|
||||
//line app/vmselect/graphite/tags_response.qtpl:12
|
||||
qw422016.N().S(`]`)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
}
|
||||
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
func WriteTagsResponse(qq422016 qtio422016.Writer, tags []string) {
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
StreamTagsResponse(qw422016, tags)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
}
|
||||
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
func TagsResponse(tags []string) string {
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
WriteTagsResponse(qb422016, tags)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
return qs422016
|
||||
//line app/vmselect/graphite/tags_response.qtpl:14
|
||||
}
|
||||
14
app/vmselect/graphite/tags_tag_multi_series_response.qtpl
Normal file
14
app/vmselect/graphite/tags_tag_multi_series_response.qtpl
Normal file
@@ -0,0 +1,14 @@
|
||||
{% stripspace %}
|
||||
|
||||
TagsTagMultiSeriesResponse generates response for /tags/tagMultiSeries .
|
||||
See https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb
|
||||
{% func TagsTagMultiSeriesResponse(canonicalPaths []string, isJSONResponse bool) %}
|
||||
{% if isJSONResponse %}[{% endif %}
|
||||
{% for i, path := range canonicalPaths %}
|
||||
{%q= path %}
|
||||
{% if i+1 < len(canonicalPaths) %},{% endif %}
|
||||
{% endfor %}
|
||||
{% if isJSONResponse %}]{% endif %}
|
||||
{% endfunc %}
|
||||
|
||||
{% endstripspace %}
|
||||
75
app/vmselect/graphite/tags_tag_multi_series_response.qtpl.go
Normal file
75
app/vmselect/graphite/tags_tag_multi_series_response.qtpl.go
Normal file
@@ -0,0 +1,75 @@
|
||||
// Code generated by qtc from "tags_tag_multi_series_response.qtpl". DO NOT EDIT.
|
||||
// See https://github.com/valyala/quicktemplate for details.
|
||||
|
||||
// TagsTagMultiSeriesResponse generates response for /tags/tagMultiSeries .See https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb
|
||||
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:5
|
||||
package graphite
|
||||
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:5
|
||||
import (
|
||||
qtio422016 "io"
|
||||
|
||||
qt422016 "github.com/valyala/quicktemplate"
|
||||
)
|
||||
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:5
|
||||
var (
|
||||
_ = qtio422016.Copy
|
||||
_ = qt422016.AcquireByteBuffer
|
||||
)
|
||||
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:5
|
||||
func StreamTagsTagMultiSeriesResponse(qw422016 *qt422016.Writer, canonicalPaths []string, isJSONResponse bool) {
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:6
|
||||
if isJSONResponse {
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:6
|
||||
qw422016.N().S(`[`)
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:6
|
||||
}
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:7
|
||||
for i, path := range canonicalPaths {
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:8
|
||||
qw422016.N().Q(path)
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:9
|
||||
if i+1 < len(canonicalPaths) {
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:9
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:9
|
||||
}
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:10
|
||||
}
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:11
|
||||
if isJSONResponse {
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:11
|
||||
qw422016.N().S(`]`)
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:11
|
||||
}
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
}
|
||||
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
func WriteTagsTagMultiSeriesResponse(qq422016 qtio422016.Writer, canonicalPaths []string, isJSONResponse bool) {
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
StreamTagsTagMultiSeriesResponse(qw422016, canonicalPaths, isJSONResponse)
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
}
|
||||
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
func TagsTagMultiSeriesResponse(canonicalPaths []string, isJSONResponse bool) string {
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
WriteTagsTagMultiSeriesResponse(qb422016, canonicalPaths, isJSONResponse)
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
return qs422016
|
||||
//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
|
||||
}
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
deleteAuthKey = flag.String("deleteAuthKey", "", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series")
|
||||
deleteAuthKey = flag.String("deleteAuthKey", "", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries")
|
||||
maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
|
||||
"It shouldn't be high, since a single request can saturate all the CPU cores. See also -search.maxQueueDuration")
|
||||
maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
|
||||
@@ -132,6 +132,16 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(path, "/tags/") && !isGraphiteTagsPath(path) {
|
||||
tagName := r.URL.Path[len("/tags/"):]
|
||||
graphiteTagValuesRequests.Inc()
|
||||
if err := graphite.TagValuesHandler(startTime, tagName, w, r); err != nil {
|
||||
graphiteTagValuesErrors.Inc()
|
||||
httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
|
||||
return true
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
switch path {
|
||||
case "/api/v1/query":
|
||||
@@ -259,22 +269,85 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
return true
|
||||
}
|
||||
return true
|
||||
case "/tags/tagSeries":
|
||||
graphiteTagsTagSeriesRequests.Inc()
|
||||
if err := graphite.TagsTagSeriesHandler(startTime, w, r); err != nil {
|
||||
graphiteTagsTagSeriesErrors.Inc()
|
||||
httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
|
||||
return true
|
||||
}
|
||||
return true
|
||||
case "/tags/tagMultiSeries":
|
||||
graphiteTagsTagMultiSeriesRequests.Inc()
|
||||
if err := graphite.TagsTagMultiSeriesHandler(startTime, w, r); err != nil {
|
||||
graphiteTagsTagMultiSeriesErrors.Inc()
|
||||
httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
|
||||
return true
|
||||
}
|
||||
return true
|
||||
case "/tags":
|
||||
graphiteTagsRequests.Inc()
|
||||
if err := graphite.TagsHandler(startTime, w, r); err != nil {
|
||||
graphiteTagsErrors.Inc()
|
||||
httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
|
||||
return true
|
||||
}
|
||||
return true
|
||||
case "/tags/findSeries":
|
||||
graphiteTagsFindSeriesRequests.Inc()
|
||||
if err := graphite.TagsFindSeriesHandler(startTime, w, r); err != nil {
|
||||
graphiteTagsFindSeriesErrors.Inc()
|
||||
httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
|
||||
return true
|
||||
}
|
||||
return true
|
||||
case "/tags/autoComplete/tags":
|
||||
graphiteTagsAutoCompleteTagsRequests.Inc()
|
||||
httpserver.EnableCORS(w, r)
|
||||
if err := graphite.TagsAutoCompleteTagsHandler(startTime, w, r); err != nil {
|
||||
graphiteTagsAutoCompleteTagsErrors.Inc()
|
||||
httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
|
||||
return true
|
||||
}
|
||||
return true
|
||||
case "/tags/autoComplete/values":
|
||||
graphiteTagsAutoCompleteValuesRequests.Inc()
|
||||
httpserver.EnableCORS(w, r)
|
||||
if err := graphite.TagsAutoCompleteValuesHandler(startTime, w, r); err != nil {
|
||||
graphiteTagsAutoCompleteValuesErrors.Inc()
|
||||
httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
|
||||
return true
|
||||
}
|
||||
return true
|
||||
case "/tags/delSeries":
|
||||
graphiteTagsDelSeriesRequests.Inc()
|
||||
authKey := r.FormValue("authKey")
|
||||
if authKey != *deleteAuthKey {
|
||||
httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -deleteAuthKey command line flag", authKey)
|
||||
return true
|
||||
}
|
||||
if err := graphite.TagsDelSeriesHandler(startTime, w, r); err != nil {
|
||||
graphiteTagsDelSeriesErrors.Inc()
|
||||
httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
|
||||
return true
|
||||
}
|
||||
return true
|
||||
case "/api/v1/rules":
|
||||
// Return dumb placeholder
|
||||
rulesRequests.Inc()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
fmt.Fprintf(w, "%s", `{"status":"success","data":{"groups":[]}}`)
|
||||
return true
|
||||
case "/api/v1/alerts":
|
||||
// Return dumb placehloder
|
||||
alertsRequests.Inc()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
fmt.Fprintf(w, "%s", `{"status":"success","data":{"alerts":[]}}`)
|
||||
return true
|
||||
case "/api/v1/metadata":
|
||||
// Return dumb placeholder
|
||||
metadataRequests.Inc()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
fmt.Fprintf(w, "%s", `{"status":"success","data":{}}`)
|
||||
return true
|
||||
case "/api/v1/admin/tsdb/delete_series":
|
||||
@@ -296,10 +369,22 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
}
|
||||
}
|
||||
|
||||
func isGraphiteTagsPath(path string) bool {
|
||||
switch path {
|
||||
// See https://graphite.readthedocs.io/en/stable/tags.html for a list of Graphite Tags API paths.
|
||||
// Do not include `/tags/<tag_name>` here, since this will fool the caller.
|
||||
case "/tags/tagSeries", "/tags/tagMultiSeries", "/tags/findSeries",
|
||||
"/tags/autoComplete/tags", "/tags/autoComplete/values", "/tags/delSeries":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
logger.Warnf("error in %q: %s", r.RequestURI, err)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
statusCode := http.StatusUnprocessableEntity
|
||||
var esc *httpserver.ErrorWithStatusCode
|
||||
if errors.As(err, &esc) {
|
||||
@@ -360,6 +445,30 @@ var (
|
||||
graphiteMetricsIndexRequests = metrics.NewCounter(`vm_http_requests_total{path="/metrics/index.json"}`)
|
||||
graphiteMetricsIndexErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/metrics/index.json"}`)
|
||||
|
||||
graphiteTagsTagSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/tagSeries"}`)
|
||||
graphiteTagsTagSeriesErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/tagSeries"}`)
|
||||
|
||||
graphiteTagsTagMultiSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/tagMultiSeries"}`)
|
||||
graphiteTagsTagMultiSeriesErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/tagMultiSeries"}`)
|
||||
|
||||
graphiteTagsRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags"}`)
|
||||
graphiteTagsErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags"}`)
|
||||
|
||||
graphiteTagValuesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/<tag_name>"}`)
|
||||
graphiteTagValuesErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/<tag_name>"}`)
|
||||
|
||||
graphiteTagsFindSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/findSeries"}`)
|
||||
graphiteTagsFindSeriesErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/findSeries"}`)
|
||||
|
||||
graphiteTagsAutoCompleteTagsRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/autoComplete/tags"}`)
|
||||
graphiteTagsAutoCompleteTagsErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/autoComplete/tags"}`)
|
||||
|
||||
graphiteTagsAutoCompleteValuesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/autoComplete/values"}`)
|
||||
graphiteTagsAutoCompleteValuesErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/autoComplete/values"}`)
|
||||
|
||||
graphiteTagsDelSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/delSeries"}`)
|
||||
graphiteTagsDelSeriesErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/delSeries"}`)
|
||||
|
||||
rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
|
||||
alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
|
||||
metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"sync"
|
||||
@@ -473,6 +474,35 @@ func GetLabelsOnTimeRange(tr storage.TimeRange, deadline searchutils.Deadline) (
|
||||
return labels, nil
|
||||
}
|
||||
|
||||
// GetGraphiteTags returns Graphite tags until the given deadline.
|
||||
func GetGraphiteTags(filter string, limit int, deadline searchutils.Deadline) ([]string, error) {
|
||||
if deadline.Exceeded() {
|
||||
return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
|
||||
}
|
||||
labels, err := GetLabels(deadline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Substitute "__name__" with "name" for Graphite compatibility
|
||||
for i := range labels {
|
||||
if labels[i] == "__name__" {
|
||||
labels[i] = "name"
|
||||
sort.Strings(labels)
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(filter) > 0 {
|
||||
labels, err = applyGraphiteRegexpFilter(filter, labels)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if limit > 0 && limit < len(labels) {
|
||||
labels = labels[:limit]
|
||||
}
|
||||
return labels, nil
|
||||
}
|
||||
|
||||
// GetLabels returns labels until the given deadline.
|
||||
func GetLabels(deadline searchutils.Deadline) ([]string, error) {
|
||||
if deadline.Exceeded() {
|
||||
@@ -512,6 +542,30 @@ func GetLabelValuesOnTimeRange(labelName string, tr storage.TimeRange, deadline
|
||||
return labelValues, nil
|
||||
}
|
||||
|
||||
// GetGraphiteTagValues returns tag values for the given tagName until the given deadline.
|
||||
func GetGraphiteTagValues(tagName, filter string, limit int, deadline searchutils.Deadline) ([]string, error) {
|
||||
if deadline.Exceeded() {
|
||||
return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
|
||||
}
|
||||
if tagName == "name" {
|
||||
tagName = ""
|
||||
}
|
||||
tagValues, err := GetLabelValues(tagName, deadline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(filter) > 0 {
|
||||
tagValues, err = applyGraphiteRegexpFilter(filter, tagValues)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if limit > 0 && limit < len(tagValues) {
|
||||
tagValues = tagValues[:limit]
|
||||
}
|
||||
return tagValues, nil
|
||||
}
|
||||
|
||||
// GetLabelValues returns label values for the given labelName
|
||||
// until the given deadline.
|
||||
func GetLabelValues(labelName string, deadline searchutils.Deadline) ([]string, error) {
|
||||
@@ -724,6 +778,32 @@ var exportWorkPool = &sync.Pool{
|
||||
},
|
||||
}
|
||||
|
||||
// SearchMetricNames returns all the metric names matching sq until the given deadline.
|
||||
func SearchMetricNames(sq *storage.SearchQuery, deadline searchutils.Deadline) ([]storage.MetricName, error) {
|
||||
if deadline.Exceeded() {
|
||||
return nil, fmt.Errorf("timeout exceeded before starting to search metric names: %s", deadline.String())
|
||||
}
|
||||
|
||||
// Setup search.
|
||||
tfss, err := setupTfss(sq.TagFilterss)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tr := storage.TimeRange{
|
||||
MinTimestamp: sq.MinTimestamp,
|
||||
MaxTimestamp: sq.MaxTimestamp,
|
||||
}
|
||||
if err := vmstorage.CheckTimeRange(tr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mns, err := vmstorage.SearchMetricNames(tfss, tr, *maxMetricsPerSearch, deadline.Deadline())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot find metric names: %w", err)
|
||||
}
|
||||
return mns, nil
|
||||
}
|
||||
|
||||
// ProcessSearchQuery performs sq until the given deadline.
|
||||
//
|
||||
// Results.RunParallel or Results.Cancel must be called on the returned Results.
|
||||
@@ -836,3 +916,20 @@ func setupTfss(tagFilterss [][]storage.TagFilter) ([]*storage.TagFilters, error)
|
||||
}
|
||||
return tfss, nil
|
||||
}
|
||||
|
||||
func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
|
||||
// Anchor filter regexp to the beginning of the string as Graphite does.
|
||||
// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157
|
||||
filter = "^(?:" + filter + ")"
|
||||
re, err := regexp.Compile(filter)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse regexp filter=%q: %w", filter, err)
|
||||
}
|
||||
dst := ss[:0]
|
||||
for _, s := range ss {
|
||||
if re.MatchString(s) {
|
||||
dst = append(dst, s)
|
||||
}
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
@@ -133,10 +133,7 @@ func (tbf *tmpBlocksFile) Finalize() error {
|
||||
return fmt.Errorf("cannot write the remaining %d bytes to %q: %w", len(tbf.buf), fname, err)
|
||||
}
|
||||
tbf.buf = tbf.buf[:0]
|
||||
r, err := fs.OpenReaderAt(fname)
|
||||
if err != nil {
|
||||
logger.Panicf("FATAL: cannot open %q: %s", fname, err)
|
||||
}
|
||||
r := fs.MustOpenReaderAt(fname)
|
||||
// Hint the OS that the file is read almost sequentiallly.
|
||||
// This should reduce the number of disk seeks, which is important
|
||||
// for HDDs.
|
||||
|
||||
@@ -78,17 +78,13 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sq := &storage.SearchQuery{
|
||||
MinTimestamp: start,
|
||||
MaxTimestamp: end,
|
||||
TagFilterss: tagFilterss,
|
||||
}
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss)
|
||||
rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
|
||||
@@ -146,12 +142,8 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sq := &storage.SearchQuery{
|
||||
MinTimestamp: start,
|
||||
MaxTimestamp: end,
|
||||
TagFilterss: tagFilterss,
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/csv")
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss)
|
||||
w.Header().Set("Content-Type", "text/csv; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
|
||||
@@ -227,11 +219,7 @@ func ExportNativeHandler(startTime time.Time, w http.ResponseWriter, r *http.Req
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sq := &storage.SearchQuery{
|
||||
MinTimestamp: start,
|
||||
MaxTimestamp: end,
|
||||
TagFilterss: tagFilterss,
|
||||
}
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss)
|
||||
w.Header().Set("Content-Type", "VictoriaMetrics/native")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
@@ -331,9 +319,9 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
|
||||
WriteExportJSONLine(bb, xb)
|
||||
resultsCh <- bb
|
||||
}
|
||||
contentType := "application/stream+json"
|
||||
contentType := "application/stream+json; charset=utf-8"
|
||||
if format == "prometheus" {
|
||||
contentType = "text/plain"
|
||||
contentType = "text/plain; charset=utf-8"
|
||||
writeLineFunc = func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
|
||||
bb := quicktemplate.AcquireByteBuffer()
|
||||
WriteExportPrometheusLine(bb, xb)
|
||||
@@ -381,11 +369,7 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sq := &storage.SearchQuery{
|
||||
MinTimestamp: start,
|
||||
MaxTimestamp: end,
|
||||
TagFilterss: tagFilterss,
|
||||
}
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss)
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
@@ -486,9 +470,7 @@ func DeleteHandler(startTime time.Time, r *http.Request) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sq := &storage.SearchQuery{
|
||||
TagFilterss: tagFilterss,
|
||||
}
|
||||
sq := storage.NewSearchQuery(0, 0, tagFilterss)
|
||||
deletedCount, err := netstorage.DeleteSeries(sq)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot delete time series matching %q: %w", matches, err)
|
||||
@@ -561,7 +543,7 @@ func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWr
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteLabelValuesResponse(bw, labelValues)
|
||||
@@ -596,32 +578,41 @@ func labelValuesWithMatches(labelName string, matches []string, start, end int64
|
||||
if start >= end {
|
||||
end = start + defaultStep
|
||||
}
|
||||
sq := &storage.SearchQuery{
|
||||
MinTimestamp: start,
|
||||
MaxTimestamp: end,
|
||||
TagFilterss: tagFilterss,
|
||||
}
|
||||
rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss)
|
||||
m := make(map[string]struct{})
|
||||
var mLock sync.Mutex
|
||||
err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
|
||||
labelValue := rs.MetricName.GetTagValue(labelName)
|
||||
if len(labelValue) == 0 {
|
||||
return nil
|
||||
if end-start > 24*3600*1000 {
|
||||
// It is cheaper to call SearchMetricNames on time ranges exceeding a day.
|
||||
mns, err := netstorage.SearchMetricNames(sq, deadline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
|
||||
}
|
||||
for _, mn := range mns {
|
||||
labelValue := mn.GetTagValue(labelName)
|
||||
if len(labelValue) == 0 {
|
||||
continue
|
||||
}
|
||||
m[string(labelValue)] = struct{}{}
|
||||
}
|
||||
} else {
|
||||
rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
var mLock sync.Mutex
|
||||
err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
|
||||
labelValue := rs.MetricName.GetTagValue(labelName)
|
||||
if len(labelValue) == 0 {
|
||||
return nil
|
||||
}
|
||||
mLock.Lock()
|
||||
m[string(labelValue)] = struct{}{}
|
||||
mLock.Unlock()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error when data fetching: %w", err)
|
||||
}
|
||||
mLock.Lock()
|
||||
m[string(labelValue)] = struct{}{}
|
||||
mLock.Unlock()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error when data fetching: %w", err)
|
||||
}
|
||||
|
||||
labelValues := make([]string, 0, len(m))
|
||||
for labelValue := range m {
|
||||
labelValues = append(labelValues, labelValue)
|
||||
@@ -639,7 +630,7 @@ func LabelsCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
|
||||
if err != nil {
|
||||
return fmt.Errorf(`cannot obtain label entries: %w`, err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteLabelsCountResponse(bw, labelEntries)
|
||||
@@ -690,7 +681,7 @@ func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
|
||||
if err != nil {
|
||||
return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %w`, date, topN, err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteTSDBStatusResponse(bw, status)
|
||||
@@ -760,7 +751,7 @@ func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteLabelsResponse(bw, labels)
|
||||
@@ -782,33 +773,41 @@ func labelsWithMatches(matches []string, start, end int64, deadline searchutils.
|
||||
if start >= end {
|
||||
end = start + defaultStep
|
||||
}
|
||||
sq := &storage.SearchQuery{
|
||||
MinTimestamp: start,
|
||||
MaxTimestamp: end,
|
||||
TagFilterss: tagFilterss,
|
||||
}
|
||||
rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss)
|
||||
m := make(map[string]struct{})
|
||||
var mLock sync.Mutex
|
||||
err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
|
||||
mLock.Lock()
|
||||
tags := rs.MetricName.Tags
|
||||
for i := range tags {
|
||||
t := &tags[i]
|
||||
m[string(t.Key)] = struct{}{}
|
||||
if end-start > 24*3600*1000 {
|
||||
// It is cheaper to call SearchMetricNames on time ranges exceeding a day.
|
||||
mns, err := netstorage.SearchMetricNames(sq, deadline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
|
||||
}
|
||||
for _, mn := range mns {
|
||||
for _, tag := range mn.Tags {
|
||||
m[string(tag.Key)] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(mns) > 0 {
|
||||
m["__name__"] = struct{}{}
|
||||
}
|
||||
} else {
|
||||
rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
var mLock sync.Mutex
|
||||
err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
|
||||
mLock.Lock()
|
||||
for _, tag := range rs.MetricName.Tags {
|
||||
m[string(tag.Key)] = struct{}{}
|
||||
}
|
||||
m["__name__"] = struct{}{}
|
||||
mLock.Unlock()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error when data fetching: %w", err)
|
||||
}
|
||||
m["__name__"] = struct{}{}
|
||||
mLock.Unlock()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error when data fetching: %w", err)
|
||||
}
|
||||
|
||||
labels := make([]string, 0, len(m))
|
||||
for label := range m {
|
||||
labels = append(labels, label)
|
||||
@@ -826,7 +825,7 @@ func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain series count: %w", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteSeriesCountResponse(bw, n)
|
||||
@@ -873,17 +872,39 @@ func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
|
||||
if start >= end {
|
||||
end = start + defaultStep
|
||||
}
|
||||
sq := &storage.SearchQuery{
|
||||
MinTimestamp: start,
|
||||
MaxTimestamp: end,
|
||||
TagFilterss: tagFilterss,
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss)
|
||||
if end-start > 24*3600*1000 {
|
||||
// It is cheaper to call SearchMetricNames on time ranges exceeding a day.
|
||||
mns, err := netstorage.SearchMetricNames(sq, deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
resultsCh := make(chan *quicktemplate.ByteBuffer)
|
||||
go func() {
|
||||
for i := range mns {
|
||||
bb := quicktemplate.AcquireByteBuffer()
|
||||
writemetricNameObject(bb, &mns[i])
|
||||
resultsCh <- bb
|
||||
}
|
||||
close(resultsCh)
|
||||
}()
|
||||
// WriteSeriesResponse must consume all the data from resultsCh.
|
||||
WriteSeriesResponse(bw, resultsCh)
|
||||
if err := bw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
seriesDuration.UpdateDuration(startTime)
|
||||
return nil
|
||||
}
|
||||
rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
resultsCh := make(chan *quicktemplate.ByteBuffer)
|
||||
@@ -1020,7 +1041,7 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteQueryResponse(bw, result)
|
||||
@@ -1119,7 +1140,7 @@ func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string,
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153
|
||||
result = removeEmptyValuesAndTimeseries(result)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
WriteQueryRangeResponse(bw, result)
|
||||
|
||||
@@ -653,11 +653,7 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc,
|
||||
} else {
|
||||
minTimestamp -= ec.Step
|
||||
}
|
||||
sq := &storage.SearchQuery{
|
||||
MinTimestamp: minTimestamp,
|
||||
MaxTimestamp: ec.End,
|
||||
TagFilterss: [][]storage.TagFilter{tfs},
|
||||
}
|
||||
sq := storage.NewSearchQuery(minTimestamp, ec.End, [][]storage.TagFilter{tfs})
|
||||
rss, err := netstorage.ProcessSearchQuery(sq, true, ec.Deadline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -15,7 +16,13 @@ import (
|
||||
"github.com/VictoriaMetrics/metricsql"
|
||||
)
|
||||
|
||||
var logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging")
|
||||
var (
|
||||
logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging")
|
||||
treatDotsAsIsInRegexps = flag.Bool("search.treatDotsAsIsInRegexps", false, "Whether to treat dots as is in regexp label filters used in queries. "+
|
||||
`For example, foo{bar=~"a.b.c"} will be automatically converted to foo{bar=~"a\\.b\\.c"}, i.e. all the dots in regexp filters will be automatically escaped `+
|
||||
`in order to match only dot char instead of matching any char. Dots in ".+", ".*" and ".{n}" regexps aren't escaped. `+
|
||||
`Such escaping can be useful when querying Graphite data`)
|
||||
)
|
||||
|
||||
var slowQueries = metrics.NewCounter(`vm_slow_queries_total`)
|
||||
|
||||
@@ -26,8 +33,8 @@ func Exec(ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result,
|
||||
defer func() {
|
||||
d := time.Since(startTime)
|
||||
if d >= *logSlowQueryDuration {
|
||||
logger.Warnf("slow query according to -search.logSlowQueryDuration=%s: duration=%.3f seconds, start=%d, end=%d, step=%d, query=%q",
|
||||
*logSlowQueryDuration, d.Seconds(), ec.Start/1000, ec.End/1000, ec.Step/1000, q)
|
||||
logger.Warnf("slow query according to -search.logSlowQueryDuration=%s: remoteAddr=%s, duration=%.3f seconds, start=%d, end=%d, step=%d, query=%q",
|
||||
*logSlowQueryDuration, ec.QuotedRemoteAddr, d.Seconds(), ec.Start/1000, ec.End/1000, ec.Step/1000, q)
|
||||
slowQueries.Inc()
|
||||
}
|
||||
}()
|
||||
@@ -177,6 +184,9 @@ func parsePromQLWithCache(q string) (metricsql.Expr, error) {
|
||||
if err == nil {
|
||||
e = metricsql.Optimize(e)
|
||||
e = adjustCmpOps(e)
|
||||
if *treatDotsAsIsInRegexps {
|
||||
e = escapeDotsInRegexpLabelFilters(e)
|
||||
}
|
||||
}
|
||||
pcv = &parseCacheValue{
|
||||
e: e,
|
||||
@@ -190,6 +200,41 @@ func parsePromQLWithCache(q string) (metricsql.Expr, error) {
|
||||
return pcv.e, nil
|
||||
}
|
||||
|
||||
func escapeDotsInRegexpLabelFilters(e metricsql.Expr) metricsql.Expr {
|
||||
metricsql.VisitAll(e, func(expr metricsql.Expr) {
|
||||
me, ok := expr.(*metricsql.MetricExpr)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
for i := range me.LabelFilters {
|
||||
f := &me.LabelFilters[i]
|
||||
if f.IsRegexp {
|
||||
f.Value = escapeDots(f.Value)
|
||||
}
|
||||
}
|
||||
})
|
||||
return e
|
||||
}
|
||||
|
||||
func escapeDots(s string) string {
|
||||
dotsCount := strings.Count(s, ".")
|
||||
if dotsCount <= 0 {
|
||||
return s
|
||||
}
|
||||
result := make([]byte, 0, len(s)+2*dotsCount)
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '.' && (i == 0 || s[i-1] != '\\') && (i+1 == len(s) || i+1 < len(s) && s[i+1] != '*' && s[i+1] != '+' && s[i+1] != '{') {
|
||||
// Escape a dot if the following conditions are met:
|
||||
// - if it isn't escaped already, i.e. if there is no `\` char before the dot.
|
||||
// - if there is no regexp modifiers such as '+', '*' or '{' after the dot.
|
||||
result = append(result, '\\', '.')
|
||||
} else {
|
||||
result = append(result, s[i])
|
||||
}
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
var parseCacheV = func() *parseCache {
|
||||
pc := &parseCache{
|
||||
m: make(map[string]*parseCacheValue),
|
||||
|
||||
@@ -7,8 +7,46 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
"github.com/VictoriaMetrics/metricsql"
|
||||
)
|
||||
|
||||
func TestEscapeDots(t *testing.T) {
|
||||
f := func(s, resultExpected string) {
|
||||
t.Helper()
|
||||
result := escapeDots(s)
|
||||
if result != resultExpected {
|
||||
t.Fatalf("unexpected result for escapeDots(%q); got\n%s\nwant\n%s", s, result, resultExpected)
|
||||
}
|
||||
}
|
||||
f("", "")
|
||||
f("a", "a")
|
||||
f("foobar", "foobar")
|
||||
f(".", `\.`)
|
||||
f(".*", `.*`)
|
||||
f(".+", `.+`)
|
||||
f("..", `\.\.`)
|
||||
f("foo.b.{2}ar..+baz.*", `foo\.b.{2}ar\..+baz.*`)
|
||||
}
|
||||
|
||||
func TestEscapeDotsInRegexpLabelFilters(t *testing.T) {
|
||||
f := func(s, resultExpected string) {
|
||||
t.Helper()
|
||||
e, err := metricsql.Parse(s)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error in metricsql.Parse(%q): %s", s, err)
|
||||
}
|
||||
e = escapeDotsInRegexpLabelFilters(e)
|
||||
result := e.AppendString(nil)
|
||||
if string(result) != resultExpected {
|
||||
t.Fatalf("unexpected result for escapeDotsInRegexpLabelFilters(%q);\ngot\n%s\nwant\n%s", s, result, resultExpected)
|
||||
}
|
||||
}
|
||||
f("2", "2")
|
||||
f(`foo.bar + 123`, `foo.bar + 123`)
|
||||
f(`foo{bar=~"baz.xx.yyy"}`, `foo{bar=~"baz\\.xx\\.yyy"}`)
|
||||
f(`foo(a.b{c="d.e",x=~"a.b.+[.a]",y!~"aaa.bb|cc.dd"}) + x.y(1,sum({x=~"aa.bb"}))`, `foo(a.b{c="d.e", x=~"a\\.b.+[\\.a]", y!~"aaa\\.bb|cc\\.dd"}) + x.y(1, sum({x=~"aa\\.bb"}))`)
|
||||
}
|
||||
|
||||
func TestExecSuccess(t *testing.T) {
|
||||
start := int64(1000e3)
|
||||
end := int64(2000e3)
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
"github.com/valyala/histogram"
|
||||
)
|
||||
|
||||
var minStalenessInterval = flag.Duration("search.minStalenessInterval", 0, "The mimimum interval for staleness calculations. "+
|
||||
var minStalenessInterval = flag.Duration("search.minStalenessInterval", 0, "The minimum interval for staleness calculations. "+
|
||||
"This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. "+
|
||||
"See also '-search.maxStalenessInterval'")
|
||||
|
||||
@@ -326,14 +326,32 @@ func getRollupFunc(funcName string) newRollupFunc {
|
||||
}
|
||||
|
||||
type rollupFuncArg struct {
|
||||
prevValue float64
|
||||
prevTimestamp int64
|
||||
values []float64
|
||||
timestamps []int64
|
||||
// The value preceeding values if it fits staleness interval.
|
||||
prevValue float64
|
||||
|
||||
// The timestamp for prevValue.
|
||||
prevTimestamp int64
|
||||
|
||||
// Values that fit window ending at currTimestamp.
|
||||
values []float64
|
||||
|
||||
// Timestamps for values.
|
||||
timestamps []int64
|
||||
|
||||
// Real value preceeding values without restrictions on staleness interval.
|
||||
realPrevValue float64
|
||||
|
||||
// Real value which goes after values.
|
||||
realNextValue float64
|
||||
|
||||
// Current timestamp for rollup evaluation.
|
||||
currTimestamp int64
|
||||
idx int
|
||||
window int64
|
||||
|
||||
// Index for the currently evaluated point relative to time range for query evaluation.
|
||||
idx int
|
||||
|
||||
// Time window for rollup calculations.
|
||||
window int64
|
||||
|
||||
tsm *timeseriesMap
|
||||
}
|
||||
@@ -507,7 +525,9 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
|
||||
ni := 0
|
||||
nj := 0
|
||||
stalenessInterval := int64(float64(scrapeInterval) * 0.9)
|
||||
canDropLastSample := rc.CanDropLastSample
|
||||
// Do not drop trailing data points for queries, which return 2 or 1 point (aka instant queries).
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/845
|
||||
canDropLastSample := rc.CanDropLastSample && len(rc.Timestamps) > 2
|
||||
for _, tEnd := range rc.Timestamps {
|
||||
tStart := tEnd - window
|
||||
ni = seekFirstTimestampIdxAfter(timestamps[i:], tStart, ni)
|
||||
@@ -526,17 +546,26 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
|
||||
}
|
||||
rfa.values = values[i:j]
|
||||
rfa.timestamps = timestamps[i:j]
|
||||
if canDropLastSample && j == len(timestamps) && j > 0 && (tEnd-timestamps[j-1] > stalenessInterval || i == j && len(timestamps) == 1) && rc.End-tEnd >= 2*rc.Step {
|
||||
if canDropLastSample && j == len(timestamps) && j > 0 && (tEnd-timestamps[j-1] > stalenessInterval || i == j && len(timestamps) == 1) {
|
||||
// Drop trailing data points in the following cases:
|
||||
// - if the distance between the last raw sample and tEnd exceeds stalenessInterval
|
||||
// - if time series contains only a single raw sample
|
||||
// This should prevent from double counting when a label changes in time series (for instance,
|
||||
// during new deployment in K8S). See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748
|
||||
// Do not drop trailing data points for instant queries. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/845
|
||||
rfa.prevValue = nan
|
||||
rfa.values = nil
|
||||
rfa.timestamps = nil
|
||||
}
|
||||
if i > 0 {
|
||||
rfa.realPrevValue = values[i-1]
|
||||
} else {
|
||||
rfa.realPrevValue = nan
|
||||
}
|
||||
if j < len(values) {
|
||||
rfa.realNextValue = values[j]
|
||||
} else {
|
||||
rfa.realNextValue = nan
|
||||
}
|
||||
rfa.currTimestamp = tEnd
|
||||
value := rc.Func(rfa)
|
||||
rfa.idx++
|
||||
@@ -1243,6 +1272,12 @@ func rollupDelta(rfa *rollupFuncArg) float64 {
|
||||
if len(values) == 0 {
|
||||
return nan
|
||||
}
|
||||
if !math.IsNaN(rfa.realPrevValue) {
|
||||
// Assume that the value didn't change during the current gap.
|
||||
// This should fix high delta() and increase() values at the end of gaps.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/894
|
||||
return values[len(values)-1] - rfa.realPrevValue
|
||||
}
|
||||
// Assume that the previous non-existing value was 0
|
||||
// only if the first value doesn't exceed too much the delta with the next value.
|
||||
//
|
||||
@@ -1255,6 +1290,8 @@ func rollupDelta(rfa *rollupFuncArg) float64 {
|
||||
d := float64(10)
|
||||
if len(values) > 1 {
|
||||
d = values[1] - values[0]
|
||||
} else if !math.IsNaN(rfa.realNextValue) {
|
||||
d = rfa.realNextValue - values[0]
|
||||
}
|
||||
if math.Abs(values[0]) < 10*(math.Abs(d)+1) {
|
||||
prevValue = 0
|
||||
|
||||
@@ -1103,11 +1103,13 @@ func testRowsEqual(t *testing.T, values []float64, timestamps []int64, valuesExp
|
||||
}
|
||||
|
||||
func TestRollupDelta(t *testing.T) {
|
||||
f := func(prevValue float64, values []float64, resultExpected float64) {
|
||||
f := func(prevValue, realPrevValue, realNextValue float64, values []float64, resultExpected float64) {
|
||||
t.Helper()
|
||||
rfa := &rollupFuncArg{
|
||||
prevValue: prevValue,
|
||||
values: values,
|
||||
prevValue: prevValue,
|
||||
values: values,
|
||||
realPrevValue: realPrevValue,
|
||||
realNextValue: realNextValue,
|
||||
}
|
||||
result := rollupDelta(rfa)
|
||||
if math.IsNaN(result) {
|
||||
@@ -1120,22 +1122,36 @@ func TestRollupDelta(t *testing.T) {
|
||||
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
|
||||
}
|
||||
}
|
||||
f(nan, nil, nan)
|
||||
f(nan, nan, nan, nil, nan)
|
||||
|
||||
// Small initial value
|
||||
f(nan, []float64{1}, 1)
|
||||
f(nan, []float64{10}, 10)
|
||||
f(nan, []float64{100}, 100)
|
||||
f(nan, []float64{1, 2, 3}, 3)
|
||||
f(1, []float64{1, 2, 3}, 2)
|
||||
f(nan, []float64{5, 6, 8}, 8)
|
||||
f(2, []float64{5, 6, 8}, 6)
|
||||
f(nan, nan, nan, []float64{1}, 1)
|
||||
f(nan, nan, nan, []float64{10}, 10)
|
||||
f(nan, nan, nan, []float64{100}, 100)
|
||||
f(nan, nan, nan, []float64{1, 2, 3}, 3)
|
||||
f(1, nan, nan, []float64{1, 2, 3}, 2)
|
||||
f(nan, nan, nan, []float64{5, 6, 8}, 8)
|
||||
f(2, nan, nan, []float64{5, 6, 8}, 6)
|
||||
|
||||
// Too big initial value must be skipped.
|
||||
f(nan, []float64{1000}, 0)
|
||||
f(nan, []float64{1000, 1001, 1002}, 2)
|
||||
f(nan, nan, nan, []float64{1000}, 0)
|
||||
f(nan, nan, nan, []float64{1000, 1001, 1002}, 2)
|
||||
|
||||
// Non-nan realPrevValue
|
||||
f(nan, 900, nan, []float64{1000}, 100)
|
||||
f(nan, 1000, nan, []float64{1000}, 0)
|
||||
f(nan, 1100, nan, []float64{1000}, -100)
|
||||
f(nan, 900, nan, []float64{1000, 1001, 1002}, 102)
|
||||
|
||||
// Small delta between realNextValue and values
|
||||
f(nan, nan, 990, []float64{1000}, 0)
|
||||
f(nan, nan, 1005, []float64{1000}, 0)
|
||||
|
||||
// Big delta between relaNextValue and values
|
||||
f(nan, nan, 800, []float64{1000}, 1000)
|
||||
f(nan, nan, 1300, []float64{1000}, 1000)
|
||||
|
||||
// Empty values
|
||||
f(1, nil, 0)
|
||||
f(100, nil, 0)
|
||||
f(1, nan, nan, nil, 0)
|
||||
f(100, nan, nan, nil, 0)
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ var (
|
||||
retentionPeriod = flagutil.NewDuration("retentionPeriod", 1, "Data with timestamps outside the retentionPeriod is automatically deleted")
|
||||
snapshotAuthKey = flag.String("snapshotAuthKey", "", "authKey, which must be passed in query string to /snapshot* pages")
|
||||
forceMergeAuthKey = flag.String("forceMergeAuthKey", "", "authKey, which must be passed in query string to /internal/force_merge pages")
|
||||
forceFlushAuthKey = flag.String("forceFlushAuthKey", "", "authKey, which must be passed in query string to /internal/force_flush pages")
|
||||
|
||||
precisionBits = flag.Int("precisionBits", 64, "The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss")
|
||||
|
||||
@@ -112,6 +113,14 @@ func AddRows(mrs []storage.MetricRow) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// RegisterMetricNames registers all the metrics from mrs in the storage.
|
||||
func RegisterMetricNames(mrs []storage.MetricRow) error {
|
||||
WG.Add(1)
|
||||
err := Storage.RegisterMetricNames(mrs)
|
||||
WG.Done()
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteMetrics deletes metrics matching tfss.
|
||||
//
|
||||
// Returns the number of deleted metrics.
|
||||
@@ -122,6 +131,14 @@ func DeleteMetrics(tfss []*storage.TagFilters) (int, error) {
|
||||
return n, err
|
||||
}
|
||||
|
||||
// SearchMetricNames returns metric names for the given tfss on the given tr.
|
||||
func SearchMetricNames(tfss []*storage.TagFilters, tr storage.TimeRange, maxMetrics int, deadline uint64) ([]storage.MetricName, error) {
|
||||
WG.Add(1)
|
||||
mns, err := Storage.SearchMetricNames(tfss, tr, maxMetrics, deadline)
|
||||
WG.Done()
|
||||
return mns, err
|
||||
}
|
||||
|
||||
// SearchTagKeysOnTimeRange searches for tag keys on tr.
|
||||
func SearchTagKeysOnTimeRange(tr storage.TimeRange, maxTagKeys int, deadline uint64) ([]string, error) {
|
||||
WG.Add(1)
|
||||
@@ -222,6 +239,16 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
}()
|
||||
return true
|
||||
}
|
||||
if path == "/internal/force_flush" {
|
||||
authKey := r.FormValue("authKey")
|
||||
if authKey != *forceFlushAuthKey {
|
||||
httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -forceFlushAuthKey command line flag", authKey)
|
||||
return true
|
||||
}
|
||||
logger.Infof("flushing storage to make pending data available for reading")
|
||||
Storage.DebugFlush()
|
||||
return true
|
||||
}
|
||||
prometheusCompatibleResponse := false
|
||||
if path == "/api/v1/admin/tsdb/snapshot" {
|
||||
// Handle Prometheus API - https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot .
|
||||
@@ -240,7 +267,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
|
||||
switch path {
|
||||
case "/create":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
snapshotPath, err := Storage.CreateSnapshot()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("cannot create snapshot: %w", err)
|
||||
@@ -254,7 +281,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
}
|
||||
return true
|
||||
case "/list":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
snapshots, err := Storage.ListSnapshots()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("cannot list snapshots: %w", err)
|
||||
@@ -271,7 +298,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
fmt.Fprintf(w, `]}`)
|
||||
return true
|
||||
case "/delete":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
snapshotName := r.FormValue("snapshot")
|
||||
if err := Storage.DeleteSnapshot(snapshotName); err != nil {
|
||||
err = fmt.Errorf("cannot delete snapshot %q: %w", snapshotName, err)
|
||||
@@ -281,7 +308,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
fmt.Fprintf(w, `{"status":"ok"}`)
|
||||
return true
|
||||
case "/delete_all":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
snapshots, err := Storage.ListSnapshots()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("cannot list snapshots: %w", err)
|
||||
|
||||
@@ -4,7 +4,7 @@ DOCKER_NAMESPACE := victoriametrics
|
||||
|
||||
ROOT_IMAGE ?= alpine:3.12.1
|
||||
CERTS_IMAGE := alpine:3.12.1
|
||||
GO_BUILDER_IMAGE := golang:1.15.4
|
||||
GO_BUILDER_IMAGE := golang:1.15.5
|
||||
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr : _)
|
||||
BASE_IMAGE := local/base:1.1.1-$(shell echo $(ROOT_IMAGE) | tr : _)-$(shell echo $(CERTS_IMAGE) | tr : _)
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
* [Calculating the Error of Quantile Estimation with Histograms](https://linuxczar.net/blog/2020/08/13/histogram-error/)
|
||||
* [Monitoring private clouds with VictoriaMetrics at LeroyMerlin](https://www.youtube.com/watch?v=74swsWqf0Uc)
|
||||
* [Monitoring Kubernetes with VictoriaMetrics+Prometheus](https://speakerdeck.com/bo0km4n/victoriametrics-plus-prometheusdegou-zhu-surufu-shu-kubernetesfalsejian-shi-ji-pan)
|
||||
* [High-performance Graphite storage solution on top of VictoriaMetrics](https://golangexample.com/a-high-performance-graphite-storage-solution/)
|
||||
|
||||
|
||||
## Our articles
|
||||
@@ -48,3 +49,4 @@
|
||||
* [Filtering and modifying time series during import to VictoriaMetrics](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-filtering-and-modifying-time-series-6d40cea4bf21)
|
||||
* [Anomaly Detection in VictoriaMetrics](https://medium.com/@VictoriaMetrics/anomaly-detection-in-victoriametrics-9528538786a7)
|
||||
* [How to use relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
|
||||
* [First look at performance comparison between InfluxDB IOx and VictoriaMetrics](https://medium.com/@VictoriaMetrics/first-look-at-perfomance-comparassion-between-influxdb-iox-and-victoriametrics-e590f847935b)
|
||||
|
||||
@@ -3,6 +3,62 @@
|
||||
# tip
|
||||
|
||||
|
||||
# [v1.48.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.48.0)
|
||||
|
||||
* FEATURE: added [Snap package for single-node VictoriaMetrics](https://snapcraft.io/victoriametrics). This simplifies installation under Ubuntu to a single command:
|
||||
```bash
|
||||
snap install victoriametrics
|
||||
```
|
||||
* FEATURE: vmselect: add `-replicationFactor` command-line flag for reducing query duration when replication is enabled and a part of vmstorage nodes
|
||||
are temporarily slow and/or temporarily unavailable. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711
|
||||
* FEATURE: vminsert: export `vm_rpc_vmstorage_is_reachable` metric, which can be used for monitoring reachability of vmstorage nodes from vminsert nodes.
|
||||
* FEATURE: vmagent: add [Netflix Eureka](https://github.com/Netflix/eureka) service discovery (aka [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)). See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/851
|
||||
* FEATURE: add `filters` option to `dockerswarm_sd_config` like Prometheus did in v2.23.0 - see https://github.com/prometheus/prometheus/pull/8074
|
||||
* FEATURE: expose `__meta_ec2_ipv6_addresses` label for `ec2_sd_config` like Prometheus will do in the next release.
|
||||
* FEATURE: add `-loggerWarnsPerSecondLimit` command-line flag for rate limiting of WARN messages in logs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/905
|
||||
* FEATURE: apply `loggerErrorsPerSecondLimit` and `-loggerWarnsPerSecondLimit` rate limit per caller. I.e. log messages are suppressed if the same caller logs the same message
|
||||
at the rate exceeding the given limit. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/905#issuecomment-729395855
|
||||
* FEATURE: add remoteAddr to slow query log in order to simplify identifying the client that sends slow queries to VictoriaMetrics.
|
||||
Slow query logging is controlled with `-search.logSlowQueryDuration` command-line flag.
|
||||
* FEATURE: add `/tags/delSeries` handler from Graphite Tags API. See https://victoriametrics.github.io/#graphite-tags-api-usage
|
||||
* FEATURE: log metric name plus all its labels when the metric timestamp is out of the configured retention. This should simplify detecting the source of metrics with unexpected timestamps.
|
||||
* FEATURE: add `-dryRun` command-line flag to single-node VictoriaMetrics in order to check config file pointed by `-promscrape.config`.
|
||||
|
||||
* BUGFIX: properly parse Prometheus metrics with [exemplars](https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md#exemplars-1) such as `foo 123 # {bar="baz"} 1`.
|
||||
* BUGFIX: properly parse "infinity" values in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md#abnf).
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/924
|
||||
|
||||
|
||||
# [v1.47.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.47.0)
|
||||
|
||||
* FEATURE: vmselect: return the original error from `vmstorage` node in query response if `-search.denyPartialResponse` is set.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/891
|
||||
* FEATURE: vmselect: add `"isPartial":{true|false}` field in JSON output for `/api/v1/*` functions
|
||||
from [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/). `"isPartial":true` is set if the response contains partial data
|
||||
because of a part of `vmstorage` nodes were unavailable during query processing.
|
||||
* FEATURE: improve performance for `/api/v1/series`, `/api/v1/labels` and `/api/v1/label/<labelName>/values` on time ranges exceeding one day.
|
||||
* FEATURE: vmagent: reduce memory usage when service discovery detects big number of scrape targets and the set of discovered targets changes over time.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825
|
||||
* FEATURE: vmagent: add `-promscrape.dropOriginalLabels` command-line option, which can be used for reducing memory usage when scraping big number of targets.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825#issuecomment-724308361 for details.
|
||||
* FEATURE: vmalert: explicitly set extra labels to alert entities. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/870
|
||||
* FEATURE: add `-search.treatDotsAsIsInRegexps` command-line flag, which can be used for automatic escaping of dots in regexp label filters used in queries.
|
||||
For example, if `-search.treatDotsAsIsInRegexps` is set, then the query `foo{bar=~"aaa.bb.cc|dd.eee"}` is automatically converted to `foo{bar=~"aaa\\.bb\\.cc|dd\\.eee"}`.
|
||||
This may be useful for querying Graphite data.
|
||||
* FEATURE: consistently return text-based HTTP responses such as `plain/text` and `application/json` with `charset=utf-8`.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/897
|
||||
* FEATURE: update Go builder from v1.15.4 to v1.15.5. This should fix [these issues in Go](https://github.com/golang/go/issues?q=milestone%3AGo1.15.5+label%3ACherryPickApproved).
|
||||
* FEATURE: added `/internal/force_flush` http handler for flushing recently ingested data from in-memory buffers to persistent storage.
|
||||
See [troubleshooting docs](https://victoriametrics.github.io/#troubleshooting) for more details.
|
||||
* FEATURE: added [Graphite Tags API](https://graphite.readthedocs.io/en/stable/tags.html) support.
|
||||
See [these docs](https://victoriametrics.github.io/#graphite-tags-api-usage) for details.
|
||||
|
||||
* BUGFIX: do not return data points in the end of the selected time range for time series ending in the middle of the selected time range.
|
||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/887 and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/845
|
||||
* BUGFIX: remove spikes at the end of time series gaps for `increase()` or `delta()` functions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/894
|
||||
* BUGFIX: vminsert: properly return HTTP 503 status code when all the vmstorage nodes are unavailable. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/896
|
||||
|
||||
|
||||
# [v1.46.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.46.0)
|
||||
|
||||
* FEATURE: optimize requests to `/api/v1/labels` and `/api/v1/label/<name>/values` when `start` and `end` args are set.
|
||||
|
||||
@@ -6,7 +6,7 @@ and feel free asking for references, reviews and additional case studies from re
|
||||
See also [articles about VictoriaMetrics from our users](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Articles#third-party-articles-and-slides).
|
||||
|
||||
|
||||
* [Adidas](#adidas)
|
||||
* [adidas](#adidas)
|
||||
* [CERN](#cern)
|
||||
* [COLOPL](#colopl)
|
||||
* [Zerodha](#zerodha)
|
||||
@@ -20,7 +20,7 @@ See also [articles about VictoriaMetrics from our users](https://github.com/Vict
|
||||
* [Idealo.de](#idealode)
|
||||
|
||||
|
||||
## Adidas
|
||||
## adidas
|
||||
|
||||
See [slides](https://promcon.io/2019-munich/slides/remote-write-storage-wars.pdf) and [video](https://youtu.be/OsH6gPdxR4s)
|
||||
from [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) talk at [PromCon 2019](https://promcon.io/2019-munich/).
|
||||
|
||||
@@ -181,7 +181,7 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr
|
||||
- `prometheus/api/v1/import/csv` - for importing arbitrary CSV data. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data) for details.
|
||||
- `prometheus/api/v1/import/prometheus` - for importing data in Prometheus exposition format. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-prometheus-exposition-format) for details.
|
||||
|
||||
* URLs for [Prmetheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/): `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:
|
||||
* URLs for [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/): `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:
|
||||
- `<accountID>` is an arbitrary number identifying data namespace for the query (aka tenant)
|
||||
- `<suffix>` may have the following values:
|
||||
- `api/v1/query` - performs [PromQL instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries).
|
||||
@@ -194,6 +194,8 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr
|
||||
- `api/v1/export/native` - exports raw data in native binary format. It may be imported into another VictoriaMetrics via `api/v1/import/native` (see above).
|
||||
- `api/v1/export/csv` - exports data in CSV. It may be imported into another VictoriaMetrics via `api/v1/import/csv` (see above).
|
||||
- `api/v1/status/tsdb` - for time series stats. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats) for details.
|
||||
VictoriaMetrics accepts optional `topN=N` and `date=YYYY-MM-DD` query args for this handler, where `N` is the number of top entries to return in the response
|
||||
and `YYYY-MM-DD` is the date for collecting the stats. By default the stats is collected for the current day.
|
||||
- `api/v1/status/active_queries` - for currently executed active queries. Note that every `vmselect` maintains an independent list of active queries,
|
||||
which is returned in the response.
|
||||
|
||||
@@ -203,6 +205,14 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr
|
||||
- `metrics/find` - searches Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find).
|
||||
- `metrics/expand` - expands Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand).
|
||||
- `metrics/index.json` - returns all the metric names. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json).
|
||||
- `tags/tagSeries` - registers time series. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb).
|
||||
- `tags/tagMultiSeries` - register multiple time series. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb).
|
||||
- `tags` - returns tag names. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags).
|
||||
- `tags/<tag_name>` - returns tag values for the given `<tag_name>`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags).
|
||||
- `tags/findSeries` - returns series matching the given `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags).
|
||||
- `tags/autoComplete/tags` - returns tags matching the given `tagPrefix` and/or `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support).
|
||||
- `tags/autoComplete/values` - returns tag values matching the given `valuePrefix` and/or `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support).
|
||||
- `tags/delSeries` - deletes series matching the given `path`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb).
|
||||
|
||||
* URL for time series deletion: `http://<vmselect>:8481/delete/<accountID>/prometheus/api/v1/admin/tsdb/delete_series?match[]=<timeseries_selector_for_delete>`.
|
||||
Note that the `delete_series` handler should be used only in exceptional cases such as deletion of accidentally ingested incorrect time series. It shouldn't
|
||||
@@ -278,7 +288,7 @@ Each instance type - `vminsert`, `vmselect` and `vmstorage` - can run on the mos
|
||||
#### vmstorage
|
||||
|
||||
* The recommended total number of vCPU cores for all the `vmstorage` instances can be calculated from the ingestion rate: `vCPUs = ingestion_rate / 150K`.
|
||||
* The recommended total amount of RAM for all the `vmstorage` instances can be calculated from the number of active time series: `RAM = active_time_series * 1KB`.
|
||||
* The recommended total amount of RAM for all the `vmstorage` instances can be calculated from the number of active time series: `RAM = 2 * active_time_series * 1KB`.
|
||||
Time series is active if it received at least a single data point during the last hour or if it has been queried during the last hour.
|
||||
The required RAM per each `vmstorage` should be multiplied by `-replicationFactor` if [replication](#replication-and-data-safety) is enabled.
|
||||
Additional RAM can be required for query processing.
|
||||
@@ -328,8 +338,9 @@ In order to enable application-level replication, `-replicationFactor=N` command
|
||||
This guarantees that all the data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable.
|
||||
For example, when `-replicationFactor=3` is passed to `vminsert`, then it replicates all the ingested data to 3 distinct `vmstorage` nodes.
|
||||
|
||||
When the replication is enabled, `-dedup.minScrapeInterval=1ms` command-line flag must be passed to `vmselect`
|
||||
in order to de-duplicate replicated data during queries. It is OK if `-dedup.minScrapeInterval` exceeds 1ms
|
||||
When the replication is enabled, `-replicationFactor=N` and `-dedup.minScrapeInterval=1ms` command-line flag must be passed to `vmselect` nodes.
|
||||
The `-replicationFactor=N` improves query performance when a part of vmstorage nodes respond slowly and/or temporarily unavailable.
|
||||
The `-dedup.minScrapeInterval=1ms` de-duplicates replicated data during queries. It is OK if `-dedup.minScrapeInterval` exceeds 1ms
|
||||
when [deduplication](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#deduplication) is used additionally to replication.
|
||||
|
||||
Note that [replication doesn't save from disaster](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883),
|
||||
|
||||
@@ -90,7 +90,7 @@ The main differences between Cortex and VictoriaMetrics:
|
||||
- Cortex may lose up to 12 hours of recent data on Ingestor failure - see [the corresponding docs](https://github.com/cortexproject/cortex/blob/fe56f1420099aa1bf1ce09316c186e05bddee879/docs/architecture.md#ingesters-failure-and-data-loss).
|
||||
VictoriaMetrics may lose only a few seconds of recent data, which isn't synced to persistent storage yet.
|
||||
See [this article for details](https://medium.com/@valyala/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704).
|
||||
- Cortex is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from Adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) and [other case studies](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies).
|
||||
- Cortex is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) and [other case studies](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies).
|
||||
- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to Prometheus remote_write protocol - InfluxDB, OpenTSDB, Graphite, CSV, JSON, native binary.
|
||||
See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data) for details.
|
||||
|
||||
@@ -111,7 +111,7 @@ The main differences between Cortex and VictoriaMetrics:
|
||||
which is much easier to setup and operate than Thanos components.
|
||||
- Thanos may be harder to setup and operate comparing to VictoriaMetrics, since it has more moving parts, which can be connected with less reliable networks.
|
||||
See [this article for details](https://medium.com/faun/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
|
||||
- Thanos is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from Adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
|
||||
- Thanos is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
|
||||
- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to Prometheus remote_write protocol - InfluxDB, OpenTSDB, Graphite, CSV, JSON, native binary.
|
||||
See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data) for details.
|
||||
|
||||
|
||||
@@ -68,15 +68,15 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
|
||||
- `step()` function for returning the step in seconds used in the query.
|
||||
- `start()` and `end()` functions for returning the start and end timestamps of the `[start ... end]` range used in the query.
|
||||
- `integrate(m[d])` for returning integral over the given duration `d` for the given metric `m`.
|
||||
- `ideriv(m)` - for calculating `instant` derivative for `m`.
|
||||
- `ideriv(m[d])` - for calculating `instant` derivative for the metric `m` over the duration `d`.
|
||||
- `deriv_fast(m[d])` - for calculating `fast` derivative for `m` based on the first and the last points from duration `d`.
|
||||
- `running_` functions - `running_sum`, `running_min`, `running_max`, `running_avg` - for calculating [running values](https://en.wikipedia.org/wiki/Running_total) on the selected time range.
|
||||
- `range_` functions - `range_sum`, `range_min`, `range_max`, `range_avg`, `range_first`, `range_last`, `range_median`, `range_quantile` - for calculating global value over the selected time range. Note that global value is based on calculated datapoints for the inner query. The calculated datapoints can differ from raw datapoints stored in the database. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness) for details.
|
||||
- `smooth_exponential(q, sf)` - smooths `q` using [exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average) with the given smooth factor `sf`.
|
||||
- `remove_resets(q)` - removes counter resets from `q`.
|
||||
- `lag(q[d])` - returns lag between the current timestamp and the timestamp from the previous data point in `q` over `d`.
|
||||
- `lifetime(q[d])` - returns lifetime of `q` over `d` in seconds. It is expected that `d` exceeds the lifetime of `q`.
|
||||
- `scrape_interval(q[d])` - returns the average interval in seconds between data points of `q` over `d` aka `scrape interval`.
|
||||
- `lag(m[d])` - returns lag between the current timestamp and the timestamp from the previous data point in `m` over `d`.
|
||||
- `lifetime(m[d])` - returns lifetime of `q` over `d` in seconds. It is expected that `d` exceeds the lifetime of `m`.
|
||||
- `scrape_interval(m[d])` - returns the average interval in seconds between data points of `m` over `d` aka `scrape interval`.
|
||||
- Trigonometric functions - `sin(q)`, `cos(q)`, `asin(q)`, `acos(q)` and `pi()`.
|
||||
- `range_over_time(m[d])` - returns value range for `m` over `d` time window, i.e. `max_over_time(m[d])-min_over_time(m[d])`.
|
||||
- `median_over_time(m[d])` - calculates median values for `m` over `d` time window. Shorthand to `quantile_over_time(0.5, m[d])`.
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# Quick Start
|
||||
|
||||
1. Download the latest VictoriaMetrics release from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
|
||||
1. If you run Ubuntu, then just run `snap install victoriametrics` command in order to install and start VictoriaMetrics, then read [these docs](https://snapcraft.io/victoriametrics).
|
||||
Otherwise download the latest VictoriaMetrics release from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
|
||||
from [Docker hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/)
|
||||
or [build it from sources](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Single-server-VictoriaMetrics#how-to-build-from-sources).
|
||||
|
||||
2. Run the binary or Docker image with the desired command-line flags. Pass `-help` in order to see description for all the available flags
|
||||
2. This step isn't needed if you run VictoriaMetrics via `snap install victoriametrics` as described above.
|
||||
Otherwise run the binary or Docker image with the desired command-line flags. Pass `-help` in order to see description for all the available flags
|
||||
and their default values. Default flag values should fit the majoirty of cases. The minimum required flags to configure are:
|
||||
|
||||
* `-storageDataPath` - path to directory where VictoriaMetrics stores all the data.
|
||||
|
||||
@@ -2,7 +2,7 @@ Release process guidance
|
||||
|
||||
## Release version and Docker images
|
||||
|
||||
0. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/CHANGELOG.md).
|
||||
0. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md).
|
||||
1. Create release tag with `git tag v1.xx.y`.
|
||||
2. Run `make release` for creating `*.tar.gz` release archive with the corresponding `_checksums.txt` inside `bin` directory.
|
||||
3. Run `make publish` for creating and publishing Docker images.
|
||||
|
||||
@@ -10,23 +10,27 @@
|
||||
|
||||
## VictoriaMetrics
|
||||
|
||||
VictoriaMetrics is fast, cost-effective and scalable time-series database.
|
||||
VictoriaMetrics is fast, cost-effective and scalable monitoring solution and time series database.
|
||||
|
||||
It is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
|
||||
[docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and
|
||||
in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics and see [how to start it](#how-to-start-victoriametrics).
|
||||
[docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Snap package](https://snapcraft.io/victoriametrics)
|
||||
and in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics and see [how to start it](#how-to-start-victoriametrics).
|
||||
If you use Ubuntu, then just run `snap install victoriametrics` in order to install and run it.
|
||||
Then read [Prometheus setup](#prometheus-setup) and [Grafana setup](#grafana-setup) docs.
|
||||
|
||||
Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
|
||||
|
||||
See our [Wiki](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki) for additional documentation.
|
||||
|
||||
[Contact us](mailto:info@victoriametrics.com) if you need paid enterprise support for VictoriaMetrics.
|
||||
See [features available for enterprise customers](https://github.com/VictoriaMetrics/VictoriaMetrics/issues?q=is%3Aissue+label%3Aenterprise).
|
||||
See [features available for enterprise customers](https://victoriametrics.com/enterprise.html).
|
||||
|
||||
|
||||
## Case studies and talks
|
||||
|
||||
* [Adidas](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#adidas)
|
||||
Click on a link in order to read the corresponding case study
|
||||
|
||||
* [adidas](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#adidas)
|
||||
* [CERN](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#cern)
|
||||
* [COLOPL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#colopl)
|
||||
* [Zerodha](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#zerodha)
|
||||
@@ -46,8 +50,8 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
|
||||
* VictoriaMetrics can be used as long-term storage for Prometheus or for [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md).
|
||||
See [these docs](#prometheus-setup) for details.
|
||||
* Supports [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/), so it can be used as Prometheus drop-in replacement in Grafana.
|
||||
VictoriaMetrics implements [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL) query language, which is inspired by PromQL.
|
||||
* Supports global query view. Multiple Prometheus instances may write data into VictoriaMetrics. Later this data may be used in a single query.
|
||||
VictoriaMetrics implements [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL) query language, which inspired by PromQL. MetricsQL is backwards-compatible with PromQL.
|
||||
* Supports global query view. Multiple Prometheus instances or any other data sources may write data into VictoriaMetrics. Later this data may be queried in a single query.
|
||||
* High performance and good scalability for both [inserts](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
|
||||
and [selects](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4).
|
||||
[Outperforms InfluxDB and TimescaleDB by up to 20x](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
|
||||
@@ -104,7 +108,9 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
|
||||
* [How to send data from OpenTSDB-compatible agents](#how-to-send-data-from-opentsdb-compatible-agents)
|
||||
* [Prometheus querying API usage](#prometheus-querying-api-usage)
|
||||
* [Prometheus querying API enhancements](#prometheus-querying-api-enhancements)
|
||||
* [Graphite Metrics API usage](#graphite-metrics-api-usage)
|
||||
* [Graphite API usage](#graphite-api-usage)
|
||||
* [Graphite Metrics API usage](#graphite-metrics-api-usage)
|
||||
* [Graphite Tags API usage](#graphite-tags-api-usage)
|
||||
* [How to build from sources](#how-to-build-from-sources)
|
||||
* [Development build](#development-build)
|
||||
* [Production build](#production-build)
|
||||
@@ -157,7 +163,7 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
|
||||
* [We kindly ask](#we-kindly-ask)
|
||||
|
||||
|
||||
### How to start VictoriaMetrics
|
||||
## How to start VictoriaMetrics
|
||||
|
||||
Start VictoriaMetrics [executable](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
|
||||
or [docker image](https://hub.docker.com/r/victoriametrics/victoria-metrics/) with the desired command-line flags.
|
||||
@@ -174,7 +180,7 @@ VictoriaMetrics accepts [Prometheus querying API requests](#prometheus-querying-
|
||||
|
||||
It is recommended setting up [monitoring](#monitoring) for VictoriaMetrics.
|
||||
|
||||
#### Environment variables
|
||||
### Environment variables
|
||||
|
||||
Each flag value can be set via environment variables according to these rules:
|
||||
|
||||
@@ -184,7 +190,7 @@ Each flag value can be set via environment variables according to these rules:
|
||||
* It is possible setting prefix for environment vars with `-envflag.prefix`. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_`
|
||||
|
||||
|
||||
### Prometheus setup
|
||||
## Prometheus setup
|
||||
|
||||
Prometheus must be configured with [remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
|
||||
in order to send data to VictoriaMetrics. Add the following lines
|
||||
@@ -240,7 +246,7 @@ Take a look also at [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics
|
||||
which can be used as faster and less resource-hungry alternative to Prometheus in certain cases.
|
||||
|
||||
|
||||
### Grafana setup
|
||||
## Grafana setup
|
||||
|
||||
Create [Prometheus datasource](http://docs.grafana.org/features/datasources/prometheus/) in Grafana with the following url:
|
||||
|
||||
@@ -255,7 +261,7 @@ or [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL
|
||||
which is used by Grafana.
|
||||
|
||||
|
||||
### How to upgrade VictoriaMetrics
|
||||
## How to upgrade VictoriaMetrics
|
||||
|
||||
It is safe upgrading VictoriaMetrics to new versions unless [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
|
||||
say otherwise. It is safe skipping multiple versions during the upgrade unless [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) say otherwise.
|
||||
@@ -273,7 +279,7 @@ Prometheus doesn't drop data during VictoriaMetrics restart.
|
||||
See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details.
|
||||
|
||||
|
||||
### How to apply new config to VictoriaMetrics
|
||||
## How to apply new config to VictoriaMetrics
|
||||
|
||||
VictoriaMetrics is configured via command-line flags, so it must be restarted when new command-line flags should be applied:
|
||||
|
||||
@@ -285,7 +291,7 @@ Prometheus doesn't drop data during VictoriaMetrics restart.
|
||||
See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details.
|
||||
|
||||
|
||||
### How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter)
|
||||
## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter)
|
||||
|
||||
VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file).
|
||||
Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets.
|
||||
@@ -300,6 +306,8 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la
|
||||
* [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
|
||||
* [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config)
|
||||
* [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)
|
||||
* [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)
|
||||
|
||||
|
||||
Other `*_sd_config` types will be supported in the future.
|
||||
|
||||
@@ -310,7 +318,7 @@ VictoriaMetrics also supports [importing data in Prometheus exposition format](#
|
||||
See also [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md), which can be used as drop-in replacement for Prometheus.
|
||||
|
||||
|
||||
### How to send data from InfluxDB-compatible agents such as [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)
|
||||
## How to send data from InfluxDB-compatible agents such as [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)
|
||||
|
||||
Use `http://<victoriametric-addr>:8428` url instead of InfluxDB url in agents' configs.
|
||||
For instance, put the following lines into `Telegraf` config, so it sends data to VictoriaMetrics instead of InfluxDB:
|
||||
@@ -372,7 +380,7 @@ Note that Influx line protocol expects [timestamps in *nanoseconds* by default](
|
||||
while VictoriaMetrics stores them with *milliseconds* precision.
|
||||
|
||||
|
||||
### How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd)
|
||||
## How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd)
|
||||
|
||||
Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command line flag. For instance,
|
||||
the following command will enable Graphite receiver in VictoriaMetrics on TCP and UDP port `2003`:
|
||||
@@ -404,21 +412,22 @@ The `/api/v1/export` endpoint should return the following response:
|
||||
{"metric":{"__name__":"foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560277406000]}
|
||||
```
|
||||
|
||||
### Querying Graphite data
|
||||
## Querying Graphite data
|
||||
|
||||
Data sent to VictoriaMetrics via `Graphite plaintext protocol` may be read via the following APIs:
|
||||
|
||||
* [Prometheus querying API](#prometheus-querying-api-usage)
|
||||
* Metric names can be explored via [Graphite metrics API](#graphite-metrics-api-usage)
|
||||
* Tags can be explored via [Graphite tags API](#graphite-tags-api-usage)
|
||||
* [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml)
|
||||
|
||||
### How to send data from OpenTSDB-compatible agents
|
||||
## How to send data from OpenTSDB-compatible agents
|
||||
|
||||
VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html)
|
||||
and [HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html) for ingesting OpenTSDB data.
|
||||
The same protocol is used for [ingesting data in KairosDB](https://kairosdb.github.io/docs/build/html/PushingData.html).
|
||||
|
||||
#### Sending data via `telnet put` protocol
|
||||
### Sending data via `telnet put` protocol
|
||||
|
||||
Enable OpenTSDB receiver in VictoriaMetrics by setting `-opentsdbListenAddr` command line flag. For instance,
|
||||
the following command enables OpenTSDB receiver in VictoriaMetrics on TCP and UDP port `4242`:
|
||||
@@ -448,7 +457,7 @@ The `/api/v1/export` endpoint should return the following response:
|
||||
{"metric":{"__name__":"foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560277292000]}
|
||||
```
|
||||
|
||||
#### Sending OpenTSDB data via HTTP `/api/put` requests
|
||||
### Sending OpenTSDB data via HTTP `/api/put` requests
|
||||
|
||||
Enable HTTP server for OpenTSDB `/api/put` requests by setting `-opentsdbHTTPListenAddr` command line flag. For instance,
|
||||
the following command enables OpenTSDB HTTP server on port `4242`:
|
||||
@@ -486,7 +495,7 @@ The `/api/v1/export` endpoint should return the following response:
|
||||
```
|
||||
|
||||
|
||||
### Prometheus querying API usage
|
||||
## Prometheus querying API usage
|
||||
|
||||
VictoriaMetrics supports the following handlers from [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/):
|
||||
|
||||
@@ -495,12 +504,14 @@ VictoriaMetrics supports the following handlers from [Prometheus querying API](h
|
||||
* [/api/v1/series](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers)
|
||||
* [/api/v1/labels](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names)
|
||||
* [/api/v1/label/.../values](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values)
|
||||
* [/api/v1/status/tsdb](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats)
|
||||
* [/api/v1/status/tsdb](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats). VictoriaMetrics accepts optional `topN=N` and `date=YYYY-MM-DD`
|
||||
query args for this handler, where `N` is the number of top entries to return in the response and `YYYY-MM-DD` is the date for collecting the stats.
|
||||
By default top 10 entries are returned and the stats is collected for the current day.
|
||||
* [/api/v1/targets](https://prometheus.io/docs/prometheus/latest/querying/api/#targets) - see [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter) for more details.
|
||||
|
||||
These handlers can be queried from Prometheus-compatible clients such as Grafana or curl.
|
||||
|
||||
#### Prometheus querying API enhancements
|
||||
### Prometheus querying API enhancements
|
||||
|
||||
Additionally to unix timestamps and [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) VictoriaMetrics accepts relative times in `time`, `start` and `end` query args.
|
||||
For example, the following query would return data for the last 30 minutes: `/api/v1/query_range?start=-30m&query=...`.
|
||||
@@ -522,6 +533,14 @@ Additionally VictoriaMetrics provides the following handlers:
|
||||
* `/api/v1/status/active_queries` - it returns a list of currently running queries.
|
||||
|
||||
|
||||
## Graphite API usage
|
||||
|
||||
VictoriaMetrics supports the following Graphite APIs:
|
||||
|
||||
* Metrics API - see [these docs](#graphite-metrics-api-usage).
|
||||
* Tags API - see [these docs](#graphite-tags-api-usage).
|
||||
|
||||
|
||||
### Graphite Metrics API usage
|
||||
|
||||
VictoriaMetrics supports the following handlers from [Graphite Metrics API](https://graphite-api.readthedocs.io/en/latest/api.html#the-metrics-api):
|
||||
@@ -536,42 +555,56 @@ VictoriaMetrics accepts the following additional query args at `/metrics/find` a
|
||||
that start with `node_`. By default `delimiter=.`.
|
||||
|
||||
|
||||
### How to build from sources
|
||||
### Graphite Tags API usage
|
||||
|
||||
VictoriaMetrics supports the following handlers from [Graphite Tags API](https://graphite.readthedocs.io/en/stable/tags.html):
|
||||
|
||||
* [/tags/tagSeries](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb)
|
||||
* [/tags/tagMultiSeries](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb)
|
||||
* [/tags](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags)
|
||||
* [/tags/{tag_name}](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags)
|
||||
* [/tags/findSeries](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags)
|
||||
* [/tags/autoComplete/tags](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support)
|
||||
* [/tags/autoComplete/values](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support)
|
||||
* [/tags/delSeries](https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb)
|
||||
|
||||
|
||||
## How to build from sources
|
||||
|
||||
We recommend using either [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or
|
||||
[docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/) instead of building VictoriaMetrics
|
||||
from sources. Building from sources is reasonable when developing additional features specific
|
||||
to your needs or when testing bugfixes.
|
||||
|
||||
#### Development build
|
||||
### Development build
|
||||
|
||||
1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
|
||||
2. Run `make victoria-metrics` from the root folder of the repository.
|
||||
It builds `victoria-metrics` binary and puts it into the `bin` folder.
|
||||
|
||||
#### Production build
|
||||
### Production build
|
||||
|
||||
1. [Install docker](https://docs.docker.com/install/).
|
||||
2. Run `make victoria-metrics-prod` from the root folder of the repository.
|
||||
It builds `victoria-metrics-prod` binary and puts it into the `bin` folder.
|
||||
|
||||
#### ARM build
|
||||
### ARM build
|
||||
|
||||
ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://blog.cloudflare.com/arm-takes-wing/).
|
||||
|
||||
#### Development ARM build
|
||||
### Development ARM build
|
||||
|
||||
1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
|
||||
2. Run `make victoria-metrics-arm` or `make victoria-metrics-arm64` from the root folder of the repository.
|
||||
It builds `victoria-metrics-arm` or `victoria-metrics-arm64` binary respectively and puts it into the `bin` folder.
|
||||
|
||||
#### Production ARM build
|
||||
### Production ARM build
|
||||
|
||||
1. [Install docker](https://docs.docker.com/install/).
|
||||
2. Run `make victoria-metrics-arm-prod` or `make victoria-metrics-arm64-prod` from the root folder of the repository.
|
||||
It builds `victoria-metrics-arm-prod` or `victoria-metrics-arm64-prod` binary respectively and puts it into the `bin` folder.
|
||||
|
||||
#### Pure Go build (CGO_ENABLED=0)
|
||||
### Pure Go build (CGO_ENABLED=0)
|
||||
|
||||
`Pure Go` mode builds only Go code without [cgo](https://golang.org/cmd/cgo/) dependencies.
|
||||
This is an experimental mode, which may result in a lower compression ratio and slower decompression performance.
|
||||
@@ -581,7 +614,7 @@ Use it with caution!
|
||||
2. Run `make victoria-metrics-pure` from the root folder of the repository.
|
||||
It builds `victoria-metrics-pure` binary and puts it into the `bin` folder.
|
||||
|
||||
#### Building docker images
|
||||
### Building docker images
|
||||
|
||||
Run `make package-victoria-metrics`. It builds `victoriametrics/victoria-metrics:<PKG_TAG>` docker image locally.
|
||||
`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
|
||||
@@ -595,17 +628,20 @@ For example, the following command builds the image on top of [scratch](https://
|
||||
ROOT_IMAGE=scratch make package-victoria-metrics
|
||||
```
|
||||
|
||||
### Start with docker-compose
|
||||
## Start with docker-compose
|
||||
|
||||
[Docker-compose](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/docker-compose.yml)
|
||||
helps to spin up VictoriaMetrics, [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md) and Grafana with one command.
|
||||
More details may be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#folder-contains-basic-images-and-tools-for-building-and-running-victoria-metrics-in-docker).
|
||||
|
||||
### Setting up service
|
||||
|
||||
## Setting up service
|
||||
|
||||
Read [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/43) on how to set up VictoriaMetrics as a service in your OS.
|
||||
There is also [snap package for Ubuntu](https://snapcraft.io/victoriametrics).
|
||||
|
||||
### How to work with snapshots
|
||||
|
||||
## How to work with snapshots
|
||||
|
||||
VictoriaMetrics can create [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
|
||||
for all the data stored under `-storageDataPath` directory.
|
||||
@@ -634,7 +670,7 @@ Steps for restoring from a snapshot:
|
||||
to the directory pointed by `-storageDataPath`.
|
||||
3. Start VictoriaMetrics.
|
||||
|
||||
### How to delete time series
|
||||
## How to delete time series
|
||||
|
||||
Send a request to `http://<victoriametrics-addr>:8428/api/v1/admin/tsdb/delete_series?match[]=<timeseries_selector_for_delete>`,
|
||||
where `<timeseries_selector_for_delete>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
|
||||
@@ -666,7 +702,7 @@ It isn't recommended using delete API for the following cases, since it brings n
|
||||
It is better using `-retentionPeriod` command-line flag for efficient pruning of old data.
|
||||
|
||||
|
||||
### Forced merge
|
||||
## Forced merge
|
||||
|
||||
VictoriaMetrics performs [data compactions in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
|
||||
in order to keep good performance characteristics when accepting new data. These compactions (merges) are performed independently on per-month partitions.
|
||||
@@ -681,7 +717,7 @@ since VictoriaMetrics automatically performs [optimal merges in background](http
|
||||
when new data is ingested into it.
|
||||
|
||||
|
||||
### How to export time series
|
||||
## How to export time series
|
||||
|
||||
VictoriaMetrics provides the following handlers for exporting data:
|
||||
|
||||
@@ -691,11 +727,20 @@ VictoriaMetrics provides the following handlers for exporting data:
|
||||
* `/api/v1/export/csv` for exporting data in CSV. See [these docs](#how-to-export-csv-data) for details.
|
||||
|
||||
|
||||
#### How to export data in native format
|
||||
### How to export data in native format
|
||||
|
||||
Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/native?match[]=<timeseries_selector_for_export>`,
|
||||
where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
|
||||
for metrics to export. Use `{__name__!=""}` selector for fetching all the time series.
|
||||
for metrics to export. Use `{__name__=~".*"}` selector for fetching all the time series.
|
||||
|
||||
On large databases you may experience problems with limit on unique timeseries (default value is 300000). In this case you need to adjust `-search.maxUniqueTimeseries` parameter:
|
||||
|
||||
```bash
|
||||
# count unique timeseries in database
|
||||
wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq '.data[0]'
|
||||
|
||||
# relaunch victoriametrics with search.maxUniqueTimeseries more than value from previous command
|
||||
```
|
||||
|
||||
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
|
||||
unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
|
||||
@@ -703,7 +748,7 @@ unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) val
|
||||
The exported data can be imported to VictoriaMetrics via [/api/v1/import/native](#how-to-import-data-in-native-format).
|
||||
|
||||
|
||||
#### How to export data in JSON line format
|
||||
### How to export data in JSON line format
|
||||
|
||||
Consider [exporting data in native format](#how-to-export-data-in-native-format) if big amounts of data must be migrated between VictoriaMetrics instances,
|
||||
since exporting in native format usually consumes lower amounts of CPU and memory resources, while the resulting exported data occupies lower amounts of disk space.
|
||||
@@ -738,7 +783,7 @@ The maximum duration for each request to `/api/v1/export` is limited by `-search
|
||||
Exported data can be imported via POST'ing it to [/api/v1/import](#how-to-import-data-in-json-line-format).
|
||||
|
||||
|
||||
#### How to export CSV data
|
||||
### How to export CSV data
|
||||
|
||||
Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/csv?format=<format>&match=<timeseries_selector_for_export>`,
|
||||
where:
|
||||
@@ -762,7 +807,7 @@ unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) val
|
||||
The exported CSV data can be imported to VictoriaMetrics via [/api/v1/import/csv](#how-to-import-csv-data).
|
||||
|
||||
|
||||
### How to import time series data
|
||||
## How to import time series data
|
||||
|
||||
Time series data can be imported via any supported ingestion protocol:
|
||||
|
||||
@@ -772,14 +817,14 @@ Time series data can be imported via any supported ingestion protocol:
|
||||
* OpenTSDB telnet put protocol. See [these docs](#sending-data-via-telnet-put-protocol) for details.
|
||||
* OpenTSDB http `/api/put` protocol. See [these docs](#sending-opentsdb-data-via-http-apiput-requests) for details.
|
||||
* `/api/v1/import` for importing data obtained from [/api/v1/export](#how-to-export-data-in-json-line-format).
|
||||
See [these docs](##how-to-import-data-in-json-line-format) for details.
|
||||
See [these docs](#how-to-import-data-in-json-line-format) for details.
|
||||
* `/api/v1/import/native` for importing data obtained from [/api/v1/export/native](#how-to-export-data-in-native-format).
|
||||
See [these docs](#how-to-import-data-in-native-format) for details.
|
||||
* `/api/v1/import/csv` for importing arbitrary CSV data. See [these docs](#how-to-import-csv-data) for details.
|
||||
* `/api/v1/import/prometheus` for importing data in Prometheus exposition format. See [these docs](#how-to-import-data-in-prometheus-exposition-format) for details.
|
||||
|
||||
|
||||
#### How to import data in native format
|
||||
### How to import data in native format
|
||||
|
||||
The most efficient protocol for importing data into VictoriaMetrics is `/api/v1/import/native`.
|
||||
Example for importing data obtained via [/api/v1/export/native](#how-to-export-data-in-native-format):
|
||||
@@ -808,7 +853,7 @@ For example, `/api/v1/import/native?extra_label=foo=bar` would add `"foo":"bar"`
|
||||
Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
|
||||
|
||||
|
||||
#### How to import data in JSON line format
|
||||
### How to import data in JSON line format
|
||||
|
||||
Example for importing data obtained via [/api/v1/export](#how-to-export-data-in-json-line-format):
|
||||
|
||||
@@ -836,7 +881,7 @@ For example, `/api/v1/import?extra_label=foo=bar` would add `"foo":"bar"` label
|
||||
Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
|
||||
|
||||
|
||||
#### How to import CSV data
|
||||
### How to import CSV data
|
||||
|
||||
Arbitrary CSV data can be imported via `/api/v1/import/csv`. The CSV data is imported according to the provided `format` query arg.
|
||||
The `format` query arg must contain comma-separated list of parsing rules for CSV fields. Each rule consists of three parts delimited by a colon:
|
||||
@@ -889,7 +934,7 @@ For example, `/api/v1/import/csv?extra_label=foo=bar` would add `"foo":"bar"` la
|
||||
Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
|
||||
|
||||
|
||||
#### How to import data in Prometheus exposition format
|
||||
### How to import data in Prometheus exposition format
|
||||
|
||||
VictoriaMetrics accepts data in [Prometheus exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format)
|
||||
via `/api/v1/import/prometheus` path. For example, the following line imports a single line in Prometheus exposition format into VictoriaMetrics:
|
||||
@@ -924,7 +969,7 @@ VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-sc
|
||||
|
||||
|
||||
|
||||
### Relabeling
|
||||
## Relabeling
|
||||
|
||||
VictoriaMetrics supports Prometheus-compatible relabeling for all the ingested metrics if `-relabelConfig` command-line flag points
|
||||
to a file containing a list of [relabel_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) entries.
|
||||
@@ -948,7 +993,7 @@ VictoriaMetrics provides the following extra actions for relabeling rules:
|
||||
See also [relabeling in vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md#relabeling).
|
||||
|
||||
|
||||
### Federation
|
||||
## Federation
|
||||
|
||||
VictoriaMetrics exports [Prometheus-compatible federation data](https://prometheus.io/docs/prometheus/latest/federation/)
|
||||
at `http://<victoriametrics-addr>:8428/federate?match[]=<timeseries_selector_for_federation>`.
|
||||
@@ -959,7 +1004,7 @@ on the interval `[now - max_lookback ... now]` is scraped for each time series.
|
||||
For instance, `/federate?match[]=up&max_lookback=1h` would return last points on the `[now - 1h ... now]` interval. This may be useful for time series federation
|
||||
with scrape intervals exceeding `5m`.
|
||||
|
||||
### Capacity planning
|
||||
## Capacity planning
|
||||
|
||||
A rough estimation of the required resources for ingestion path:
|
||||
|
||||
@@ -999,7 +1044,8 @@ The required resources for query path:
|
||||
* Network usage: depends on the frequency and the type of incoming requests. Typical Grafana dashboards usually
|
||||
require negligible network bandwidth.
|
||||
|
||||
### High availability
|
||||
|
||||
## High availability
|
||||
|
||||
* Install multiple VictoriaMetrics instances in distinct datacenters (availability zones).
|
||||
* Pass addresses of these instances to [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md) via `-remoteWrite.url` command-line flag:
|
||||
@@ -1040,7 +1086,7 @@ Another option is to write data simultaneously from Prometheus HA pair to a pair
|
||||
with the enabled de-duplication. See [this section](#deduplication) for details.
|
||||
|
||||
|
||||
### Deduplication
|
||||
## Deduplication
|
||||
|
||||
VictoriaMetrics de-duplicates data points if `-dedup.minScrapeInterval` command-line flag
|
||||
is set to positive duration. For example, `-dedup.minScrapeInterval=60s` would de-duplicate data points
|
||||
@@ -1051,7 +1097,7 @@ write data to the same VictoriaMetrics instance. Note that these Prometheus inst
|
||||
`external_labels` section in their configs, so they write data to the same time series.
|
||||
|
||||
|
||||
### Retention
|
||||
## Retention
|
||||
|
||||
Retention is configured with `-retentionPeriod` command-line flag. For instance, `-retentionPeriod=3` means
|
||||
that the data will be stored for 3 months and then deleted.
|
||||
@@ -1066,7 +1112,7 @@ VictoriaMetrics supports retention smaller than 1 month. For example, `-retentio
|
||||
Older data is eventually deleted during [background merge](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
|
||||
|
||||
|
||||
### Multiple retentions
|
||||
## Multiple retentions
|
||||
|
||||
Just start multiple VictoriaMetrics instances with distinct values for the following flags:
|
||||
|
||||
@@ -1079,7 +1125,7 @@ so it could route requests from particular user to VictoriaMetrics with the desi
|
||||
The same scheme could be implemented for multiple tenants in [VictoriaMetrics cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md).
|
||||
|
||||
|
||||
### Downsampling
|
||||
## Downsampling
|
||||
|
||||
There is no downsampling support at the moment, but:
|
||||
|
||||
@@ -1095,11 +1141,13 @@ It is possible to (ab)use [-dedup.minScrapeInterval](#deduplication) for basic d
|
||||
For instance, if interval between the ingested data points is 15s, then `-dedup.minScrapeInterval=5m` will leave
|
||||
only a single data point out of 20 initial data points per each 5m interval.
|
||||
|
||||
### Multi-tenancy
|
||||
|
||||
## Multi-tenancy
|
||||
|
||||
Single-node VictoriaMetrics doesn't support multi-tenancy. Use [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster) instead.
|
||||
|
||||
### Scalability and cluster version
|
||||
|
||||
## Scalability and cluster version
|
||||
|
||||
Though single-node VictoriaMetrics cannot scale to multiple nodes, it is optimized for resource usage - storage size / bandwidth / IOPS, RAM, CPU.
|
||||
This means that a single-node VictoriaMetrics may scale vertically and substitute a moderately sized cluster built with competing solutions
|
||||
@@ -1109,7 +1157,8 @@ So try single-node VictoriaMetrics at first and then [switch to cluster version]
|
||||
horizontally scalable long-term remote storage for really large Prometheus deployments.
|
||||
[Contact us](mailto:info@victoriametrics.com) for paid support.
|
||||
|
||||
### Alerting
|
||||
|
||||
## Alerting
|
||||
|
||||
It is recommended using [vmalert](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md) for alerting.
|
||||
|
||||
@@ -1120,7 +1169,7 @@ Additionally, alerting can be set up with the following tools:
|
||||
* With Grafana - see [the corresponding docs](https://grafana.com/docs/alerting/rules/).
|
||||
|
||||
|
||||
### Security
|
||||
## Security
|
||||
|
||||
Do not forget protecting sensitive endpoints in VictoriaMetrics when exposing it to untrusted networks such as the internet.
|
||||
Consider setting the following command-line flags:
|
||||
@@ -1140,7 +1189,7 @@ Prefer authorizing all the incoming requests from untrusted networks with [vmaut
|
||||
or similar auth proxy.
|
||||
|
||||
|
||||
### Tuning
|
||||
## Tuning
|
||||
|
||||
* There is no need for VictoriaMetrics tuning since it uses reasonable defaults for command-line flags,
|
||||
which are automatically adjusted for the available CPU and RAM resources.
|
||||
@@ -1156,7 +1205,7 @@ or similar auth proxy.
|
||||
mkfs.ext4 ... -O 64bit,huge_file,extent -T huge
|
||||
```
|
||||
|
||||
### Monitoring
|
||||
## Monitoring
|
||||
|
||||
VictoriaMetrics exports internal metrics in Prometheus format at `/metrics` page.
|
||||
These metrics may be collected by [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md)
|
||||
@@ -1186,7 +1235,7 @@ The most interesting metrics are:
|
||||
VictoriaMetrics also exposes currently running queries with their execution times at `/api/v1/status/active_queries` page.
|
||||
|
||||
|
||||
### Troubleshooting
|
||||
## Troubleshooting
|
||||
|
||||
* It is recommended to use default command-line flag values (i.e. don't set them explicitly) until the need
|
||||
of tweaking these flag values arises.
|
||||
@@ -1196,13 +1245,18 @@ VictoriaMetrics also exposes currently running queries with their execution time
|
||||
|
||||
* It is recommended inspecting logs during troubleshooting, since they may contain useful information.
|
||||
|
||||
* VictoriaMetrics buffers incoming data in memory for up to a few seconds before flushing it to persistent storage.
|
||||
This may lead to the following "issues":
|
||||
* Data becomes available for querying in a few seconds after inserting. It is possible to flush in-memory buffers to persistent storage
|
||||
by requesting `/internal/force_flush` http handler. This handler is mostly needed for testing and debugging purposes.
|
||||
* The last few seconds of inserted data may be lost on unclean shutdown (i.e. OOM, `kill -9` or hardware reset).
|
||||
See [this article for technical details](https://valyala.medium.com/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704).
|
||||
|
||||
* If VictoriaMetrics works slowly and eats more than a CPU core per 100K ingested data points per second,
|
||||
then it is likely you have too many active time series for the current amount of RAM.
|
||||
VictoriaMetrics [exposes](#monitoring) `vm_slow_*` metrics, which could be used as an indicator of low amounts of RAM.
|
||||
It is recommended increasing the amount of RAM on the node with VictoriaMetrics in order to improve
|
||||
ingestion and query performance in this case.
|
||||
Another option is to increase `-memory.allowedPercent` command-line flag value. Be careful with this
|
||||
option, since too big value for `-memory.allowedPercent` may result in high I/O usage.
|
||||
|
||||
* VictoriaMetrics prioritizes data ingestion over data querying. So if it has no enough resources for data ingestion,
|
||||
then data querying may slow down significantly.
|
||||
@@ -1217,9 +1271,9 @@ VictoriaMetrics also exposes currently running queries with their execution time
|
||||
which would start background merge if they had more free disk space.
|
||||
|
||||
* If VictoriaMetrics doesn't work because of certain parts are corrupted due to disk errors,
|
||||
then just remove directories with broken parts. This will recover VictoriaMetrics at the cost
|
||||
of data loss stored in the broken parts. In the future, `vmrecover` tool will be created
|
||||
for automatic recovering from such errors.
|
||||
then just remove directories with broken parts. It is safe removing subdirectories under `<-storageDataPath>/data/{big,small}/YYYY_MM` directories
|
||||
when VictoriaMetrics isn't running. This recovers VictoriaMetrics at the cost of data loss stored in the deleted broken parts.
|
||||
In the future, `vmrecover` tool will be created for automatic recovering from such errors.
|
||||
|
||||
* If you see gaps on the graphs, try resetting the cache by sending request to `/internal/resetRollupResultCache`.
|
||||
If this removes gaps on the graphs, then it is likely data with timestamps older than `-search.cacheTimestampOffset`
|
||||
@@ -1241,10 +1295,15 @@ VictoriaMetrics also exposes currently running queries with their execution time
|
||||
This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
|
||||
metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
|
||||
|
||||
* If you store Graphite metrics like `foo.bar.baz` in VictoriaMetrics, then `-search.treatDotsAsIsInRegexps` command-line flag could be useful.
|
||||
By default `.` chars in regexps match any char. If you need matching only dots, then the `\\.` must be used in regexp filters.
|
||||
When `-search.treatDotsAsIsInRegexps` option is enabled, then dots in regexps are automatically escaped in order to match only dots instead of arbitrary chars.
|
||||
This may significantly increase performance when locating time series for the given label filters.
|
||||
|
||||
* VictoriaMetrics ignores `NaN` values during data ingestion.
|
||||
|
||||
|
||||
### Backfilling
|
||||
## Backfilling
|
||||
|
||||
VictoriaMetrics accepts historical data in arbitrary order of time via [any supported ingestion method](#how-to-import-time-series-data).
|
||||
Make sure that configured `-retentionPeriod` covers timestamps for the backfilled data.
|
||||
@@ -1260,7 +1319,7 @@ Yet another solution is to increase `-search.cacheTimestampOffset` flag value in
|
||||
for data with timestamps close to the current time.
|
||||
|
||||
|
||||
### Data updates
|
||||
## Data updates
|
||||
|
||||
VictoriaMetrics doesn't support updating already existing sample values to new ones. It stores all the ingested data points
|
||||
for the same time series with identical timestamps. While is possible substituting old time series with new time series via
|
||||
@@ -1268,7 +1327,7 @@ for the same time series with identical timestamps. While is possible substituti
|
||||
should be used only for one-off updates. It shouldn't be used for frequent updates because of non-zero overhead related to data removal.
|
||||
|
||||
|
||||
### Replication
|
||||
## Replication
|
||||
|
||||
Single-node VictoriaMetrics doesn't support application-level replication. Use cluster version instead.
|
||||
See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#replication-and-data-safety) for details.
|
||||
@@ -1278,14 +1337,14 @@ Storage-level replication may be offloaded to durable persistent storage such as
|
||||
See also [high availability docs](#high-availability) and [backup docs](#backups).
|
||||
|
||||
|
||||
### Backups
|
||||
## Backups
|
||||
|
||||
VictoriaMetrics supports backups via [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md)
|
||||
and [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md) tools.
|
||||
We also provide provide `vmbackuper` tool for paid enterprise subscribers - see [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/466) for details.
|
||||
|
||||
|
||||
### Profiling
|
||||
## Profiling
|
||||
|
||||
VictoriaMetrics provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
|
||||
|
||||
@@ -1305,6 +1364,7 @@ The command for collecting CPU profile waits for 30 seconds before returning.
|
||||
|
||||
The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
|
||||
|
||||
|
||||
## Integrations
|
||||
|
||||
* [Helm charts for single-node and cluster versions of VictoriaMetrics](https://github.com/VictoriaMetrics/helm-charts).
|
||||
@@ -1316,6 +1376,8 @@ The collected profiles may be analyzed with [go tool pprof](https://github.com/g
|
||||
See [this example](https://github.com/go-graphite/carbonapi/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml).
|
||||
* [Ansible role for installing single-node VictoriaMetrics](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
|
||||
* [Ansible role for installing cluster VictoriaMetrics](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
|
||||
* [Snap package for VictoriaMetrics](https://snapcraft.io/victoriametrics).
|
||||
|
||||
|
||||
## Third-party contributions
|
||||
|
||||
@@ -1324,10 +1386,12 @@ The collected profiles may be analyzed with [go tool pprof](https://github.com/g
|
||||
* [Prometheus -> VictoriaMetrics exporter #2](https://github.com/AnchorFree/tsdb-remote-write)
|
||||
* [Prometheus Oauth proxy](https://gitlab.com/optima_public/prometheus_oauth_proxy) - see [this article](https://medium.com/@richard.holly/powerful-saas-solution-for-detection-metrics-c67b9208d362) for details.
|
||||
|
||||
|
||||
## Contacts
|
||||
|
||||
Contact us with any questions regarding VictoriaMetrics at [info@victoriametrics.com](mailto:info@victoriametrics.com).
|
||||
|
||||
|
||||
## Community and contributions
|
||||
|
||||
Feel free asking any questions regarding VictoriaMetrics:
|
||||
|
||||
@@ -63,6 +63,22 @@ Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://gi
|
||||
Pass `-help` to `vmagent` in order to see the full list of supported command-line flags with their descriptions.
|
||||
|
||||
|
||||
### Configuration update
|
||||
|
||||
`vmagent` should be restarted in order to update config options set via command-line args.
|
||||
|
||||
`vmagent` supports multiple approaches for reloading configs from updated config files such as `-promscrape.config`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`:
|
||||
|
||||
* Sending `SUGHUP` signal to `vmagent` process:
|
||||
```bash
|
||||
kill -SIGHUP `pidof vmagent`
|
||||
```
|
||||
|
||||
* Sending HTTP request to `http://vmagent:8429/-/reload` endpoint.
|
||||
|
||||
There is also `-promscrape.configCheckInterval` command-line option, which can be used for automatic reloading configs from updated `-promscrape.config` file.
|
||||
|
||||
|
||||
### Use cases
|
||||
|
||||
|
||||
@@ -153,6 +169,8 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
|
||||
[OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
|
||||
* `dockerswarm_sd_configs` - for scraping Docker Swarm targets.
|
||||
See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
|
||||
* `eureka_sd_configs` - for scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka).
|
||||
See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details.
|
||||
|
||||
File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
|
||||
|
||||
@@ -197,6 +215,7 @@ The relabeling can be defined in the following places:
|
||||
|
||||
Read more about relabeling in the following articles:
|
||||
|
||||
* [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
|
||||
* [Life of a label](https://www.robustperception.io/life-of-a-label)
|
||||
* [Discarding targets and timeseries with relabeling](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts)
|
||||
* [Dropping labels at scrape time](https://www.robustperception.io/dropping-metrics-at-scrape-time-with-prometheus)
|
||||
@@ -237,6 +256,9 @@ It may be useful for performing `vmagent` rolling update without scrape loss.
|
||||
* The `/api/v1/targets` page could be useful for debugging relabeling process for scrape targets.
|
||||
This page contains original labels for targets dropped during relabeling (see "droppedTargets" section in the page output). By default up to `-promscrape.maxDroppedTargets` targets are shown here. If your setup drops more targets during relabeling, then increase `-promscrape.maxDroppedTargets` command-line flag value in order to see all the dropped targets. Note that tracking each dropped target requires up to 10Kb of RAM, so big values for `-promscrape.maxDroppedTargets` may result in increased memory usage if big number of scrape targets are dropped during relabeling.
|
||||
|
||||
* If `vmagent` scrapes big number of targets, then `-promscrape.dropOriginalLabels` command-line option may be passed to `vmagent` in order to reduce memory usage.
|
||||
This option drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page, which may result in reduced debuggability for improperly configured per-target relabeling.
|
||||
|
||||
* If `vmagent` scrapes targets with millions of metrics per each target (for instance, when scraping [federation endpoints](https://prometheus.io/docs/prometheus/latest/federation/)),
|
||||
then it is recommended enabling `stream parsing mode` in order to reduce memory usage during scraping. This mode may be enabled either globally for all the scrape targets
|
||||
by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example:
|
||||
|
||||
20
go.mod
20
go.mod
@@ -1,7 +1,7 @@
|
||||
module github.com/VictoriaMetrics/VictoriaMetrics
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.71.0 // indirect
|
||||
cloud.google.com/go v0.72.0 // indirect
|
||||
cloud.google.com/go/storage v1.12.0
|
||||
github.com/VictoriaMetrics/fastcache v1.5.7
|
||||
|
||||
@@ -10,25 +10,23 @@ require (
|
||||
github.com/VictoriaMetrics/fasthttp v1.0.7
|
||||
github.com/VictoriaMetrics/metrics v1.12.3
|
||||
github.com/VictoriaMetrics/metricsql v0.7.2
|
||||
github.com/aws/aws-sdk-go v1.35.22
|
||||
github.com/aws/aws-sdk-go v1.35.31
|
||||
github.com/cespare/xxhash/v2 v2.1.1
|
||||
github.com/golang/snappy v0.0.2
|
||||
github.com/klauspost/compress v1.11.2
|
||||
github.com/klauspost/compress v1.11.3
|
||||
github.com/stretchr/testify v1.5.1 // indirect
|
||||
github.com/valyala/fastjson v1.6.1
|
||||
github.com/valyala/fastjson v1.6.3
|
||||
github.com/valyala/fastrand v1.0.0
|
||||
github.com/valyala/fasttemplate v1.2.1
|
||||
github.com/valyala/gozstd v1.8.3
|
||||
github.com/valyala/histogram v1.1.2
|
||||
github.com/valyala/quicktemplate v1.6.3
|
||||
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
|
||||
golang.org/x/sys v0.0.0-20201106081118-db71ae66460a
|
||||
golang.org/x/text v0.3.4 // indirect
|
||||
golang.org/x/tools v0.0.0-20201105220310-78b158585360 // indirect
|
||||
google.golang.org/api v0.34.0
|
||||
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68
|
||||
golang.org/x/tools v0.0.0-20201119132711-4783bc9bebf0 // indirect
|
||||
google.golang.org/api v0.35.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20201105153401-9d023cd09d72 // indirect
|
||||
google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4 // indirect
|
||||
gopkg.in/yaml.v2 v2.3.0
|
||||
)
|
||||
|
||||
|
||||
83
go.sum
83
go.sum
@@ -9,55 +9,39 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T
|
||||
cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
|
||||
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
|
||||
cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
|
||||
cloud.google.com/go v0.56.0 h1:WRz29PgAsVEyPSDHyk+0fpEkwEFyfhHn+JbksT6gIL4=
|
||||
cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
|
||||
cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms=
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0 h1:RmDygqvj27Zf3fCQjQRtLyC7KwFcHkeJitcO0OoGOcA=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go v0.66.0 h1:DZeAkuQGQqnm9Xv36SbMJEU8aFBz4wL04UpMWPWwjzg=
|
||||
cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko=
|
||||
cloud.google.com/go v0.71.0 h1:2ha722Z08cmRa0orJrzBaszYQcLbLFcsZHsGSj/kIF4=
|
||||
cloud.google.com/go v0.71.0/go.mod h1:qZfY4Y7AEIQwG/fQYD3xrxLNkQZ0Xzf3HGeqCkA6LVM=
|
||||
cloud.google.com/go v0.72.0 h1:eWRCuwubtDrCJG0oSUMgnsbD4CmPFQF2ei4OFbXvwww=
|
||||
cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU=
|
||||
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
|
||||
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
|
||||
cloud.google.com/go/bigquery v1.7.0 h1:a/O/bK/vWrYGOTFtH8di4rBxMZnmkjy+Y5LxpDwo+dA=
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680=
|
||||
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
|
||||
cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU=
|
||||
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8=
|
||||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
cloud.google.com/go/storage v1.8.0 h1:86K1Gel7BQ9/WmNWn7dTKMvTLFzwtBe5FNqYbi9X35g=
|
||||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4=
|
||||
cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw=
|
||||
github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
|
||||
github.com/VictoriaMetrics/fasthttp v1.0.7 h1:9RntF8jE9z+ooyXy0tb3Pl76jan4DLaCQoCQFp/cIUE=
|
||||
github.com/VictoriaMetrics/fasthttp v1.0.7/go.mod h1:eaGv8oDDOWE8JQa6GgBgpCwjJhRxavmprtf2oWw8b3o=
|
||||
github.com/VictoriaMetrics/metrics v1.12.2 h1:SG8iAmqavDNuh7GIdHPoGHUhDL23KeKfvSZSozucNeA=
|
||||
github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
|
||||
github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I=
|
||||
github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
|
||||
@@ -66,8 +50,8 @@ github.com/VictoriaMetrics/metricsql v0.7.2/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94
|
||||
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
|
||||
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
|
||||
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
|
||||
github.com/aws/aws-sdk-go v1.35.22 h1:Do184JJHgque0dK/udq6MC14W4hCEGkjLDtzRjxLKGM=
|
||||
github.com/aws/aws-sdk-go v1.35.22/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
|
||||
github.com/aws/aws-sdk-go v1.35.31 h1:6tlaYq4Q311qfhft/fIaND33XI27aW3zIdictcHxifE=
|
||||
github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
@@ -110,11 +94,9 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
|
||||
github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
@@ -126,13 +108,11 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
|
||||
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60=
|
||||
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
@@ -163,10 +143,9 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/klauspost/compress v1.11.0 h1:wJbzvpYMVGG9iTI9VxpnNZfd4DzMPoCWze3GgSqz8yg=
|
||||
github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/klauspost/compress v1.11.2 h1:MiK62aErc3gIiVEtyzKfeOHgW7atJb5g/KNX5m3c2nQ=
|
||||
github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc=
|
||||
github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
@@ -185,8 +164,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
|
||||
github.com/valyala/fastjson v1.6.1 h1:qJs/Kz/HebWzk8LmhOrSm7kdOyJBr1XB+zSkYtEEfQE=
|
||||
github.com/valyala/fastjson v1.6.1/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
|
||||
github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
|
||||
github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
|
||||
github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI=
|
||||
github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
|
||||
github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
|
||||
@@ -206,7 +185,6 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0=
|
||||
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
|
||||
@@ -272,24 +250,21 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/
|
||||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201026091529-146b70c837a4/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102 h1:42cLlJJdEh+ySyeUUbEQ5bsTiq8voBeTuweGVkY6Puw=
|
||||
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc=
|
||||
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58 h1:Mj83v+wSRNEar42a/MQgxk9X42TdEmrOl9i+y8WbxLo=
|
||||
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -328,13 +303,12 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201106081118-db71ae66460a h1:ALUFBKlIyeY7y5ZgPJmblk/vKz+zBQSnNiPkt41sgeg=
|
||||
golang.org/x/sys v0.0.0-20201106081118-db71ae66460a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
|
||||
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
@@ -385,9 +359,9 @@ golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc
|
||||
golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
|
||||
golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
|
||||
golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
|
||||
golang.org/x/tools v0.0.0-20201030143252-cf7a54d06671/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201105220310-78b158585360 h1:/9CzsU8hOpnSUCtem1vfWNgsVeCTgkMdx+VE5YIYxnU=
|
||||
golang.org/x/tools v0.0.0-20201105220310-78b158585360/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201119132711-4783bc9bebf0 h1:26vapYZ9m+DJd68m3DCFP/swNNErXAU7tOMFG7NyUuM=
|
||||
golang.org/x/tools v0.0.0-20201119132711-4783bc9bebf0/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -408,20 +382,16 @@ google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
|
||||
google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0 h1:yfrXXP61wVuLb0vBcG6qaOoIoqYEzOQS8jum51jkv2w=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.31.0 h1:1w5Sz/puhxFo9lTtip2n47k7toB/U2nCqOKNHd3Yrbo=
|
||||
google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo=
|
||||
google.golang.org/api v0.32.0 h1:Le77IccnTqEa8ryp9wIpX5W3zYm7Gf9LhOp9PHcwFts=
|
||||
google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
|
||||
google.golang.org/api v0.34.0 h1:k40adF3uR+6x/+hO5Dh4ZFUqFp67vxvbpafFiJxl10A=
|
||||
google.golang.org/api v0.34.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
|
||||
google.golang.org/api v0.35.0 h1:TBCmTTxUrRDA1iTctnK/fIeitxIZ+TQuaf0j29fmCGo=
|
||||
google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
@@ -455,14 +425,12 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d h1:92D1fum1bJLKSdr11OJ+54YeCMCGYIygTA7R/YZxH5M=
|
||||
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201030142918-24207fddd1c3 h1:sg8vLDNIxFPHTchfhH1E3AI32BL3f23oie38xUWnJM8=
|
||||
google.golang.org/genproto v0.0.0-20201030142918-24207fddd1c3/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201105153401-9d023cd09d72 h1:y5Oh0goCgfBLNYTXMIwrAWyOECSQLVdZ51A14It40OI=
|
||||
google.golang.org/genproto v0.0.0-20201105153401-9d023cd09d72/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4 h1:Rt0FRalMgdSlXAVJvX4pr65KfqaxHXSLkSJRD9pw6g0=
|
||||
google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -474,14 +442,11 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
|
||||
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
|
||||
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
|
||||
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.1 h1:SfXqXS5hkufcdZ/mHtYCh53P2b+92WQq/DZcKLgsFRs=
|
||||
google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
|
||||
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.33.1 h1:DGeFlSan2f+WEtCERJ4J9GJWk15TxUi8QGagfI87Xyc=
|
||||
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
|
||||
google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o=
|
||||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
|
||||
@@ -158,13 +158,13 @@ func (r *ReaderAt) MustFadviseSequentialRead(prefetch bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// OpenReaderAt opens ReaderAt for reading from filename.
|
||||
// MustOpenReaderAt opens ReaderAt for reading from filename.
|
||||
//
|
||||
// MustClose must be called on the returned ReaderAt when it is no longer needed.
|
||||
func OpenReaderAt(path string) (*ReaderAt, error) {
|
||||
func MustOpenReaderAt(path string) *ReaderAt {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot open file %q for reader: %w", path, err)
|
||||
logger.Panicf("FATAL: cannot open file %q for reading: %s", path, err)
|
||||
}
|
||||
var r ReaderAt
|
||||
r.f = f
|
||||
@@ -172,7 +172,8 @@ func OpenReaderAt(path string) (*ReaderAt, error) {
|
||||
if !*disableMmap {
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error in stat: %w", err)
|
||||
MustClose(f)
|
||||
logger.Panicf("FATAL: error in fstat(%q): %s", path, err)
|
||||
}
|
||||
size := fi.Size()
|
||||
bm := &pageCacheBitmap{
|
||||
@@ -188,12 +189,12 @@ func OpenReaderAt(path string) (*ReaderAt, error) {
|
||||
data, err := mmapFile(f, size)
|
||||
if err != nil {
|
||||
MustClose(f)
|
||||
return nil, fmt.Errorf("cannot init reader for %q: %w", path, err)
|
||||
logger.Panicf("FATAL: cannot mmap %q: %s", path, err)
|
||||
}
|
||||
r.mmapData = data
|
||||
}
|
||||
readersCount.Inc()
|
||||
return &r, nil
|
||||
return &r
|
||||
}
|
||||
|
||||
func pageCacheBitmapCleaner(pcbm *atomic.Value, stopCh <-chan struct{}) {
|
||||
|
||||
@@ -22,10 +22,7 @@ func testReaderAt(t *testing.T, bufSize int) {
|
||||
t.Fatalf("cannot create %q: %s", path, err)
|
||||
}
|
||||
defer MustRemoveAll(path)
|
||||
r, err := OpenReaderAt(path)
|
||||
if err != nil {
|
||||
t.Fatalf("error in OpenReaderAt(%q): %s", path, err)
|
||||
}
|
||||
r := MustOpenReaderAt(path)
|
||||
defer r.MustClose()
|
||||
|
||||
buf := make([]byte, bufSize)
|
||||
|
||||
@@ -29,10 +29,7 @@ func benchmarkReaderAtMustReadAt(b *testing.B, isMmap bool) {
|
||||
b.Fatalf("cannot create %q: %s", path, err)
|
||||
}
|
||||
defer MustRemoveAll(path)
|
||||
r, err := OpenReaderAt(path)
|
||||
if err != nil {
|
||||
b.Fatalf("error in OpenReaderAt(%q): %s", path, err)
|
||||
}
|
||||
r := MustOpenReaderAt(path)
|
||||
defer r.MustClose()
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
@@ -208,7 +208,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
|
||||
r.URL.Path = path
|
||||
switch r.URL.Path {
|
||||
case "/health":
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
deadline := atomic.LoadInt64(&s.shutdownDelayDeadline)
|
||||
if deadline <= 0 {
|
||||
w.Write([]byte("OK"))
|
||||
@@ -244,7 +244,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
|
||||
return
|
||||
}
|
||||
startTime := time.Now()
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
WritePrometheusMetrics(w)
|
||||
metricsHandlerDuration.UpdateDuration(startTime)
|
||||
return
|
||||
@@ -395,7 +395,7 @@ func (zrw *gzipResponseWriter) Write(p []byte) (int, error) {
|
||||
if h.Get("Content-Type") == "" {
|
||||
// Disable auto-detection of content-type, since it
|
||||
// is incorrectly detected after the compression.
|
||||
h.Set("Content-Type", "text/html")
|
||||
h.Set("Content-Type", "text/html; charset=utf-8")
|
||||
}
|
||||
}
|
||||
zrw.writeHeader()
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
|
||||
@@ -23,8 +22,10 @@ var (
|
||||
loggerOutput = flag.String("loggerOutput", "stderr", "Output for the logs. Supported values: stderr, stdout")
|
||||
disableTimestamps = flag.Bool("loggerDisableTimestamps", false, "Whether to disable writing timestamps in logs")
|
||||
|
||||
errorsPerSecondLimit = flag.Int("loggerErrorsPerSecondLimit", 10, "Per-second limit on the number of ERROR messages. If more than the given number of errors "+
|
||||
errorsPerSecondLimit = flag.Int("loggerErrorsPerSecondLimit", 0, "Per-second limit on the number of ERROR messages. If more than the given number of errors "+
|
||||
"are emitted per second, then the remaining errors are suppressed. Zero value disables the rate limit")
|
||||
warnsPerSecondLimit = flag.Int("loggerWarnsPerSecondLimit", 0, "Per-second limit on the number of WARN messages. If more than the given number of warns "+
|
||||
"are emitted per second, then the remaining warns are suppressed. Zero value disables the rate limit")
|
||||
)
|
||||
|
||||
// Init initializes the logger.
|
||||
@@ -36,7 +37,7 @@ func Init() {
|
||||
setLoggerOutput()
|
||||
validateLoggerLevel()
|
||||
validateLoggerFormat()
|
||||
go errorsLoggedCleaner()
|
||||
go logLimiterCleaner()
|
||||
logAllFlags()
|
||||
}
|
||||
|
||||
@@ -125,14 +126,60 @@ func logLevelSkipframes(skipframes int, level, format string, args ...interface{
|
||||
logMessage(level, msg, 3+skipframes)
|
||||
}
|
||||
|
||||
func errorsLoggedCleaner() {
|
||||
func logLimiterCleaner() {
|
||||
for {
|
||||
time.Sleep(time.Second)
|
||||
atomic.StoreUint64(&errorsLogged, 0)
|
||||
logLimiter.reset()
|
||||
}
|
||||
}
|
||||
|
||||
var errorsLogged uint64
|
||||
var logLimiter = newLogLimit()
|
||||
|
||||
func newLogLimit() *logLimit {
|
||||
return &logLimit{
|
||||
m: make(map[string]uint64),
|
||||
}
|
||||
}
|
||||
|
||||
type logLimit struct {
|
||||
mu sync.Mutex
|
||||
m map[string]uint64
|
||||
}
|
||||
|
||||
func (ll *logLimit) reset() {
|
||||
ll.mu.Lock()
|
||||
ll.m = make(map[string]uint64, len(ll.m))
|
||||
ll.mu.Unlock()
|
||||
}
|
||||
|
||||
// needSuppress checks if the number of calls for the given location exceeds the given limit.
|
||||
//
|
||||
// When the number of calls equals limit, log message prefix returned.
|
||||
func (ll *logLimit) needSuppress(location string, limit uint64) (bool, string) {
|
||||
// fast path
|
||||
var msg string
|
||||
if limit == 0 {
|
||||
return false, msg
|
||||
}
|
||||
ll.mu.Lock()
|
||||
defer ll.mu.Unlock()
|
||||
|
||||
if n, ok := ll.m[location]; ok {
|
||||
if n >= limit {
|
||||
switch n {
|
||||
// report only once
|
||||
case limit:
|
||||
msg = fmt.Sprintf("suppressing log message with rate limit=%d: ", limit)
|
||||
default:
|
||||
return true, msg
|
||||
}
|
||||
}
|
||||
ll.m[location] = n + 1
|
||||
} else {
|
||||
ll.m[location] = 1
|
||||
}
|
||||
return false, msg
|
||||
}
|
||||
|
||||
type logWriter struct {
|
||||
}
|
||||
@@ -143,13 +190,6 @@ func (lw *logWriter) Write(p []byte) (int, error) {
|
||||
}
|
||||
|
||||
func logMessage(level, msg string, skipframes int) {
|
||||
// rate limit ERROR log messages
|
||||
if level == "ERROR" {
|
||||
if n := atomic.AddUint64(&errorsLogged, 1); *errorsPerSecondLimit > 0 && n > uint64(*errorsPerSecondLimit) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
timestamp := ""
|
||||
if !*disableTimestamps {
|
||||
timestamp = time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
|
||||
@@ -164,23 +204,39 @@ func logMessage(level, msg string, skipframes int) {
|
||||
// Strip /VictoriaMetrics/ prefix
|
||||
file = file[n+len("/VictoriaMetrics/"):]
|
||||
}
|
||||
location := fmt.Sprintf("%s:%d", file, line)
|
||||
|
||||
// rate limit ERROR and WARN log messages with given limit.
|
||||
if level == "ERROR" || level == "WARN" {
|
||||
limit := uint64(*errorsPerSecondLimit)
|
||||
if level == "WARN" {
|
||||
limit = uint64(*warnsPerSecondLimit)
|
||||
}
|
||||
ok, suppressMessage := logLimiter.needSuppress(location, limit)
|
||||
if ok {
|
||||
return
|
||||
}
|
||||
if len(suppressMessage) > 0 {
|
||||
msg = suppressMessage + msg
|
||||
}
|
||||
}
|
||||
|
||||
for len(msg) > 0 && msg[len(msg)-1] == '\n' {
|
||||
msg = msg[:len(msg)-1]
|
||||
}
|
||||
var logMsg string
|
||||
switch *loggerFormat {
|
||||
case "json":
|
||||
caller := fmt.Sprintf("%s:%d", file, line)
|
||||
if *disableTimestamps {
|
||||
logMsg = fmt.Sprintf(`{"level":%q,"caller":%q,"msg":%q}`+"\n", levelLowercase, caller, msg)
|
||||
logMsg = fmt.Sprintf(`{"level":%q,"caller":%q,"msg":%q}`+"\n", levelLowercase, location, msg)
|
||||
} else {
|
||||
logMsg = fmt.Sprintf(`{"ts":%q,"level":%q,"caller":%q,"msg":%q}`+"\n", timestamp, levelLowercase, caller, msg)
|
||||
logMsg = fmt.Sprintf(`{"ts":%q,"level":%q,"caller":%q,"msg":%q}`+"\n", timestamp, levelLowercase, location, msg)
|
||||
}
|
||||
default:
|
||||
if *disableTimestamps {
|
||||
logMsg = fmt.Sprintf("%s\t%s:%d\t%s\n", levelLowercase, file, line, msg)
|
||||
logMsg = fmt.Sprintf("%s\t%s\t%s\n", levelLowercase, location, msg)
|
||||
} else {
|
||||
logMsg = fmt.Sprintf("%s\t%s\t%s:%d\t%s\n", timestamp, levelLowercase, file, line, msg)
|
||||
logMsg = fmt.Sprintf("%s\t%s\t%s\t%s\n", timestamp, levelLowercase, location, msg)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,7 +246,6 @@ func logMessage(level, msg string, skipframes int) {
|
||||
mu.Unlock()
|
||||
|
||||
// Increment vm_log_messages_total
|
||||
location := fmt.Sprintf("%s:%d", file, line)
|
||||
counterName := fmt.Sprintf(`vm_log_messages_total{app_version=%q, level=%q, location=%q}`, buildinfo.Version, levelLowercase, location)
|
||||
metrics.GetOrCreateCounter(counterName).Inc()
|
||||
|
||||
|
||||
@@ -31,6 +31,12 @@ type inmemoryBlock struct {
|
||||
func (ib *inmemoryBlock) Reset() {
|
||||
ib.commonPrefix = ib.commonPrefix[:0]
|
||||
ib.data = ib.data[:0]
|
||||
|
||||
items := ib.items
|
||||
for i := range items {
|
||||
// Remove reference to by slice, so GC could free the byte slice.
|
||||
items[i] = nil
|
||||
}
|
||||
ib.items = ib.items[:0]
|
||||
}
|
||||
|
||||
|
||||
@@ -78,30 +78,15 @@ func openFilePart(path string) (*part, error) {
|
||||
metaindexSize := fs.MustFileSize(metaindexPath)
|
||||
|
||||
indexPath := path + "/index.bin"
|
||||
indexFile, err := fs.OpenReaderAt(indexPath)
|
||||
if err != nil {
|
||||
metaindexFile.MustClose()
|
||||
return nil, fmt.Errorf("cannot open %q: %w", indexPath, err)
|
||||
}
|
||||
indexFile := fs.MustOpenReaderAt(indexPath)
|
||||
indexSize := fs.MustFileSize(indexPath)
|
||||
|
||||
itemsPath := path + "/items.bin"
|
||||
itemsFile, err := fs.OpenReaderAt(itemsPath)
|
||||
if err != nil {
|
||||
metaindexFile.MustClose()
|
||||
indexFile.MustClose()
|
||||
return nil, fmt.Errorf("cannot open %q: %w", itemsPath, err)
|
||||
}
|
||||
itemsFile := fs.MustOpenReaderAt(itemsPath)
|
||||
itemsSize := fs.MustFileSize(itemsPath)
|
||||
|
||||
lensPath := path + "/lens.bin"
|
||||
lensFile, err := fs.OpenReaderAt(lensPath)
|
||||
if err != nil {
|
||||
metaindexFile.MustClose()
|
||||
indexFile.MustClose()
|
||||
itemsFile.MustClose()
|
||||
return nil, fmt.Errorf("cannot open %q: %w", lensPath, err)
|
||||
}
|
||||
lensFile := fs.MustOpenReaderAt(lensPath)
|
||||
lensSize := fs.MustFileSize(lensPath)
|
||||
|
||||
size := metaindexSize + indexSize + itemsSize + lensSize
|
||||
@@ -260,7 +245,7 @@ func (idxbc *indexBlockCache) Get(k uint64) *indexBlock {
|
||||
func (idxbc *indexBlockCache) Put(k uint64, idxb *indexBlock) {
|
||||
idxbc.mu.Lock()
|
||||
|
||||
// Remove superflouos entries.
|
||||
// Remove superfluous entries.
|
||||
if overflow := len(idxbc.m) - getMaxCachedIndexBlocksPerPart(); overflow > 0 {
|
||||
// Remove 10% of items from the cache.
|
||||
overflow = int(float64(len(idxbc.m)) * 0.1)
|
||||
@@ -408,7 +393,7 @@ func (ibc *inmemoryBlockCache) Get(k inmemoryBlockCacheKey) *inmemoryBlock {
|
||||
func (ibc *inmemoryBlockCache) Put(k inmemoryBlockCacheKey, ib *inmemoryBlock) {
|
||||
ibc.mu.Lock()
|
||||
|
||||
// Clean superflouos entries in cache.
|
||||
// Clean superfluous entries in cache.
|
||||
if overflow := len(ibc.m) - getMaxCachedInmemoryBlocksPerPart(); overflow > 0 {
|
||||
// Remove 10% of items from the cache.
|
||||
overflow = int(float64(len(ibc.m)) * 0.1)
|
||||
|
||||
@@ -369,7 +369,7 @@ func (tb *Table) AddItems(items [][]byte) error {
|
||||
tb.rawItemsBlocks = append(tb.rawItemsBlocks, ib)
|
||||
}
|
||||
}
|
||||
if len(tb.rawItemsBlocks) >= 1024 {
|
||||
if len(tb.rawItemsBlocks) >= 512 {
|
||||
blocksToMerge = tb.rawItemsBlocks
|
||||
tb.rawItemsBlocks = nil
|
||||
tb.rawItemsLastFlushTime = fasttime.UnixTimestamp()
|
||||
|
||||
@@ -136,7 +136,7 @@ func testTableSearchSerial(tb *Table, items []string) error {
|
||||
n++
|
||||
}
|
||||
if ts.NextItem() {
|
||||
return fmt.Errorf("superflouos item found at position %d when searching for %q: %q", n, key, ts.Item)
|
||||
return fmt.Errorf("superfluous item found at position %d when searching for %q: %q", n, key, ts.Item)
|
||||
}
|
||||
if err := ts.Error(); err != nil {
|
||||
return fmt.Errorf("unexpected error when searching for %q: %w", key, err)
|
||||
|
||||
@@ -13,18 +13,18 @@ import (
|
||||
//
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config
|
||||
type TLSConfig struct {
|
||||
CAFile string `yaml:"ca_file"`
|
||||
CertFile string `yaml:"cert_file"`
|
||||
KeyFile string `yaml:"key_file"`
|
||||
ServerName string `yaml:"server_name"`
|
||||
InsecureSkipVerify bool `yaml:"insecure_skip_verify"`
|
||||
CAFile string `yaml:"ca_file,omitempty"`
|
||||
CertFile string `yaml:"cert_file,omitempty"`
|
||||
KeyFile string `yaml:"key_file,omitempty"`
|
||||
ServerName string `yaml:"server_name,omitempty"`
|
||||
InsecureSkipVerify bool `yaml:"insecure_skip_verify,omitempty"`
|
||||
}
|
||||
|
||||
// BasicAuthConfig represents basic auth config.
|
||||
type BasicAuthConfig struct {
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
PasswordFile string `yaml:"password_file"`
|
||||
Password string `yaml:"password,omitempty"`
|
||||
PasswordFile string `yaml:"password_file,omitempty"`
|
||||
}
|
||||
|
||||
// Config is auth config.
|
||||
|
||||
@@ -14,13 +14,13 @@ import (
|
||||
//
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
|
||||
type RelabelConfig struct {
|
||||
SourceLabels []string `yaml:"source_labels"`
|
||||
Separator *string `yaml:"separator"`
|
||||
TargetLabel string `yaml:"target_label"`
|
||||
Regex *string `yaml:"regex"`
|
||||
Modulus uint64 `yaml:"modulus"`
|
||||
Replacement *string `yaml:"replacement"`
|
||||
Action string `yaml:"action"`
|
||||
SourceLabels []string `yaml:"source_labels,flow,omitempty"`
|
||||
Separator *string `yaml:"separator,omitempty"`
|
||||
TargetLabel string `yaml:"target_label,omitempty"`
|
||||
Regex *string `yaml:"regex,omitempty"`
|
||||
Modulus uint64 `yaml:"modulus,omitempty"`
|
||||
Replacement *string `yaml:"replacement,omitempty"`
|
||||
Action string `yaml:"action,omitempty"`
|
||||
}
|
||||
|
||||
// LoadRelabelConfigs loads relabel configs from the given path.
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -20,6 +19,7 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dockerswarm"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/eureka"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack"
|
||||
@@ -32,6 +32,9 @@ var (
|
||||
dryRun = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+
|
||||
"Returns non-zero exit code on parsing errors and emits these errors to stderr. "+
|
||||
"Pass -loggerLevel=ERROR if you don't need to see info messages in the output")
|
||||
dropOriginalLabels = flag.Bool("promscrape.dropOriginalLabels", false, "Whether to drop original labels for scrape targets at /targets and /api/v1/targets pages. "+
|
||||
"This may be needed for reducing memory usage when original labels for big number of scrape targets occupy big amounts of memory. "+
|
||||
"Note that this reduces debuggability for improper per-target relabeling configs")
|
||||
)
|
||||
|
||||
// Config represents essential parts from Prometheus config defined at https://prometheus.io/docs/prometheus/latest/configuration/configuration/
|
||||
@@ -47,9 +50,9 @@ type Config struct {
|
||||
//
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/
|
||||
type GlobalConfig struct {
|
||||
ScrapeInterval time.Duration `yaml:"scrape_interval"`
|
||||
ScrapeTimeout time.Duration `yaml:"scrape_timeout"`
|
||||
ExternalLabels map[string]string `yaml:"external_labels"`
|
||||
ScrapeInterval time.Duration `yaml:"scrape_interval,omitempty"`
|
||||
ScrapeTimeout time.Duration `yaml:"scrape_timeout,omitempty"`
|
||||
ExternalLabels map[string]string `yaml:"external_labels,omitempty"`
|
||||
}
|
||||
|
||||
// ScrapeConfig represents essential parts for `scrape_config` section of Prometheus config.
|
||||
@@ -57,34 +60,35 @@ type GlobalConfig struct {
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config
|
||||
type ScrapeConfig struct {
|
||||
JobName string `yaml:"job_name"`
|
||||
ScrapeInterval time.Duration `yaml:"scrape_interval"`
|
||||
ScrapeTimeout time.Duration `yaml:"scrape_timeout"`
|
||||
MetricsPath string `yaml:"metrics_path"`
|
||||
HonorLabels bool `yaml:"honor_labels"`
|
||||
HonorTimestamps bool `yaml:"honor_timestamps"`
|
||||
Scheme string `yaml:"scheme"`
|
||||
Params map[string][]string `yaml:"params"`
|
||||
BasicAuth *promauth.BasicAuthConfig `yaml:"basic_auth"`
|
||||
BearerToken string `yaml:"bearer_token"`
|
||||
BearerTokenFile string `yaml:"bearer_token_file"`
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config"`
|
||||
StaticConfigs []StaticConfig `yaml:"static_configs"`
|
||||
FileSDConfigs []FileSDConfig `yaml:"file_sd_configs"`
|
||||
KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs"`
|
||||
OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs"`
|
||||
ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs"`
|
||||
DockerSwarmConfigs []dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs"`
|
||||
DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs"`
|
||||
EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs"`
|
||||
GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs"`
|
||||
RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs"`
|
||||
MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs"`
|
||||
SampleLimit int `yaml:"sample_limit"`
|
||||
ScrapeInterval time.Duration `yaml:"scrape_interval,omitempty"`
|
||||
ScrapeTimeout time.Duration `yaml:"scrape_timeout,omitempty"`
|
||||
MetricsPath string `yaml:"metrics_path,omitempty"`
|
||||
HonorLabels bool `yaml:"honor_labels,omitempty"`
|
||||
HonorTimestamps bool `yaml:"honor_timestamps,omitempty"`
|
||||
Scheme string `yaml:"scheme,omitempty"`
|
||||
Params map[string][]string `yaml:"params,omitempty"`
|
||||
BasicAuth *promauth.BasicAuthConfig `yaml:"basic_auth,omitempty"`
|
||||
BearerToken string `yaml:"bearer_token,omitempty"`
|
||||
BearerTokenFile string `yaml:"bearer_token_file,omitempty"`
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config,omitempty"`
|
||||
StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
|
||||
FileSDConfigs []FileSDConfig `yaml:"file_sd_configs,omitempty"`
|
||||
KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"`
|
||||
OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"`
|
||||
ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
|
||||
EurekaSDConfigs []eureka.SDConfig `yaml:"eureka_sd_configs,omitempty"`
|
||||
DockerSwarmConfigs []dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
|
||||
DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
|
||||
EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"`
|
||||
GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs,omitempty"`
|
||||
RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"`
|
||||
MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs,omitempty"`
|
||||
SampleLimit int `yaml:"sample_limit,omitempty"`
|
||||
|
||||
// These options are supported only by lib/promscrape.
|
||||
DisableCompression bool `yaml:"disable_compression"`
|
||||
DisableKeepAlive bool `yaml:"disable_keepalive"`
|
||||
StreamParse bool `yaml:"stream_parse"`
|
||||
DisableCompression bool `yaml:"disable_compression,omitempty"`
|
||||
DisableKeepAlive bool `yaml:"disable_keepalive,omitempty"`
|
||||
StreamParse bool `yaml:"stream_parse,omitempty"`
|
||||
|
||||
// This is set in loadConfig
|
||||
swc *scrapeWorkConfig
|
||||
@@ -103,7 +107,7 @@ type FileSDConfig struct {
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config
|
||||
type StaticConfig struct {
|
||||
Targets []string `yaml:"targets"`
|
||||
Labels map[string]string `yaml:"labels"`
|
||||
Labels map[string]string `yaml:"labels,omitempty"`
|
||||
}
|
||||
|
||||
func loadStaticConfigs(path string) ([]StaticConfig, error) {
|
||||
@@ -129,16 +133,14 @@ func loadConfig(path string) (cfg *Config, data []byte, err error) {
|
||||
if err := cfgObj.parse(data, path); err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot parse Prometheus config from %q: %w", path, err)
|
||||
}
|
||||
if *dryRun {
|
||||
// This is a dirty hack for checking Prometheus config only.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/362
|
||||
// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/508 for details.
|
||||
logger.Infof("Success: the config at %q has no errors; exitting with 0 status code", path)
|
||||
os.Exit(0)
|
||||
}
|
||||
return &cfgObj, data, nil
|
||||
}
|
||||
|
||||
// IsDryRun returns true if -promscrape.config.dryRun command-line flag is set
|
||||
func IsDryRun() bool {
|
||||
return *dryRun
|
||||
}
|
||||
|
||||
func (cfg *Config) parse(data []byte, path string) error {
|
||||
if err := unmarshalMaybeStrict(data, cfg); err != nil {
|
||||
return fmt.Errorf("cannot unmarshal data: %w", err)
|
||||
@@ -181,7 +183,7 @@ func getSWSByJob(sws []ScrapeWork) map[string][]ScrapeWork {
|
||||
// getKubernetesSDScrapeWork returns `kubernetes_sd_configs` ScrapeWork from cfg.
|
||||
func (cfg *Config) getKubernetesSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
swsPrevByJob := getSWSByJob(prev)
|
||||
var dst []ScrapeWork
|
||||
dst := make([]ScrapeWork, 0, len(prev))
|
||||
for i := range cfg.ScrapeConfigs {
|
||||
sc := &cfg.ScrapeConfigs[i]
|
||||
dstLen := len(dst)
|
||||
@@ -209,7 +211,7 @@ func (cfg *Config) getKubernetesSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
// getOpenStackSDScrapeWork returns `openstack_sd_configs` ScrapeWork from cfg.
|
||||
func (cfg *Config) getOpenStackSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
swsPrevByJob := getSWSByJob(prev)
|
||||
var dst []ScrapeWork
|
||||
dst := make([]ScrapeWork, 0, len(prev))
|
||||
for i := range cfg.ScrapeConfigs {
|
||||
sc := &cfg.ScrapeConfigs[i]
|
||||
dstLen := len(dst)
|
||||
@@ -237,7 +239,7 @@ func (cfg *Config) getOpenStackSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
// getDockerSwarmSDScrapeWork returns `dockerswarm_sd_configs` ScrapeWork from cfg.
|
||||
func (cfg *Config) getDockerSwarmSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
swsPrevByJob := getSWSByJob(prev)
|
||||
var dst []ScrapeWork
|
||||
dst := make([]ScrapeWork, 0, len(prev))
|
||||
for i := range cfg.ScrapeConfigs {
|
||||
sc := &cfg.ScrapeConfigs[i]
|
||||
dstLen := len(dst)
|
||||
@@ -265,7 +267,7 @@ func (cfg *Config) getDockerSwarmSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
// getConsulSDScrapeWork returns `consul_sd_configs` ScrapeWork from cfg.
|
||||
func (cfg *Config) getConsulSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
swsPrevByJob := getSWSByJob(prev)
|
||||
var dst []ScrapeWork
|
||||
dst := make([]ScrapeWork, 0, len(prev))
|
||||
for i := range cfg.ScrapeConfigs {
|
||||
sc := &cfg.ScrapeConfigs[i]
|
||||
dstLen := len(dst)
|
||||
@@ -290,10 +292,38 @@ func (cfg *Config) getConsulSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
return dst
|
||||
}
|
||||
|
||||
// getEurekaSDScrapeWork returns `eureka_sd_configs` ScrapeWork from cfg.
|
||||
func (cfg *Config) getEurekaSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
swsPrevByJob := getSWSByJob(prev)
|
||||
dst := make([]ScrapeWork, 0, len(prev))
|
||||
for i := range cfg.ScrapeConfigs {
|
||||
sc := &cfg.ScrapeConfigs[i]
|
||||
dstLen := len(dst)
|
||||
ok := true
|
||||
for j := range sc.EurekaSDConfigs {
|
||||
sdc := &sc.EurekaSDConfigs[j]
|
||||
var okLocal bool
|
||||
dst, okLocal = appendEurekaScrapeWork(dst, sdc, cfg.baseDir, sc.swc)
|
||||
if ok {
|
||||
ok = okLocal
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
swsPrev := swsPrevByJob[sc.swc.jobName]
|
||||
if len(swsPrev) > 0 {
|
||||
logger.Errorf("there were errors when discovering eureka targets for job %q, so preserving the previous targets", sc.swc.jobName)
|
||||
dst = append(dst[:dstLen], swsPrev...)
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// getDNSSDScrapeWork returns `dns_sd_configs` ScrapeWork from cfg.
|
||||
func (cfg *Config) getDNSSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
swsPrevByJob := getSWSByJob(prev)
|
||||
var dst []ScrapeWork
|
||||
dst := make([]ScrapeWork, 0, len(prev))
|
||||
for i := range cfg.ScrapeConfigs {
|
||||
sc := &cfg.ScrapeConfigs[i]
|
||||
dstLen := len(dst)
|
||||
@@ -321,7 +351,7 @@ func (cfg *Config) getDNSSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
// getEC2SDScrapeWork returns `ec2_sd_configs` ScrapeWork from cfg.
|
||||
func (cfg *Config) getEC2SDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
swsPrevByJob := getSWSByJob(prev)
|
||||
var dst []ScrapeWork
|
||||
dst := make([]ScrapeWork, 0, len(prev))
|
||||
for i := range cfg.ScrapeConfigs {
|
||||
sc := &cfg.ScrapeConfigs[i]
|
||||
dstLen := len(dst)
|
||||
@@ -349,7 +379,7 @@ func (cfg *Config) getEC2SDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
// getGCESDScrapeWork returns `gce_sd_configs` ScrapeWork from cfg.
|
||||
func (cfg *Config) getGCESDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
swsPrevByJob := getSWSByJob(prev)
|
||||
var dst []ScrapeWork
|
||||
dst := make([]ScrapeWork, 0, len(prev))
|
||||
for i := range cfg.ScrapeConfigs {
|
||||
sc := &cfg.ScrapeConfigs[i]
|
||||
dstLen := len(dst)
|
||||
@@ -387,7 +417,7 @@ func (cfg *Config) getFileSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
swsMapPrev[filepath] = append(swsMapPrev[filepath], *sw)
|
||||
}
|
||||
}
|
||||
var dst []ScrapeWork
|
||||
dst := make([]ScrapeWork, 0, len(prev))
|
||||
for i := range cfg.ScrapeConfigs {
|
||||
sc := &cfg.ScrapeConfigs[i]
|
||||
for j := range sc.FileSDConfigs {
|
||||
@@ -534,6 +564,15 @@ func appendConsulScrapeWork(dst []ScrapeWork, sdc *consul.SDConfig, baseDir stri
|
||||
return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "consul_sd_config"), true
|
||||
}
|
||||
|
||||
func appendEurekaScrapeWork(dst []ScrapeWork, sdc *eureka.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]ScrapeWork, bool) {
|
||||
targetLabels, err := eureka.GetLabels(sdc, baseDir)
|
||||
if err != nil {
|
||||
logger.Errorf("error when discovering eureka targets for `job_name` %q: %s; skipping it", swc.jobName, err)
|
||||
return dst, false
|
||||
}
|
||||
return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "eureka_sd_config"), true
|
||||
}
|
||||
|
||||
func appendDNSScrapeWork(dst []ScrapeWork, sdc *dns.SDConfig, swc *scrapeWorkConfig) ([]ScrapeWork, bool) {
|
||||
targetLabels, err := dns.GetLabels(sdc)
|
||||
if err != nil {
|
||||
@@ -639,14 +678,17 @@ func (stc *StaticConfig) appendScrapeWork(dst []ScrapeWork, swc *scrapeWorkConfi
|
||||
|
||||
func appendScrapeWork(dst []ScrapeWork, swc *scrapeWorkConfig, target string, extraLabels, metaLabels map[string]string) ([]ScrapeWork, error) {
|
||||
labels := mergeLabels(swc.jobName, swc.scheme, target, swc.metricsPath, extraLabels, swc.externalLabels, metaLabels, swc.params)
|
||||
originalLabels := append([]prompbmarshal.Label{}, labels...)
|
||||
promrelabel.SortLabels(originalLabels)
|
||||
var originalLabels []prompbmarshal.Label
|
||||
if !*dropOriginalLabels {
|
||||
originalLabels = append([]prompbmarshal.Label{}, labels...)
|
||||
promrelabel.SortLabels(originalLabels)
|
||||
}
|
||||
labels = promrelabel.ApplyRelabelConfigs(labels, 0, swc.relabelConfigs, false)
|
||||
labels = promrelabel.RemoveMetaLabels(labels[:0], labels)
|
||||
// Remove references to already deleted labels, so GC could clean strings for label name and label value.
|
||||
// Remove references to already deleted labels, so GC could clean strings for label name and label value past len(labels).
|
||||
// This should reduce memory usage when relabeling creates big number of temporary labels with long names and/or values.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825 for details.
|
||||
promrelabel.CleanLabels(labels[len(labels):cap(labels)])
|
||||
labels = append([]prompbmarshal.Label{}, labels...)
|
||||
|
||||
if len(labels) == 0 {
|
||||
// Drop target without labels.
|
||||
|
||||
@@ -475,7 +475,7 @@ scrape_configs:
|
||||
- job_name: foo
|
||||
static_configs:
|
||||
- targets: ["xxx"]
|
||||
`, nil)
|
||||
`, []ScrapeWork{})
|
||||
f(`
|
||||
scrape_configs:
|
||||
- job_name: foo
|
||||
|
||||
@@ -10,18 +10,18 @@ import (
|
||||
//
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
|
||||
type SDConfig struct {
|
||||
Server string `yaml:"server"`
|
||||
Server string `yaml:"server,omitempty"`
|
||||
Token *string `yaml:"token"`
|
||||
Datacenter string `yaml:"datacenter"`
|
||||
Scheme string `yaml:"scheme"`
|
||||
Scheme string `yaml:"scheme,omitempty"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config"`
|
||||
Services []string `yaml:"services"`
|
||||
Tags []string `yaml:"tags"`
|
||||
NodeMeta map[string]string `yaml:"node_meta"`
|
||||
TagSeparator *string `yaml:"tag_separator"`
|
||||
AllowStale bool `yaml:"allow_stale"`
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config,omitempty"`
|
||||
Services []string `yaml:"services,omitempty"`
|
||||
Tags []string `yaml:"tags,omitempty"`
|
||||
NodeMeta map[string]string `yaml:"node_meta,omitempty"`
|
||||
TagSeparator *string `yaml:"tag_separator,omitempty"`
|
||||
AllowStale bool `yaml:"allow_stale,omitempty"`
|
||||
// RefreshInterval time.Duration `yaml:"refresh_interval"`
|
||||
// refresh_interval is obtained from `-promscrape.consulSDCheckInterval` command-line option.
|
||||
}
|
||||
|
||||
@@ -17,8 +17,8 @@ import (
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
|
||||
type SDConfig struct {
|
||||
Names []string `yaml:"names"`
|
||||
Type string `yaml:"type"`
|
||||
Port *int `yaml:"port"`
|
||||
Type string `yaml:"type,omitempty"`
|
||||
Port *int `yaml:"port,omitempty"`
|
||||
// RefreshInterval time.Duration `yaml:"refresh_interval"`
|
||||
// refresh_interval is obtained from `-promscrape.dnsSDCheckInterval` command-line option.
|
||||
}
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
package dockerswarm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
|
||||
)
|
||||
@@ -12,6 +16,9 @@ var configMap = discoveryutils.NewConfigMap()
|
||||
type apiConfig struct {
|
||||
client *discoveryutils.Client
|
||||
port int
|
||||
|
||||
// filtersQueryArg contains escaped `filters` query arg to add to each request to Docker Swarm API.
|
||||
filtersQueryArg string
|
||||
}
|
||||
|
||||
func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
||||
@@ -24,7 +31,8 @@ func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
||||
|
||||
func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
||||
cfg := &apiConfig{
|
||||
port: sdc.Port,
|
||||
port: sdc.Port,
|
||||
filtersQueryArg: getFiltersQueryArg(sdc.Filters),
|
||||
}
|
||||
ac, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig)
|
||||
if err != nil {
|
||||
@@ -37,3 +45,36 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
||||
cfg.client = client
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (cfg *apiConfig) getAPIResponse(path string) ([]byte, error) {
|
||||
if len(cfg.filtersQueryArg) > 0 {
|
||||
separator := "?"
|
||||
if strings.Contains(path, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
path += separator + "filters=" + cfg.filtersQueryArg
|
||||
}
|
||||
return cfg.client.GetAPIResponse(path)
|
||||
}
|
||||
|
||||
func getFiltersQueryArg(filters []Filter) string {
|
||||
if len(filters) == 0 {
|
||||
return ""
|
||||
}
|
||||
m := make(map[string]map[string]bool)
|
||||
for _, f := range filters {
|
||||
x := m[f.Name]
|
||||
if x == nil {
|
||||
x = make(map[string]bool)
|
||||
m[f.Name] = x
|
||||
}
|
||||
for _, value := range f.Values {
|
||||
x[value] = true
|
||||
}
|
||||
}
|
||||
buf, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
logger.Panicf("BUG: unexpected error in json.Marshal: %s", err)
|
||||
}
|
||||
return url.QueryEscape(string(buf))
|
||||
}
|
||||
|
||||
26
lib/promscrape/discovery/dockerswarm/api_test.go
Normal file
26
lib/promscrape/discovery/dockerswarm/api_test.go
Normal file
@@ -0,0 +1,26 @@
|
||||
package dockerswarm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGetFiltersQueryArg(t *testing.T) {
|
||||
f := func(filters []Filter, queryArgExpected string) {
|
||||
t.Helper()
|
||||
queryArg := getFiltersQueryArg(filters)
|
||||
if queryArg != queryArgExpected {
|
||||
t.Fatalf("unexpected query arg; got %s; want %s", queryArg, queryArgExpected)
|
||||
}
|
||||
}
|
||||
f(nil, "")
|
||||
f([]Filter{
|
||||
{
|
||||
Name: "name",
|
||||
Values: []string{"foo", "bar"},
|
||||
},
|
||||
{
|
||||
Name: "xxx",
|
||||
Values: []string{"aa"},
|
||||
},
|
||||
}, "%7B%22name%22%3A%7B%22bar%22%3Atrue%2C%22foo%22%3Atrue%7D%2C%22xxx%22%3A%7B%22aa%22%3Atrue%7D%7D")
|
||||
}
|
||||
@@ -10,15 +10,23 @@ import (
|
||||
//
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config
|
||||
type SDConfig struct {
|
||||
Host string `yaml:"host"`
|
||||
Host string `yaml:"host"`
|
||||
Role string `yaml:"role"`
|
||||
Port int `yaml:"port,omitempty"`
|
||||
Filters []Filter `yaml:"filters,omitempty"`
|
||||
|
||||
// TODO: add support for proxy_url
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config"`
|
||||
Role string `yaml:"role"`
|
||||
Port int `yaml:"port"`
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config,omitempty"`
|
||||
// refresh_interval is obtained from `-promscrape.dockerswarmSDCheckInterval` command-line option
|
||||
BasicAuth *promauth.BasicAuthConfig `yaml:"basic_auth"`
|
||||
BearerToken string `yaml:"bearer_token"`
|
||||
BearerTokenFile string `yaml:"bearer_token_file"`
|
||||
BasicAuth *promauth.BasicAuthConfig `yaml:"basic_auth,omitempty"`
|
||||
BearerToken string `yaml:"bearer_token,omitempty"`
|
||||
BearerTokenFile string `yaml:"bearer_token_file,omitempty"`
|
||||
}
|
||||
|
||||
// Filter is a filter, which can be passed to SDConfig.
|
||||
type Filter struct {
|
||||
Name string `yaml:"name"`
|
||||
Values []string `yaml:"values"`
|
||||
}
|
||||
|
||||
// GetLabels returns dockerswarm labels according to sdc.
|
||||
|
||||
@@ -27,7 +27,7 @@ func getNetworksLabelsByNetworkID(cfg *apiConfig) (map[string]map[string]string,
|
||||
}
|
||||
|
||||
func getNetworks(cfg *apiConfig) ([]network, error) {
|
||||
resp, err := cfg.client.GetAPIResponse("/networks")
|
||||
resp, err := cfg.getAPIResponse("/networks")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query dockerswarm api for networks: %w", err)
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ func getNodesLabels(cfg *apiConfig) ([]map[string]string, error) {
|
||||
}
|
||||
|
||||
func getNodes(cfg *apiConfig) ([]node, error) {
|
||||
resp, err := cfg.client.GetAPIResponse("/nodes")
|
||||
resp, err := cfg.getAPIResponse("/nodes")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query dockerswarm api for nodes: %w", err)
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ func getServicesLabels(cfg *apiConfig) ([]map[string]string, error) {
|
||||
}
|
||||
|
||||
func getServices(cfg *apiConfig) ([]service, error) {
|
||||
data, err := cfg.client.GetAPIResponse("/services")
|
||||
data, err := cfg.getAPIResponse("/services")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query dockerswarm api for services: %w", err)
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ func getTasksLabels(cfg *apiConfig) ([]map[string]string, error) {
|
||||
}
|
||||
|
||||
func getTasks(cfg *apiConfig) ([]task, error) {
|
||||
resp, err := cfg.client.GetAPIResponse("/tasks")
|
||||
resp, err := cfg.getAPIResponse("/tasks")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query dockerswarm api for tasks: %w", err)
|
||||
}
|
||||
|
||||
@@ -8,17 +8,17 @@ import (
|
||||
//
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config
|
||||
type SDConfig struct {
|
||||
Region string `yaml:"region"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
AccessKey string `yaml:"access_key"`
|
||||
SecretKey string `yaml:"secret_key"`
|
||||
Region string `yaml:"region,omitempty"`
|
||||
Endpoint string `yaml:"endpoint,omitempty"`
|
||||
AccessKey string `yaml:"access_key,omitempty"`
|
||||
SecretKey string `yaml:"secret_key,omitempty"`
|
||||
// TODO add support for Profile, not working atm
|
||||
Profile string `yaml:"profile"`
|
||||
RoleARN string `yaml:"role_arn"`
|
||||
Profile string `yaml:"profile,omitempty"`
|
||||
RoleARN string `yaml:"role_arn,omitempty"`
|
||||
// RefreshInterval time.Duration `yaml:"refresh_interval"`
|
||||
// refresh_interval is obtained from `-promscrape.ec2SDCheckInterval` command-line option.
|
||||
Port *int `yaml:"port"`
|
||||
Filters []Filter `yaml:"filters"`
|
||||
Port *int `yaml:"port,omitempty"`
|
||||
Filters []Filter `yaml:"filters,omitempty"`
|
||||
}
|
||||
|
||||
// Filter is ec2 filter.
|
||||
|
||||
@@ -104,7 +104,13 @@ type NetworkInterfaceSet struct {
|
||||
|
||||
// NetworkInterface represents NetworkInterface from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_InstanceNetworkInterface.html
|
||||
type NetworkInterface struct {
|
||||
SubnetID string `xml:"subnetId"`
|
||||
SubnetID string `xml:"subnetId"`
|
||||
IPv6AddressesSet Ipv6AddressesSet `xml:"ipv6AddressesSet"`
|
||||
}
|
||||
|
||||
// Ipv6AddressesSet represents ipv6AddressesSet from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_InstanceNetworkInterface.html
|
||||
type Ipv6AddressesSet struct {
|
||||
Items []string `xml:"item"`
|
||||
}
|
||||
|
||||
// TagSet represents TagSet from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Instance.html
|
||||
@@ -151,21 +157,27 @@ func (inst *Instance) appendTargetLabels(ms []map[string]string, ownerID string,
|
||||
"__meta_ec2_vpc_id": inst.VPCID,
|
||||
}
|
||||
if len(inst.VPCID) > 0 {
|
||||
// Deduplicate VPC Subnet IDs maintaining the order of the network interfaces returned by EC2.
|
||||
subnets := make([]string, 0, len(inst.NetworkInterfaceSet.Items))
|
||||
seenSubnets := make(map[string]bool, len(inst.NetworkInterfaceSet.Items))
|
||||
var ipv6Addrs []string
|
||||
for _, ni := range inst.NetworkInterfaceSet.Items {
|
||||
if len(ni.SubnetID) == 0 {
|
||||
continue
|
||||
}
|
||||
// Deduplicate VPC Subnet IDs maintaining the order of the network interfaces returned by EC2.
|
||||
if !seenSubnets[ni.SubnetID] {
|
||||
seenSubnets[ni.SubnetID] = true
|
||||
subnets = append(subnets, ni.SubnetID)
|
||||
}
|
||||
// Collect ipv6 addresses
|
||||
ipv6Addrs = append(ipv6Addrs, ni.IPv6AddressesSet.Items...)
|
||||
}
|
||||
// We surround the separated list with the separator as well. This way regular expressions
|
||||
// in relabeling rules don't have to consider tag positions.
|
||||
m["__meta_ec2_subnet_id"] = "," + strings.Join(subnets, ",") + ","
|
||||
if len(ipv6Addrs) > 0 {
|
||||
m["__meta_ec2_ipv6_addresses"] = "," + strings.Join(ipv6Addrs, ",") + ","
|
||||
}
|
||||
}
|
||||
for _, t := range inst.TagSet.Items {
|
||||
if len(t.Key) == 0 || len(t.Value) == 0 {
|
||||
|
||||
75
lib/promscrape/discovery/eureka/api.go
Normal file
75
lib/promscrape/discovery/eureka/api.go
Normal file
@@ -0,0 +1,75 @@
|
||||
package eureka
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
|
||||
)
|
||||
|
||||
var configMap = discoveryutils.NewConfigMap()
|
||||
|
||||
type apiConfig struct {
|
||||
client *discoveryutils.Client
|
||||
}
|
||||
|
||||
func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
||||
token := ""
|
||||
if sdc.Token != nil {
|
||||
token = *sdc.Token
|
||||
}
|
||||
var ba *promauth.BasicAuthConfig
|
||||
if len(sdc.Username) > 0 {
|
||||
ba = &promauth.BasicAuthConfig{
|
||||
Username: sdc.Username,
|
||||
Password: sdc.Password,
|
||||
}
|
||||
token = ""
|
||||
}
|
||||
ac, err := promauth.NewConfig(baseDir, ba, token, "", sdc.TLSConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse auth config: %w", err)
|
||||
}
|
||||
apiServer := sdc.Server
|
||||
if apiServer == "" {
|
||||
apiServer = "localhost:8080/eureka/v2"
|
||||
}
|
||||
if !strings.Contains(apiServer, "://") {
|
||||
scheme := sdc.Scheme
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
apiServer = scheme + "://" + apiServer
|
||||
}
|
||||
client, err := discoveryutils.NewClient(apiServer, ac)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err)
|
||||
}
|
||||
cfg := &apiConfig{
|
||||
client: client,
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
||||
v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return v.(*apiConfig), nil
|
||||
|
||||
}
|
||||
|
||||
func getAPIResponse(cfg *apiConfig, path string) ([]byte, error) {
|
||||
return cfg.client.GetAPIResponse(path)
|
||||
}
|
||||
|
||||
func parseAPIResponse(data []byte) (*applications, error) {
|
||||
var apps applications
|
||||
if err := xml.Unmarshal(data, &apps); err != nil {
|
||||
return nil, fmt.Errorf("failed parse eureka api response: %q, err: %w", data, err)
|
||||
}
|
||||
return &apps, nil
|
||||
}
|
||||
107
lib/promscrape/discovery/eureka/api_test.go
Normal file
107
lib/promscrape/discovery/eureka/api_test.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package eureka
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func Test_parseAPIResponse(t *testing.T) {
|
||||
type args struct {
|
||||
data []byte
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want *applications
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "parse ok 1 app with instance",
|
||||
args: args{
|
||||
data: []byte(`<applications>
|
||||
<versions__delta>1</versions__delta>
|
||||
<apps__hashcode>UP_1_</apps__hashcode>
|
||||
<application>
|
||||
<name>HELLO-NETFLIX-OSS</name>
|
||||
<instance>
|
||||
<hostName>98de25ebef42</hostName>
|
||||
<app>HELLO-NETFLIX-OSS</app>
|
||||
<ipAddr>10.10.0.3</ipAddr>
|
||||
<status>UP</status>
|
||||
<overriddenstatus>UNKNOWN</overriddenstatus>
|
||||
<port enabled="true">8080</port>
|
||||
<securePort enabled="false">443</securePort>
|
||||
<countryId>1</countryId>
|
||||
<dataCenterInfo class="com.netflix.appinfo.InstanceInfo$DefaultDataCenterInfo">
|
||||
<name>MyOwn</name>
|
||||
</dataCenterInfo>
|
||||
<leaseInfo>
|
||||
<renewalIntervalInSecs>30</renewalIntervalInSecs>
|
||||
<durationInSecs>90</durationInSecs>
|
||||
<registrationTimestamp>1605757726477</registrationTimestamp>
|
||||
<lastRenewalTimestamp>1605759135484</lastRenewalTimestamp>
|
||||
<evictionTimestamp>0</evictionTimestamp>
|
||||
<serviceUpTimestamp>1605757725913</serviceUpTimestamp>
|
||||
</leaseInfo>
|
||||
<metadata class="java.util.Collections$EmptyMap"/>
|
||||
<appGroupName>UNKNOWN</appGroupName>
|
||||
<homePageUrl>http://98de25ebef42:8080/</homePageUrl>
|
||||
<statusPageUrl>http://98de25ebef42:8080/Status</statusPageUrl>
|
||||
<healthCheckUrl>http://98de25ebef42:8080/healthcheck</healthCheckUrl>
|
||||
<vipAddress>HELLO-NETFLIX-OSS</vipAddress>
|
||||
<isCoordinatingDiscoveryServer>false</isCoordinatingDiscoveryServer>
|
||||
<lastUpdatedTimestamp>1605757726478</lastUpdatedTimestamp>
|
||||
<lastDirtyTimestamp>1605757725753</lastDirtyTimestamp>
|
||||
<actionType>ADDED</actionType>
|
||||
</instance>
|
||||
</application>
|
||||
</applications>`),
|
||||
},
|
||||
want: &applications{
|
||||
Applications: []Application{
|
||||
{
|
||||
Name: "HELLO-NETFLIX-OSS",
|
||||
Instances: []Instance{
|
||||
{
|
||||
HostName: "98de25ebef42",
|
||||
HomePageURL: "http://98de25ebef42:8080/",
|
||||
StatusPageURL: "http://98de25ebef42:8080/Status",
|
||||
HealthCheckURL: "http://98de25ebef42:8080/healthcheck",
|
||||
App: "HELLO-NETFLIX-OSS",
|
||||
IPAddr: "10.10.0.3",
|
||||
VipAddress: "HELLO-NETFLIX-OSS",
|
||||
SecureVipAddress: "",
|
||||
Status: "UP",
|
||||
Port: Port{
|
||||
Enabled: true,
|
||||
Port: 8080,
|
||||
},
|
||||
SecurePort: Port{
|
||||
Port: 443,
|
||||
},
|
||||
DataCenterInfo: DataCenterInfo{
|
||||
Name: "MyOwn",
|
||||
},
|
||||
Metadata: MetaData{},
|
||||
CountryID: 1,
|
||||
InstanceID: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseAPIResponse(tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("parseAPIResponse() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("unxpected response for parseAPIResponse() \ngot = %v, \nwant %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
150
lib/promscrape/discovery/eureka/eureka.go
Normal file
150
lib/promscrape/discovery/eureka/eureka.go
Normal file
@@ -0,0 +1,150 @@
|
||||
package eureka
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
)
|
||||
|
||||
const appsAPIPath = "/apps"
|
||||
|
||||
// SDConfig represents service discovery config for eureka.
|
||||
//
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka
|
||||
type SDConfig struct {
|
||||
Server string `yaml:"server,omitempty"`
|
||||
Token *string `yaml:"token"`
|
||||
Datacenter string `yaml:"datacenter"`
|
||||
Scheme string `yaml:"scheme,omitempty"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config,omitempty"`
|
||||
// RefreshInterval time.Duration `yaml:"refresh_interval"`
|
||||
// refresh_interval is obtained from `-promscrape.ec2SDCheckInterval` command-line option.
|
||||
Port *int `yaml:"port,omitempty"`
|
||||
}
|
||||
|
||||
type applications struct {
|
||||
Applications []Application `xml:"application"`
|
||||
}
|
||||
|
||||
// Application - eureka application https://github.com/Netflix/eureka/wiki/Eureka-REST-operations/
|
||||
type Application struct {
|
||||
Name string `xml:"name"`
|
||||
Instances []Instance `xml:"instance"`
|
||||
}
|
||||
|
||||
// Port - eureka instance port.
|
||||
type Port struct {
|
||||
Port int `xml:",chardata"`
|
||||
Enabled bool `xml:"enabled,attr"`
|
||||
}
|
||||
|
||||
// Instance - eureka instance https://github.com/Netflix/eureka/wiki/Eureka-REST-operations
|
||||
type Instance struct {
|
||||
HostName string `xml:"hostName"`
|
||||
HomePageURL string `xml:"homePageUrl"`
|
||||
StatusPageURL string `xml:"statusPageUrl"`
|
||||
HealthCheckURL string `xml:"healthCheckUrl"`
|
||||
App string `xml:"app"`
|
||||
IPAddr string `xml:"ipAddr"`
|
||||
VipAddress string `xml:"vipAddress"`
|
||||
SecureVipAddress string `xml:"secureVipAddress"`
|
||||
Status string `xml:"status"`
|
||||
Port Port `xml:"port"`
|
||||
SecurePort Port `xml:"securePort"`
|
||||
DataCenterInfo DataCenterInfo `xml:"dataCenterInfo"`
|
||||
Metadata MetaData `xml:"metadata"`
|
||||
CountryID int `xml:"countryId"`
|
||||
InstanceID string `xml:"instanceId"`
|
||||
}
|
||||
|
||||
// MetaData - eureka objects metadata.
|
||||
type MetaData struct {
|
||||
Items []Tag `xml:",any"`
|
||||
}
|
||||
|
||||
// Tag - eureka metadata tag - list of k/v values.
|
||||
type Tag struct {
|
||||
XMLName xml.Name
|
||||
Content string `xml:",innerxml"`
|
||||
}
|
||||
|
||||
// DataCenterInfo -eureka datacentre metadata
|
||||
type DataCenterInfo struct {
|
||||
Name string `xml:"name"`
|
||||
Metadata MetaData `xml:"metadata"`
|
||||
}
|
||||
|
||||
// GetLabels returns Eureka labels according to sdc.
|
||||
func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) {
|
||||
cfg, err := getAPIConfig(sdc, baseDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot get API config: %w", err)
|
||||
}
|
||||
data, err := getAPIResponse(cfg, appsAPIPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
apps, err := parseAPIResponse(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
port := 80
|
||||
if sdc.Port != nil {
|
||||
port = *sdc.Port
|
||||
}
|
||||
return addInstanceLabels(apps, port), nil
|
||||
}
|
||||
|
||||
func addInstanceLabels(apps *applications, port int) []map[string]string {
|
||||
var ms []map[string]string
|
||||
for _, app := range apps.Applications {
|
||||
for _, instance := range app.Instances {
|
||||
instancePort := port
|
||||
if instance.Port.Port != 0 {
|
||||
instancePort = instance.Port.Port
|
||||
}
|
||||
targetAddress := discoveryutils.JoinHostPort(instance.HostName, instancePort)
|
||||
m := map[string]string{
|
||||
"__address__": targetAddress,
|
||||
"instance": instance.InstanceID,
|
||||
"__meta_eureka_app_name": app.Name,
|
||||
"__meta_eureka_app_instance_hostname": instance.HostName,
|
||||
"__meta_eureka_app_instance_homepage_url": instance.HomePageURL,
|
||||
"__meta_eureka_app_instance_statuspage_url": instance.StatusPageURL,
|
||||
"__meta_eureka_app_instance_healthcheck_url": instance.HealthCheckURL,
|
||||
"__meta_eureka_app_instance_ip_addr": instance.IPAddr,
|
||||
"__meta_eureka_app_instance_vip_address": instance.VipAddress,
|
||||
"__meta_eureka_app_instance_secure_vip_address": instance.SecureVipAddress,
|
||||
"__meta_eureka_app_instance_status": instance.Status,
|
||||
"__meta_eureka_app_instance_country_id": strconv.Itoa(instance.CountryID),
|
||||
"__meta_eureka_app_instance_id": instance.InstanceID,
|
||||
}
|
||||
if instance.Port.Port != 0 {
|
||||
m["__meta_eureka_app_instance_port"] = strconv.Itoa(instance.Port.Port)
|
||||
m["__meta_eureka_app_instance_port_enabled"] = strconv.FormatBool(instance.Port.Enabled)
|
||||
}
|
||||
if instance.SecurePort.Port != 0 {
|
||||
m["__meta_eureka_app_instance_secure_port"] = strconv.Itoa(instance.SecurePort.Port)
|
||||
m["__meta_eureka_app_instance_secure_port_enabled"] = strconv.FormatBool(instance.SecurePort.Enabled)
|
||||
|
||||
}
|
||||
if len(instance.DataCenterInfo.Name) > 0 {
|
||||
m["__meta_eureka_app_instance_datacenterinfo_name"] = instance.DataCenterInfo.Name
|
||||
for _, tag := range instance.DataCenterInfo.Metadata.Items {
|
||||
m["__meta_eureka_app_instance_datacenterinfo_metadata_"+discoveryutils.SanitizeLabelName(tag.XMLName.Local)] = tag.Content
|
||||
}
|
||||
}
|
||||
for _, tag := range instance.Metadata.Items {
|
||||
m["__meta_eureka_app_instance_metadata_"+discoveryutils.SanitizeLabelName(tag.XMLName.Local)] = tag.Content
|
||||
}
|
||||
ms = append(ms, m)
|
||||
}
|
||||
}
|
||||
return ms
|
||||
}
|
||||
84
lib/promscrape/discovery/eureka/eureka_test.go
Normal file
84
lib/promscrape/discovery/eureka/eureka_test.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package eureka
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
|
||||
)
|
||||
|
||||
func Test_addInstanceLabels(t *testing.T) {
|
||||
type args struct {
|
||||
applications *applications
|
||||
port int
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want [][]prompbmarshal.Label
|
||||
}{
|
||||
{
|
||||
name: "1 application",
|
||||
args: args{
|
||||
port: 9100,
|
||||
applications: &applications{
|
||||
Applications: []Application{
|
||||
{
|
||||
Name: "test-app",
|
||||
Instances: []Instance{
|
||||
{
|
||||
Status: "Ok",
|
||||
HealthCheckURL: "some-url",
|
||||
HomePageURL: "some-home-url",
|
||||
StatusPageURL: "some-status-url",
|
||||
HostName: "host-1",
|
||||
IPAddr: "10.15.11.11",
|
||||
CountryID: 5,
|
||||
VipAddress: "10.15.11.11",
|
||||
InstanceID: "some-id",
|
||||
Metadata: MetaData{Items: []Tag{
|
||||
{
|
||||
Content: "value-1",
|
||||
XMLName: struct{ Space, Local string }{Local: "key-1"},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
want: [][]prompbmarshal.Label{
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "host-1:9100",
|
||||
"instance": "some-id",
|
||||
"__meta_eureka_app_instance_hostname": "host-1",
|
||||
"__meta_eureka_app_name": "test-app",
|
||||
"__meta_eureka_app_instance_healthcheck_url": "some-url",
|
||||
"__meta_eureka_app_instance_ip_addr": "10.15.11.11",
|
||||
"__meta_eureka_app_instance_vip_address": "10.15.11.11",
|
||||
"__meta_eureka_app_instance_secure_vip_address": "",
|
||||
"__meta_eureka_app_instance_country_id": "5",
|
||||
"__meta_eureka_app_instance_homepage_url": "some-home-url",
|
||||
"__meta_eureka_app_instance_statuspage_url": "some-status-url",
|
||||
"__meta_eureka_app_instance_id": "some-id",
|
||||
"__meta_eureka_app_instance_metadata_key_1": "value-1",
|
||||
"__meta_eureka_app_instance_status": "Ok",
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := addInstanceLabels(tt.args.applications, tt.args.port)
|
||||
var sortedLabelss [][]prompbmarshal.Label
|
||||
for _, labels := range got {
|
||||
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
|
||||
}
|
||||
if !reflect.DeepEqual(sortedLabelss, tt.want) {
|
||||
t.Fatalf("unexpected labels \ngot : %v, \nwant: %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -10,11 +10,11 @@ import (
|
||||
type SDConfig struct {
|
||||
Project string `yaml:"project"`
|
||||
Zone ZoneYAML `yaml:"zone"`
|
||||
Filter string `yaml:"filter"`
|
||||
Filter string `yaml:"filter,omitempty"`
|
||||
// RefreshInterval time.Duration `yaml:"refresh_interval"`
|
||||
// refresh_interval is obtained from `-promscrape.gceSDCheckInterval` command-line option.
|
||||
Port *int `yaml:"port"`
|
||||
TagSeparator *string `yaml:"tag_separator"`
|
||||
Port *int `yaml:"port,omitempty"`
|
||||
TagSeparator *string `yaml:"tag_separator,omitempty"`
|
||||
}
|
||||
|
||||
// ZoneYAML holds info about zones.
|
||||
|
||||
@@ -10,14 +10,14 @@ import (
|
||||
//
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config
|
||||
type SDConfig struct {
|
||||
APIServer string `yaml:"api_server"`
|
||||
APIServer string `yaml:"api_server,omitempty"`
|
||||
Role string `yaml:"role"`
|
||||
BasicAuth *promauth.BasicAuthConfig `yaml:"basic_auth"`
|
||||
BearerToken string `yaml:"bearer_token"`
|
||||
BearerTokenFile string `yaml:"bearer_token_file"`
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config"`
|
||||
Namespaces Namespaces `yaml:"namespaces"`
|
||||
Selectors []Selector `yaml:"selectors"`
|
||||
BasicAuth *promauth.BasicAuthConfig `yaml:"basic_auth,omitempty"`
|
||||
BearerToken string `yaml:"bearer_token,omitempty"`
|
||||
BearerTokenFile string `yaml:"bearer_token_file,omitempty"`
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config,omitempty"`
|
||||
Namespaces Namespaces `yaml:"namespaces,omitempty"`
|
||||
Selectors []Selector `yaml:"selectors,omitempty"`
|
||||
}
|
||||
|
||||
// Namespaces represents namespaces for SDConfig
|
||||
|
||||
@@ -10,25 +10,25 @@ import (
|
||||
//
|
||||
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config
|
||||
type SDConfig struct {
|
||||
IdentityEndpoint string `yaml:"identity_endpoint"`
|
||||
Username string `yaml:"username"`
|
||||
UserID string `yaml:"userid"`
|
||||
Password string `yaml:"password"`
|
||||
ProjectName string `yaml:"project_name"`
|
||||
ProjectID string `yaml:"project_id"`
|
||||
DomainName string `yaml:"domain_name"`
|
||||
DomainID string `yaml:"domain_id"`
|
||||
ApplicationCredentialName string `yaml:"application_credential_name"`
|
||||
ApplicationCredentialID string `yaml:"application_credential_id"`
|
||||
ApplicationCredentialSecret string `yaml:"application_credential_secret"`
|
||||
IdentityEndpoint string `yaml:"identity_endpoint,omitempty"`
|
||||
Username string `yaml:"username,omitempty"`
|
||||
UserID string `yaml:"userid,omitempty"`
|
||||
Password string `yaml:"password,omitempty"`
|
||||
ProjectName string `yaml:"project_name,omitempty"`
|
||||
ProjectID string `yaml:"project_id,omitempty"`
|
||||
DomainName string `yaml:"domain_name,omitempty"`
|
||||
DomainID string `yaml:"domain_id,omitempty"`
|
||||
ApplicationCredentialName string `yaml:"application_credential_name,omitempty"`
|
||||
ApplicationCredentialID string `yaml:"application_credential_id,omitempty"`
|
||||
ApplicationCredentialSecret string `yaml:"application_credential_secret,omitempty"`
|
||||
Role string `yaml:"role"`
|
||||
Region string `yaml:"region"`
|
||||
// RefreshInterval time.Duration `yaml:"refresh_interval"`
|
||||
// refresh_interval is obtained from `-promscrape.openstackSDCheckInterval` command-line option.
|
||||
Port int `yaml:"port"`
|
||||
AllTenants bool `yaml:"all_tenants"`
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config"`
|
||||
Availability string `yaml:"availability"`
|
||||
Port int `yaml:"port,omitempty"`
|
||||
AllTenants bool `yaml:"all_tenants,omitempty"`
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config,omitempty"`
|
||||
Availability string `yaml:"availability,omitempty"`
|
||||
}
|
||||
|
||||
// GetLabels returns gce labels according to sdc.
|
||||
|
||||
@@ -28,6 +28,9 @@ var (
|
||||
consulSDCheckInterval = flag.Duration("promscrape.consulSDCheckInterval", 30*time.Second, "Interval for checking for changes in consul. "+
|
||||
"This works only if `consul_sd_configs` is configured in '-promscrape.config' file. "+
|
||||
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details")
|
||||
eurekaSDCheckInterval = flag.Duration("promscrape.eurekaSDCheckInterval", 30*time.Second, "Interval for checking for changes in eureka. "+
|
||||
"This works only if `eureka_sd_configs` is configured in '-promscrape.config' file. "+
|
||||
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config for details")
|
||||
dnsSDCheckInterval = flag.Duration("promscrape.dnsSDCheckInterval", 30*time.Second, "Interval for checking for changes in dns. "+
|
||||
"This works only if `dns_sd_configs` is configured in '-promscrape.config' file. "+
|
||||
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config for details")
|
||||
@@ -99,6 +102,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
|
||||
scs.add("kubernetes_sd_configs", *kubernetesSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getKubernetesSDScrapeWork(swsPrev) })
|
||||
scs.add("openstack_sd_configs", *openstackSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getOpenStackSDScrapeWork(swsPrev) })
|
||||
scs.add("consul_sd_configs", *consulSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getConsulSDScrapeWork(swsPrev) })
|
||||
scs.add("eureka_sd_configs", *eurekaSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getEurekaSDScrapeWork(swsPrev) })
|
||||
scs.add("dns_sd_configs", *dnsSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getDNSSDScrapeWork(swsPrev) })
|
||||
scs.add("ec2_sd_configs", *ec2SDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getEC2SDScrapeWork(swsPrev) })
|
||||
scs.add("gce_sd_configs", *gceSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getGCESDScrapeWork(swsPrev) })
|
||||
@@ -319,7 +323,7 @@ func (sg *scraperGroup) update(sws []ScrapeWork) {
|
||||
|
||||
// Stop deleted scrapers, which are missing in sws.
|
||||
for key, sc := range sg.m {
|
||||
if swsMap[key] == nil {
|
||||
if _, ok := swsMap[key]; !ok {
|
||||
close(sc.stopCh)
|
||||
delete(sg.m, key)
|
||||
deletionsCount++
|
||||
|
||||
@@ -2,6 +2,7 @@ package promscrape
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -52,6 +53,9 @@ func statDial(addr string) (conn net.Conn, err error) {
|
||||
dialsTotal.Inc()
|
||||
if err != nil {
|
||||
dialErrors.Inc()
|
||||
if !netutil.TCP6Enabled() {
|
||||
err = fmt.Errorf("%w; try -enableTCP6 command-line flag", err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
conns.Inc()
|
||||
|
||||
@@ -64,7 +64,7 @@ func (tsm *targetStatusMap) Reset() {
|
||||
func (tsm *targetStatusMap) Register(sw *ScrapeWork) {
|
||||
tsm.mu.Lock()
|
||||
tsm.m[sw.ID] = targetStatus{
|
||||
sw: sw,
|
||||
sw: *sw,
|
||||
}
|
||||
tsm.mu.Unlock()
|
||||
}
|
||||
@@ -78,7 +78,7 @@ func (tsm *targetStatusMap) Unregister(sw *ScrapeWork) {
|
||||
func (tsm *targetStatusMap) Update(sw *ScrapeWork, group string, up bool, scrapeTime, scrapeDuration int64, err error) {
|
||||
tsm.mu.Lock()
|
||||
tsm.m[sw.ID] = targetStatus{
|
||||
sw: sw,
|
||||
sw: *sw,
|
||||
up: up,
|
||||
scrapeGroup: group,
|
||||
scrapeTime: scrapeTime,
|
||||
@@ -221,7 +221,7 @@ type jobStatus struct {
|
||||
}
|
||||
|
||||
type targetStatus struct {
|
||||
sw *ScrapeWork
|
||||
sw ScrapeWork
|
||||
up bool
|
||||
scrapeGroup string
|
||||
scrapeTime int64
|
||||
|
||||
@@ -83,7 +83,7 @@ func parseRows(sc *scanner, dst []Row, tags []Tag, metrics []metric, cds []Colum
|
||||
tagsLen := len(tags)
|
||||
for sc.NextColumn() {
|
||||
if col >= uint(len(cds)) {
|
||||
// Skip superflouous column.
|
||||
// Skip superfluous column.
|
||||
continue
|
||||
}
|
||||
cd := &cds[col]
|
||||
|
||||
@@ -165,7 +165,7 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
// Superflouos columns
|
||||
// Superfluous columns
|
||||
f("1:metric:foo", `123,456,foo,bar`, []Row{
|
||||
{
|
||||
Metric: "foo",
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
@@ -24,7 +23,6 @@ var (
|
||||
// ParseStream parses csv from req and calls callback for the parsed rows.
|
||||
//
|
||||
// The callback can be called concurrently multiple times for streamed data from req.
|
||||
// The callback can be called after ParseStream returns.
|
||||
//
|
||||
// callback shouldn't hold rows after returning.
|
||||
func ParseStream(req *http.Request, callback func(rows []Row) error) error {
|
||||
@@ -47,12 +45,26 @@ func ParseStream(req *http.Request, callback func(rows []Row) error) error {
|
||||
defer putStreamContext(ctx)
|
||||
for ctx.Read() {
|
||||
uw := getUnmarshalWork()
|
||||
uw.callback = callback
|
||||
uw.callback = func(rows []Row) {
|
||||
if err := callback(rows); err != nil {
|
||||
ctx.callbackErrLock.Lock()
|
||||
if ctx.callbackErr == nil {
|
||||
ctx.callbackErr = fmt.Errorf("error when processing imported data: %w", err)
|
||||
}
|
||||
ctx.callbackErrLock.Unlock()
|
||||
}
|
||||
ctx.wg.Done()
|
||||
}
|
||||
uw.cds = cds
|
||||
uw.reqBuf, ctx.reqBuf = ctx.reqBuf, uw.reqBuf
|
||||
ctx.wg.Add(1)
|
||||
common.ScheduleUnmarshalWork(uw)
|
||||
}
|
||||
return ctx.Error()
|
||||
ctx.wg.Wait()
|
||||
if err := ctx.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.callbackErr
|
||||
}
|
||||
|
||||
func (ctx *streamContext) Read() bool {
|
||||
@@ -82,6 +94,10 @@ type streamContext struct {
|
||||
reqBuf []byte
|
||||
tailBuf []byte
|
||||
err error
|
||||
|
||||
wg sync.WaitGroup
|
||||
callbackErrLock sync.Mutex
|
||||
callbackErr error
|
||||
}
|
||||
|
||||
func (ctx *streamContext) Error() error {
|
||||
@@ -96,6 +112,7 @@ func (ctx *streamContext) reset() {
|
||||
ctx.reqBuf = ctx.reqBuf[:0]
|
||||
ctx.tailBuf = ctx.tailBuf[:0]
|
||||
ctx.err = nil
|
||||
ctx.callbackErr = nil
|
||||
}
|
||||
|
||||
func getStreamContext(r io.Reader) *streamContext {
|
||||
@@ -129,7 +146,7 @@ var streamContextPoolCh = make(chan *streamContext, runtime.GOMAXPROCS(-1))
|
||||
|
||||
type unmarshalWork struct {
|
||||
rows Rows
|
||||
callback func(rows []Row) error
|
||||
callback func(rows []Row)
|
||||
cds []ColumnDescriptor
|
||||
reqBuf []byte
|
||||
}
|
||||
@@ -164,11 +181,7 @@ func (uw *unmarshalWork) Unmarshal() {
|
||||
}
|
||||
}
|
||||
|
||||
if err := uw.callback(rows); err != nil {
|
||||
logger.Errorf("error when processing imported data: %s", err)
|
||||
putUnmarshalWork(uw)
|
||||
return
|
||||
}
|
||||
uw.callback(rows)
|
||||
putUnmarshalWork(uw)
|
||||
}
|
||||
|
||||
|
||||
@@ -55,6 +55,33 @@ func (r *Row) reset() {
|
||||
r.Timestamp = 0
|
||||
}
|
||||
|
||||
// UnmarshalMetricAndTags unmarshals metric and optional tags from s.
|
||||
func (r *Row) UnmarshalMetricAndTags(s string, tagsPool []Tag) ([]Tag, error) {
|
||||
if strings.Contains(s, " ") {
|
||||
return tagsPool, fmt.Errorf("unexpected whitespace found in %q", s)
|
||||
}
|
||||
n := strings.IndexByte(s, ';')
|
||||
if n < 0 {
|
||||
// No tags
|
||||
r.Metric = s
|
||||
} else {
|
||||
// Tags found
|
||||
r.Metric = s[:n]
|
||||
tagsStart := len(tagsPool)
|
||||
var err error
|
||||
tagsPool, err = unmarshalTags(tagsPool, s[n+1:])
|
||||
if err != nil {
|
||||
return tagsPool, fmt.Errorf("cannot umarshal tags: %w", err)
|
||||
}
|
||||
tags := tagsPool[tagsStart:]
|
||||
r.Tags = tags[:len(tags):len(tags)]
|
||||
}
|
||||
if len(r.Metric) == 0 {
|
||||
return tagsPool, fmt.Errorf("metric cannot be empty")
|
||||
}
|
||||
return tagsPool, nil
|
||||
}
|
||||
|
||||
func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) {
|
||||
r.reset()
|
||||
n := strings.IndexByte(s, ' ')
|
||||
@@ -64,24 +91,9 @@ func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) {
|
||||
metricAndTags := s[:n]
|
||||
tail := s[n+1:]
|
||||
|
||||
n = strings.IndexByte(metricAndTags, ';')
|
||||
if n < 0 {
|
||||
// No tags
|
||||
r.Metric = metricAndTags
|
||||
} else {
|
||||
// Tags found
|
||||
r.Metric = metricAndTags[:n]
|
||||
tagsStart := len(tagsPool)
|
||||
var err error
|
||||
tagsPool, err = unmarshalTags(tagsPool, metricAndTags[n+1:])
|
||||
if err != nil {
|
||||
return tagsPool, fmt.Errorf("cannot umarshal tags: %w", err)
|
||||
}
|
||||
tags := tagsPool[tagsStart:]
|
||||
r.Tags = tags[:len(tags):len(tags)]
|
||||
}
|
||||
if len(r.Metric) == 0 {
|
||||
return tagsPool, fmt.Errorf("metric cannot be empty")
|
||||
tagsPool, err := r.UnmarshalMetricAndTags(metricAndTags, tagsPool)
|
||||
if err != nil {
|
||||
return tagsPool, err
|
||||
}
|
||||
|
||||
n = strings.IndexByte(tail, ' ')
|
||||
|
||||
@@ -7,6 +7,57 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUnmarshalMetricAndTagsFailure(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
var r Row
|
||||
_, err := r.UnmarshalMetricAndTags(s, nil)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error for UnmarshalMetricAndTags(%q)", s)
|
||||
}
|
||||
}
|
||||
f("")
|
||||
f(";foo=bar")
|
||||
f(" ")
|
||||
f("foo;bar")
|
||||
f("foo ;bar=baz")
|
||||
f("f oo;bar=baz")
|
||||
f("foo;bar=baz ")
|
||||
f("foo;bar= baz")
|
||||
f("foo;bar=b az")
|
||||
f("foo;b ar=baz")
|
||||
}
|
||||
|
||||
func TestUnmarshalMetricAndTagsSuccess(t *testing.T) {
|
||||
f := func(s string, rExpected *Row) {
|
||||
t.Helper()
|
||||
var r Row
|
||||
_, err := r.UnmarshalMetricAndTags(s, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error in UnmarshalMetricAndTags(%q): %s", s, err)
|
||||
}
|
||||
if !reflect.DeepEqual(&r, rExpected) {
|
||||
t.Fatalf("unexpected row;\ngot\n%+v\nwant\n%+v", &r, rExpected)
|
||||
}
|
||||
}
|
||||
f("foo", &Row{
|
||||
Metric: "foo",
|
||||
})
|
||||
f("foo;bar=123;baz=aabb", &Row{
|
||||
Metric: "foo",
|
||||
Tags: []Tag{
|
||||
{
|
||||
Key: "bar",
|
||||
Value: "123",
|
||||
},
|
||||
{
|
||||
Key: "baz",
|
||||
Value: "aabb",
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestRowsUnmarshalFailure(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
@@ -200,7 +251,7 @@ func Test_streamContext_Read(t *testing.T) {
|
||||
}
|
||||
uw := getUnmarshalWork()
|
||||
callbackCalls := 0
|
||||
uw.callback = func(rows []Row) error {
|
||||
uw.callback = func(rows []Row) {
|
||||
callbackCalls++
|
||||
if len(rows) != len(rowsExpected.Rows) {
|
||||
t.Fatalf("different len of expected rows;\ngot\n%+v;\nwant\n%+v", rows, rowsExpected.Rows)
|
||||
@@ -208,7 +259,6 @@ func Test_streamContext_Read(t *testing.T) {
|
||||
if !reflect.DeepEqual(rows, rowsExpected.Rows) {
|
||||
t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows, rowsExpected.Rows)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
uw.reqBuf = append(uw.reqBuf[:0], ctx.reqBuf...)
|
||||
uw.Unmarshal()
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
@@ -24,7 +23,6 @@ var (
|
||||
// ParseStream parses Graphite lines from r and calls callback for the parsed rows.
|
||||
//
|
||||
// The callback can be called concurrently multiple times for streamed data from r.
|
||||
// The callback can be called after ParseStream returns.
|
||||
//
|
||||
// callback shouldn't hold rows after returning.
|
||||
func ParseStream(r io.Reader, callback func(rows []Row) error) error {
|
||||
@@ -33,11 +31,25 @@ func ParseStream(r io.Reader, callback func(rows []Row) error) error {
|
||||
|
||||
for ctx.Read() {
|
||||
uw := getUnmarshalWork()
|
||||
uw.callback = callback
|
||||
uw.callback = func(rows []Row) {
|
||||
if err := callback(rows); err != nil {
|
||||
ctx.callbackErrLock.Lock()
|
||||
if ctx.callbackErr == nil {
|
||||
ctx.callbackErr = fmt.Errorf("error when processing imported data: %w", err)
|
||||
}
|
||||
ctx.callbackErrLock.Unlock()
|
||||
}
|
||||
ctx.wg.Done()
|
||||
}
|
||||
uw.reqBuf, ctx.reqBuf = ctx.reqBuf, uw.reqBuf
|
||||
ctx.wg.Add(1)
|
||||
common.ScheduleUnmarshalWork(uw)
|
||||
}
|
||||
return ctx.Error()
|
||||
ctx.wg.Wait()
|
||||
if err := ctx.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.callbackErr
|
||||
}
|
||||
|
||||
func (ctx *streamContext) Read() bool {
|
||||
@@ -61,6 +73,10 @@ type streamContext struct {
|
||||
reqBuf []byte
|
||||
tailBuf []byte
|
||||
err error
|
||||
|
||||
wg sync.WaitGroup
|
||||
callbackErrLock sync.Mutex
|
||||
callbackErr error
|
||||
}
|
||||
|
||||
func (ctx *streamContext) Error() error {
|
||||
@@ -75,6 +91,7 @@ func (ctx *streamContext) reset() {
|
||||
ctx.reqBuf = ctx.reqBuf[:0]
|
||||
ctx.tailBuf = ctx.tailBuf[:0]
|
||||
ctx.err = nil
|
||||
ctx.callbackErr = nil
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -114,7 +131,7 @@ var streamContextPoolCh = make(chan *streamContext, runtime.GOMAXPROCS(-1))
|
||||
|
||||
type unmarshalWork struct {
|
||||
rows Rows
|
||||
callback func(rows []Row) error
|
||||
callback func(rows []Row)
|
||||
reqBuf []byte
|
||||
}
|
||||
|
||||
@@ -152,11 +169,7 @@ func (uw *unmarshalWork) Unmarshal() {
|
||||
}
|
||||
}
|
||||
|
||||
if err := uw.callback(rows); err != nil {
|
||||
logger.Errorf("error when processing imported data: %s", err)
|
||||
putUnmarshalWork(uw)
|
||||
return
|
||||
}
|
||||
uw.callback(rows)
|
||||
putUnmarshalWork(uw)
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
@@ -25,7 +24,6 @@ var (
|
||||
// ParseStream parses r with the given args and calls callback for the parsed rows.
|
||||
//
|
||||
// The callback can be called concurrently multiple times for streamed data from r.
|
||||
// The callback can be called after ParseStream returns.
|
||||
//
|
||||
// callback shouldn't hold rows after returning.
|
||||
func ParseStream(r io.Reader, isGzipped bool, precision, db string, callback func(db string, rows []Row) error) error {
|
||||
@@ -59,13 +57,27 @@ func ParseStream(r io.Reader, isGzipped bool, precision, db string, callback fun
|
||||
defer putStreamContext(ctx)
|
||||
for ctx.Read() {
|
||||
uw := getUnmarshalWork()
|
||||
uw.callback = callback
|
||||
uw.callback = func(db string, rows []Row) {
|
||||
if err := callback(db, rows); err != nil {
|
||||
ctx.callbackErrLock.Lock()
|
||||
if ctx.callbackErr == nil {
|
||||
ctx.callbackErr = fmt.Errorf("error when processing imported data: %w", err)
|
||||
}
|
||||
ctx.callbackErrLock.Unlock()
|
||||
}
|
||||
ctx.wg.Done()
|
||||
}
|
||||
uw.db = db
|
||||
uw.tsMultiplier = tsMultiplier
|
||||
uw.reqBuf, ctx.reqBuf = ctx.reqBuf, uw.reqBuf
|
||||
ctx.wg.Add(1)
|
||||
common.ScheduleUnmarshalWork(uw)
|
||||
}
|
||||
return ctx.Error()
|
||||
ctx.wg.Wait()
|
||||
if err := ctx.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.callbackErr
|
||||
}
|
||||
|
||||
func (ctx *streamContext) Read() bool {
|
||||
@@ -95,6 +107,10 @@ type streamContext struct {
|
||||
reqBuf []byte
|
||||
tailBuf []byte
|
||||
err error
|
||||
|
||||
wg sync.WaitGroup
|
||||
callbackErrLock sync.Mutex
|
||||
callbackErr error
|
||||
}
|
||||
|
||||
func (ctx *streamContext) Error() error {
|
||||
@@ -109,6 +125,7 @@ func (ctx *streamContext) reset() {
|
||||
ctx.reqBuf = ctx.reqBuf[:0]
|
||||
ctx.tailBuf = ctx.tailBuf[:0]
|
||||
ctx.err = nil
|
||||
ctx.callbackErr = nil
|
||||
}
|
||||
|
||||
func getStreamContext(r io.Reader) *streamContext {
|
||||
@@ -142,7 +159,7 @@ var streamContextPoolCh = make(chan *streamContext, runtime.GOMAXPROCS(-1))
|
||||
|
||||
type unmarshalWork struct {
|
||||
rows Rows
|
||||
callback func(db string, rows []Row) error
|
||||
callback func(db string, rows []Row)
|
||||
db string
|
||||
tsMultiplier int64
|
||||
reqBuf []byte
|
||||
@@ -195,11 +212,7 @@ func (uw *unmarshalWork) Unmarshal() {
|
||||
}
|
||||
}
|
||||
|
||||
if err := uw.callback(uw.db, rows); err != nil {
|
||||
logger.Errorf("error when processing imported data: %s", err)
|
||||
putUnmarshalWork(uw)
|
||||
return
|
||||
}
|
||||
uw.callback(uw.db, rows)
|
||||
putUnmarshalWork(uw)
|
||||
}
|
||||
|
||||
|
||||
@@ -18,10 +18,8 @@ import (
|
||||
// ParseStream parses /api/v1/import/native lines from req and calls callback for parsed blocks.
|
||||
//
|
||||
// The callback can be called concurrently multiple times for streamed data from req.
|
||||
// The callback can be called after ParseStream returns.
|
||||
//
|
||||
// callback shouldn't hold block after returning.
|
||||
// callback can be called in parallel from multiple concurrent goroutines.
|
||||
func ParseStream(req *http.Request, callback func(block *Block) error) error {
|
||||
r := req.Body
|
||||
if req.Header.Get("Content-Encoding") == "gzip" {
|
||||
@@ -47,30 +45,49 @@ func ParseStream(req *http.Request, callback func(block *Block) error) error {
|
||||
|
||||
// Read native blocks and feed workers with work.
|
||||
sizeBuf := make([]byte, 4)
|
||||
var wg sync.WaitGroup
|
||||
var (
|
||||
callbackErrLock sync.Mutex
|
||||
callbackErr error
|
||||
)
|
||||
for {
|
||||
uw := getUnmarshalWork()
|
||||
uw.tr = tr
|
||||
uw.callback = callback
|
||||
uw.callback = func(block *Block) {
|
||||
if err := callback(block); err != nil {
|
||||
processErrors.Inc()
|
||||
callbackErrLock.Lock()
|
||||
if callbackErr == nil {
|
||||
callbackErr = fmt.Errorf("error when processing native block: %w", err)
|
||||
}
|
||||
callbackErrLock.Unlock()
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
|
||||
// Read uw.metricNameBuf
|
||||
if _, err := io.ReadFull(br, sizeBuf); err != nil {
|
||||
if err == io.EOF {
|
||||
// End of stream
|
||||
putUnmarshalWork(uw)
|
||||
return nil
|
||||
wg.Wait()
|
||||
return callbackErr
|
||||
}
|
||||
readErrors.Inc()
|
||||
wg.Wait()
|
||||
return fmt.Errorf("cannot read metricName size: %w", err)
|
||||
}
|
||||
readCalls.Inc()
|
||||
bufSize := encoding.UnmarshalUint32(sizeBuf)
|
||||
if bufSize > 1024*1024 {
|
||||
parseErrors.Inc()
|
||||
wg.Wait()
|
||||
return fmt.Errorf("too big metricName size; got %d; shouldn't exceed %d", bufSize, 1024*1024)
|
||||
}
|
||||
uw.metricNameBuf = bytesutil.Resize(uw.metricNameBuf, int(bufSize))
|
||||
if _, err := io.ReadFull(br, uw.metricNameBuf); err != nil {
|
||||
readErrors.Inc()
|
||||
wg.Wait()
|
||||
return fmt.Errorf("cannot read metricName with size %d bytes: %w", bufSize, err)
|
||||
}
|
||||
readCalls.Inc()
|
||||
@@ -78,22 +95,26 @@ func ParseStream(req *http.Request, callback func(block *Block) error) error {
|
||||
// Read uw.blockBuf
|
||||
if _, err := io.ReadFull(br, sizeBuf); err != nil {
|
||||
readErrors.Inc()
|
||||
wg.Wait()
|
||||
return fmt.Errorf("cannot read native block size: %w", err)
|
||||
}
|
||||
readCalls.Inc()
|
||||
bufSize = encoding.UnmarshalUint32(sizeBuf)
|
||||
if bufSize > 1024*1024 {
|
||||
parseErrors.Inc()
|
||||
wg.Wait()
|
||||
return fmt.Errorf("too big native block size; got %d; shouldn't exceed %d", bufSize, 1024*1024)
|
||||
}
|
||||
uw.blockBuf = bytesutil.Resize(uw.blockBuf, int(bufSize))
|
||||
if _, err := io.ReadFull(br, uw.blockBuf); err != nil {
|
||||
readErrors.Inc()
|
||||
wg.Wait()
|
||||
return fmt.Errorf("cannot read native block with size %d bytes: %w", bufSize, err)
|
||||
}
|
||||
readCalls.Inc()
|
||||
blocksRead.Inc()
|
||||
|
||||
wg.Add(1)
|
||||
common.ScheduleUnmarshalWork(uw)
|
||||
}
|
||||
}
|
||||
@@ -123,7 +144,7 @@ var (
|
||||
|
||||
type unmarshalWork struct {
|
||||
tr storage.TimeRange
|
||||
callback func(block *Block) error
|
||||
callback func(block *Block)
|
||||
metricNameBuf []byte
|
||||
blockBuf []byte
|
||||
block Block
|
||||
@@ -144,12 +165,7 @@ func (uw *unmarshalWork) Unmarshal() {
|
||||
putUnmarshalWork(uw)
|
||||
return
|
||||
}
|
||||
if err := uw.callback(&uw.block); err != nil {
|
||||
processErrors.Inc()
|
||||
logger.Errorf("error when processing native block: %s", err)
|
||||
putUnmarshalWork(uw)
|
||||
return
|
||||
}
|
||||
uw.callback(&uw.block)
|
||||
putUnmarshalWork(uw)
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
@@ -24,7 +23,6 @@ var (
|
||||
// ParseStream parses OpenTSDB lines from r and calls callback for the parsed rows.
|
||||
//
|
||||
// The callback can be called concurrently multiple times for streamed data from r.
|
||||
// The callback can be called after ParseStream returns.
|
||||
//
|
||||
// callback shouldn't hold rows after returning.
|
||||
func ParseStream(r io.Reader, callback func(rows []Row) error) error {
|
||||
@@ -32,11 +30,25 @@ func ParseStream(r io.Reader, callback func(rows []Row) error) error {
|
||||
defer putStreamContext(ctx)
|
||||
for ctx.Read() {
|
||||
uw := getUnmarshalWork()
|
||||
uw.callback = callback
|
||||
uw.callback = func(rows []Row) {
|
||||
if err := callback(rows); err != nil {
|
||||
ctx.callbackErrLock.Lock()
|
||||
if ctx.callbackErr == nil {
|
||||
ctx.callbackErr = fmt.Errorf("error when processing imported data: %w", err)
|
||||
}
|
||||
ctx.callbackErrLock.Unlock()
|
||||
}
|
||||
ctx.wg.Done()
|
||||
}
|
||||
uw.reqBuf, ctx.reqBuf = ctx.reqBuf, uw.reqBuf
|
||||
ctx.wg.Add(1)
|
||||
common.ScheduleUnmarshalWork(uw)
|
||||
}
|
||||
return ctx.Error()
|
||||
ctx.wg.Wait()
|
||||
if err := ctx.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.callbackErr
|
||||
}
|
||||
|
||||
func (ctx *streamContext) Read() bool {
|
||||
@@ -60,6 +72,10 @@ type streamContext struct {
|
||||
reqBuf []byte
|
||||
tailBuf []byte
|
||||
err error
|
||||
|
||||
wg sync.WaitGroup
|
||||
callbackErrLock sync.Mutex
|
||||
callbackErr error
|
||||
}
|
||||
|
||||
func (ctx *streamContext) Error() error {
|
||||
@@ -74,6 +90,7 @@ func (ctx *streamContext) reset() {
|
||||
ctx.reqBuf = ctx.reqBuf[:0]
|
||||
ctx.tailBuf = ctx.tailBuf[:0]
|
||||
ctx.err = nil
|
||||
ctx.callbackErr = nil
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -113,7 +130,7 @@ var streamContextPoolCh = make(chan *streamContext, runtime.GOMAXPROCS(-1))
|
||||
|
||||
type unmarshalWork struct {
|
||||
rows Rows
|
||||
callback func(rows []Row) error
|
||||
callback func(rows []Row)
|
||||
reqBuf []byte
|
||||
}
|
||||
|
||||
@@ -151,11 +168,7 @@ func (uw *unmarshalWork) Unmarshal() {
|
||||
}
|
||||
}
|
||||
|
||||
if err := uw.callback(rows); err != nil {
|
||||
logger.Errorf("error when processing imported data: %s", err)
|
||||
putUnmarshalWork(uw)
|
||||
return
|
||||
}
|
||||
uw.callback(rows)
|
||||
putUnmarshalWork(uw)
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
@@ -27,7 +26,6 @@ var (
|
||||
// ParseStream parses OpenTSDB http lines from req and calls callback for the parsed rows.
|
||||
//
|
||||
// The callback can be called concurrently multiple times for streamed data from req.
|
||||
// The callback can be called after ParseStream returns.
|
||||
//
|
||||
// callback shouldn't hold rows after returning.
|
||||
func ParseStream(req *http.Request, callback func(rows []Row) error) error {
|
||||
@@ -58,10 +56,50 @@ func ParseStream(req *http.Request, callback func(rows []Row) error) error {
|
||||
return fmt.Errorf("too big HTTP OpenTSDB request; mustn't exceed `-opentsdbhttp.maxInsertRequestSize=%d` bytes", maxInsertRequestSize.N)
|
||||
}
|
||||
|
||||
uw := getUnmarshalWork()
|
||||
uw.callback = callback
|
||||
uw.reqBuf, ctx.reqBuf.B = ctx.reqBuf.B, uw.reqBuf
|
||||
common.ScheduleUnmarshalWork(uw)
|
||||
// Process the request synchronously, since there is no sense in processing a single request asynchronously.
|
||||
// Sync code is easier to read and understand.
|
||||
p := getJSONParser()
|
||||
defer putJSONParser(p)
|
||||
v, err := p.ParseBytes(ctx.reqBuf.B)
|
||||
if err != nil {
|
||||
unmarshalErrors.Inc()
|
||||
return fmt.Errorf("cannot parse HTTP OpenTSDB json: %w", err)
|
||||
}
|
||||
rs := getRows()
|
||||
defer putRows(rs)
|
||||
rs.Unmarshal(v)
|
||||
rows := rs.Rows
|
||||
rowsRead.Add(len(rows))
|
||||
|
||||
// Fill in missing timestamps
|
||||
currentTimestamp := int64(fasttime.UnixTimestamp())
|
||||
for i := range rows {
|
||||
r := &rows[i]
|
||||
if r.Timestamp == 0 {
|
||||
r.Timestamp = currentTimestamp
|
||||
}
|
||||
}
|
||||
|
||||
// Convert timestamps in seconds to milliseconds if needed.
|
||||
// See http://opentsdb.net/docs/javadoc/net/opentsdb/core/Const.html#SECOND_MASK
|
||||
for i := range rows {
|
||||
r := &rows[i]
|
||||
if r.Timestamp&secondMask == 0 {
|
||||
r.Timestamp *= 1e3
|
||||
}
|
||||
}
|
||||
|
||||
// Trim timestamps if required.
|
||||
if tsTrim := trimTimestamp.Milliseconds(); tsTrim > 1 {
|
||||
for i := range rows {
|
||||
row := &rows[i]
|
||||
row.Timestamp -= row.Timestamp % tsTrim
|
||||
}
|
||||
}
|
||||
|
||||
if err := callback(rows); err != nil {
|
||||
return fmt.Errorf("error when processing imported data: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -113,77 +151,17 @@ func putStreamContext(ctx *streamContext) {
|
||||
var streamContextPool sync.Pool
|
||||
var streamContextPoolCh = make(chan *streamContext, runtime.GOMAXPROCS(-1))
|
||||
|
||||
type unmarshalWork struct {
|
||||
rows Rows
|
||||
callback func(rows []Row) error
|
||||
reqBuf []byte
|
||||
}
|
||||
|
||||
func (uw *unmarshalWork) reset() {
|
||||
uw.rows.Reset()
|
||||
uw.callback = nil
|
||||
uw.reqBuf = uw.reqBuf[:0]
|
||||
}
|
||||
|
||||
// Unmarshal implements common.UnmarshalWork
|
||||
func (uw *unmarshalWork) Unmarshal() {
|
||||
p := getJSONParser()
|
||||
defer putJSONParser(p)
|
||||
v, err := p.ParseBytes(uw.reqBuf)
|
||||
if err != nil {
|
||||
unmarshalErrors.Inc()
|
||||
logger.Errorf("cannot parse HTTP OpenTSDB json: %s", err)
|
||||
return
|
||||
}
|
||||
uw.rows.Unmarshal(v)
|
||||
rows := uw.rows.Rows
|
||||
rowsRead.Add(len(rows))
|
||||
|
||||
// Fill in missing timestamps
|
||||
currentTimestamp := int64(fasttime.UnixTimestamp())
|
||||
for i := range rows {
|
||||
r := &rows[i]
|
||||
if r.Timestamp == 0 {
|
||||
r.Timestamp = currentTimestamp
|
||||
}
|
||||
}
|
||||
|
||||
// Convert timestamps in seconds to milliseconds if needed.
|
||||
// See http://opentsdb.net/docs/javadoc/net/opentsdb/core/Const.html#SECOND_MASK
|
||||
for i := range rows {
|
||||
r := &rows[i]
|
||||
if r.Timestamp&secondMask == 0 {
|
||||
r.Timestamp *= 1e3
|
||||
}
|
||||
}
|
||||
|
||||
// Trim timestamps if required.
|
||||
if tsTrim := trimTimestamp.Milliseconds(); tsTrim > 1 {
|
||||
for i := range rows {
|
||||
row := &rows[i]
|
||||
row.Timestamp -= row.Timestamp % tsTrim
|
||||
}
|
||||
}
|
||||
|
||||
if err := uw.callback(rows); err != nil {
|
||||
logger.Errorf("error when processing imported data: %s", err)
|
||||
putUnmarshalWork(uw)
|
||||
return
|
||||
}
|
||||
putUnmarshalWork(uw)
|
||||
}
|
||||
|
||||
func getUnmarshalWork() *unmarshalWork {
|
||||
v := unmarshalWorkPool.Get()
|
||||
func getRows() *Rows {
|
||||
v := rowsPool.Get()
|
||||
if v == nil {
|
||||
return &unmarshalWork{}
|
||||
return &Rows{}
|
||||
}
|
||||
return v.(*unmarshalWork)
|
||||
return v.(*Rows)
|
||||
}
|
||||
|
||||
func putUnmarshalWork(uw *unmarshalWork) {
|
||||
uw.reset()
|
||||
unmarshalWorkPool.Put(uw)
|
||||
func putRows(rs *Rows) {
|
||||
rs.Reset()
|
||||
rowsPool.Put(rs)
|
||||
}
|
||||
|
||||
var unmarshalWorkPool sync.Pool
|
||||
var rowsPool sync.Pool
|
||||
|
||||
@@ -70,6 +70,14 @@ func (r *Row) reset() {
|
||||
r.Timestamp = 0
|
||||
}
|
||||
|
||||
func skipTrailingComment(s string) string {
|
||||
n := strings.IndexByte(s, '#')
|
||||
if n < 0 {
|
||||
return s
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
func skipLeadingWhitespace(s string) string {
|
||||
// Prometheus treats ' ' and '\t' as whitespace
|
||||
// according to https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-format-details
|
||||
@@ -133,6 +141,7 @@ func (r *Row) unmarshal(s string, tagsPool []Tag, noEscapes bool) ([]Tag, error)
|
||||
return tagsPool, fmt.Errorf("metric cannot be empty")
|
||||
}
|
||||
s = skipLeadingWhitespace(s)
|
||||
s = skipTrailingComment(s)
|
||||
if len(s) == 0 {
|
||||
return tagsPool, fmt.Errorf("value cannot be empty")
|
||||
}
|
||||
@@ -146,17 +155,21 @@ func (r *Row) unmarshal(s string, tagsPool []Tag, noEscapes bool) ([]Tag, error)
|
||||
r.Value = v
|
||||
return tagsPool, nil
|
||||
}
|
||||
// There is timestamp.
|
||||
// There is a timestamp.
|
||||
v, err := fastfloat.Parse(s[:n])
|
||||
if err != nil {
|
||||
return tagsPool, fmt.Errorf("cannot parse value %q: %w", s[:n], err)
|
||||
}
|
||||
r.Value = v
|
||||
s = skipLeadingWhitespace(s[n+1:])
|
||||
if len(s) == 0 {
|
||||
// There is no timestamp - just a whitespace after the value.
|
||||
return tagsPool, nil
|
||||
}
|
||||
ts, err := fastfloat.ParseInt64(s)
|
||||
if err != nil {
|
||||
return tagsPool, fmt.Errorf("cannot parse timestamp %q: %w", s, err)
|
||||
}
|
||||
r.Value = v
|
||||
r.Timestamp = ts
|
||||
return tagsPool, nil
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user