improve documentation

Add `last` value to VMUI graph legend statistics.

`last` is intentionally shown only in the legend, not in the tooltip.
The tooltip has limited space and already shows the value for the
hovered timestamp, so the latest value can still be checked by hovering
over the last point.

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10759
PR https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11110

<img width="371" height="60" alt="image"
src="https://github.com/user-attachments/assets/b57ed7e3-a6a1-44ac-ab60-89911f71eb6b"
/>


<!-- This is an auto-generated description by cubic. -->
<a
href="https://cubic.dev/pr/VictoriaMetrics/VictoriaMetrics/pull/11110?utm_source=github"
target="_blank" rel="noopener noreferrer"
data-no-image-dialog="true"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source
media="(prefers-color-scheme: light)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img
alt="Review in cubic"
src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>
<!-- End of auto-generated description by cubic. -->

Signed-off-by: Yury Molodov <yurymolodov@gmail.com>
Co-authored-by: Max Kotliar <mkotlyar@victoriametrics.com>

.github/workflows/build.yml

@@ -66,6 +66,8 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
 
       - name: Setup Go
         id: go

.github/workflows/changelog-linter.yml

@@ -17,6 +17,7 @@ jobs:
         with:
           # needed for proper diff
           fetch-depth: 0
+          persist-credentials: false
 
       - name: 'Validate that changelog changes are under ## tip'
         run: |

.github/workflows/check-commit-signed.yml

@@ -15,6 +15,7 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           fetch-depth: 0  # we need full history for commit verification
+          persist-credentials: false
 
       - name: Check commit signatures
         run: |

.github/workflows/check-licenses.yml

@@ -18,6 +18,8 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
 
       - name: Setup Go
         id: go

.github/workflows/codeql-analysis-go.yml

@@ -32,6 +32,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
 
       - name: Set up Go
         id: go

.github/workflows/docs.yaml

@@ -21,6 +21,7 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           path: __vm
+          persist-credentials: false
 
       - name: Checkout private code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -28,6 +29,7 @@ jobs:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: __vm-docs
+          persist-credentials: true
 
       - name: Import GPG key
         uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd  # v7.0.0

.github/workflows/test.yml

@@ -35,6 +35,8 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
 
       - name: Setup Go
         id: go
@@ -78,6 +80,8 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
 
       - name: Setup Go
         id: go
@@ -103,6 +107,8 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
 
       - name: Setup Go
         id: go

.github/workflows/vmui.yml

@@ -33,6 +33,8 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
 
       - name: Cache node_modules
         id: cache

.golangci.yml

@@ -1,9 +1,18 @@
 version: "2"
 linters:
+  enable:
+    - errorlint
   settings:
     errcheck:
       exclude-functions:
         - (net/http.ResponseWriter).Write
+    errorlint:
+      errorf: true
+      # Do not enable `comparison` and `asserts`: they produce false positives,
+      # since many call sites intentionally compare sentinel errors directly (e.g. err == io.EOF)
+      # when the producer is documented to return them unwrapped. See https://github.com/VictoriaMetrics/VictoriaLogs/pull/1490
+      comparison: false
+      asserts: false
   exclusions:
     generated: lax
     presets:

app/vmctl/main.go

@@ -457,7 +457,7 @@ func main() {
 						auth.WithBearer(c.String(vmNativeDstBearerToken)),
 						auth.WithHeaders(c.String(vmNativeDstHeaders)))
 					if err != nil {
-						return fmt.Errorf("error initialize auth config for destination: %s: %s", dstAddr, err)
+						return fmt.Errorf("error initialize auth config for destination: %s: %w", dstAddr, err)
 					}
 
 					// create TLS config
@@ -601,7 +601,7 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 		auth.WithBearer(c.String(vmBearerToken)),
 		auth.WithHeaders(c.String(vmHeaders)))
 	if err != nil {
-		return vm.Config{}, fmt.Errorf("error initialize auth config for destination: %s: %s", addr, err)
+		return vm.Config{}, fmt.Errorf("error initialize auth config for destination: %s: %w", addr, err)
 	}
 
 	return vm.Config{

app/vmctl/utils.go

@@ -74,9 +74,9 @@ func wrapErr(vmErr *vm.ImportError, verbose bool) error {
 		verboseMsg = "(enable `--verbose` output to get more details)"
 	}
 	if vmErr.Err == nil {
-		return fmt.Errorf("%s\n\tLatest delivered batch for timestamps range %d - %d %s\n%s",
+		return fmt.Errorf("%w\n\tLatest delivered batch for timestamps range %d - %d %s\n%s",
 			vmErr.Err, minTS, maxTS, verboseMsg, errTS)
 	}
-	return fmt.Errorf("%s\n\tImporting batch failed for timestamps range %d - %d %s\n%s",
+	return fmt.Errorf("%w\n\tImporting batch failed for timestamps range %d - %d %s\n%s",
 		vmErr.Err, minTS, maxTS, verboseMsg, errTS)
 }

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -5,7 +5,7 @@ import uPlot from "uplot";
 import Button from "../../Main/Button/Button";
 import { CloseIcon, DragIcon } from "../../Main/Icons";
 import { SeriesItemStatsFormatted } from "../../../types";
-import { STATS_ORDER } from "../../../constants/graph";
+import { STATS_ORDER_TOOLTIP } from "../../../constants/graph";
 
 export interface ChartTooltipProps {
   u?: uPlot;
@@ -164,7 +164,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
       </div>
       {statsFormatted && (
         <table className="vm-chart-tooltip-stats">
-          {STATS_ORDER.map((key, i) => (
+          {STATS_ORDER_TOOLTIP.map((key, i) => (
             <div
               className="vm-chart-tooltip-stats-row"
               key={i}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendConfigs/LegendConfigs.tsx

@@ -61,7 +61,7 @@ const LegendConfigs: FC<Props> = ({ data, isCompact }) => {
       label: "Hide Statistics",
       value: hideStats,
       onChange: onChangeStats,
-      info: "If enabled, hides the display of min, median, and max values.",
+      info: "If enabled, hides the display of min, median, max, and last values.",
     }
   ];
 

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendItem/LegendItem.tsx

@@ -5,7 +5,7 @@ import "./style.scss";
 import classNames from "classnames";
 import { getFreeFields } from "./helpers";
 import useCopyToClipboard from "../../../../../hooks/useCopyToClipboard";
-import { STATS_ORDER } from "../../../../../constants/graph";
+import { STATS_ORDER_LEGEND } from "../../../../../constants/graph";
 import { useShowStats } from "../hooks/useShowStats";
 import { useLegendFormat } from "../hooks/useLegendFormat";
 import { getLabelAlias } from "../../../../../utils/metric";
@@ -80,7 +80,7 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields })
       </div>
       {!hideStats && showStats && (
         <div className="vm-legend-item-stats">
-          {STATS_ORDER.map((key, i) => (
+          {STATS_ORDER_LEGEND.map((key, i) => (
             <div
               className="vm-legend-item-stats-row"
               key={i}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendViews/LegendTable.tsx

@@ -4,11 +4,11 @@ import "./style.scss";
 import { LegendItemType } from "../../../../../types";
 import { MouseEvent } from "react";
 import classNames from "classnames";
-import { STATS_ORDER } from "../../../../../constants/graph";
+import { STATS_ORDER_LEGEND } from "../../../../../constants/graph";
 import { useShowStats } from "../hooks/useShowStats";
 import { getValueByPath } from "../../../../../utils/object";
 
-const statsColumns = STATS_ORDER.map(k => ({
+const statsColumns = STATS_ORDER_LEGEND.map(k => ({
   key: `statsFormatted.${k}`,
   title: k
 }));

app/vmui/packages/vmui/src/constants/graph.ts

@@ -26,4 +26,5 @@ export const GRAPH_SIZES: GraphSize[] = [
   },
 ];
 
-export const STATS_ORDER: (keyof SeriesItemStatsFormatted)[] = ["min", "median", "max"];
+export const STATS_ORDER_LEGEND: (keyof SeriesItemStatsFormatted)[] = ["min", "median", "max", "last"];
+export const STATS_ORDER_TOOLTIP: (keyof SeriesItemStatsFormatted)[] = ["min", "median", "max"];

app/vmui/packages/vmui/src/types/uplot.ts

@@ -4,6 +4,7 @@ export interface SeriesItemStatsFormatted {
     min: string,
     max: string,
     median: string,
+    last: string,
 }
 
 export interface SeriesItem extends Series {

app/vmui/packages/vmui/src/utils/uplot/series.ts

@@ -53,6 +53,7 @@ const getSeriesStatistics = (d: MetricResult) => {
       min: formatPrettyNumber(min, min, max),
       max: formatPrettyNumber(max, min, max),
       median: formatPrettyNumber(median, min, max),
+      last: formatPrettyNumber(values.at(-1), min, max),
     },
   };
 };

docs/victoriametrics/CONTRIBUTING.md

@@ -28,7 +28,7 @@ If you like VictoriaMetrics and want to contribute, then it would be great:
 ## Issues
 
 When making a new issue, make sure to create no duplicates. Use GitHub search to find whether similar issues exist already.
-The new issue should be written in English and contain concise description of the problem and environment where it exists.
+The new issue should be written in English and contain a concise description of the problem and the environment where it exists.
 We'd very much prefer to have a specific use-case included in the description, since it could have workaround or alternative solutions.
 
 When looking for an issue to contribute, always prefer working on [bugs](https://github.com/VictoriaMetrics/VictoriaMetrics/issues?q=is%3Aopen+is%3Aissue+label%3Abug)
@@ -48,7 +48,7 @@ We use [labels](https://docs.github.com/en/issues/using-labels-and-milestones-to
 1. `need more info`, assigned to issues that require elaboration from the issue creator.
   For example, if we weren't able to reproduce the reported bug based on the ticket description then we ask additional
   questions which could help to reproduce the issue and add `need more info` label. This label helps other maintainers
-  to understand that this issue wasn't forgotten but waits for the feedback from user.
+  to understand that this issue wasn't forgotten but waits for the feedback from the user.
 1. `completed`, assigned to issues that required code changes and those changes were merged to upstream, but not released yet.
   Once a release is made, maintainers go through all labeled issues, leave a comment about the new release, and close the issue.
 1. `vmui`, assigned to issues related to [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) or [VictoriaLogs webui](https://docs.victoriametrics.com/victorialogs/querying/#web-ui)
@@ -63,32 +63,31 @@ Pull requests requirements:
 1. Don't use `master` branch for making PRs, as it makes it impossible for reviewers to modify the changes.
 1. All commits need to be [signed](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits).
 1. Pull request title should be prefixed with `<dir>/<component>:` to show what component has been changed, i.e. `app/vmalert: fix...`.
-   Pull request description should contain clear and concise description of what was done, why it is needed and for what purpose.
+   Pull request description should contain a clear and concise description of what was done, why it is needed and for what purpose.
    Use clear language, so reviewers can quickly understand the change and its impact.
 1. A link to the issue(s) related to the change, if any. Use `Fixes [issue link]` if the PR resolves the issue, or `Related to [issue link]` for reference.
 1. Tests proving that the change is effective. Tests are expected for non-trivial new functionality or non-trivial modifications.
    Bug fixes must include tests unless a maintainer explicitly agrees otherwise.
-   See [this style guide](https://itnext.io/f-tests-as-a-replacement-for-table-driven-tests-in-go-8814a8b19e9e) for tests.
-   To run tests and code checks locally, execute commands `make test-full` and `make check-all`.
+   See [this style guide](https://itnext.io/f-tests-as-a-replacement-for-table-driven-tests-in-go-8814a8b19e9e) for tests. See [this section](#testing) for how to run tests.
 1. Try to not extend the scope of the pull requests outside the issue, do not make unrelated changes.
-1. Update [docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs) if needed. For example, adding a new flag or changing behavior of existing flags or features
+1. Update [docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs) if needed. For example, adding a new flag or changing the behavior of existing flags or features
    requires reflecting these changes in the documentation. For new features add `{{%/* available_from "#" */%}}` shortcode to the documentation.
    It will be later automatically replaced with an actual release version.
 1. A line in the [changelog](https://docs.victoriametrics.com/victoriametrics/changelog/#tip) mentioning the change and related issue in a way
    that would be clear to other readers even if they don't have the full context.
-1. Avoid modifying code in the `/vendor` folder manually, even when the vendored package originates are from the VictoriaMetrics GitHub organization.
+1. Avoid modifying code in the `/vendor` folder manually, even when the vendored package originates from the VictoriaMetrics GitHub organization.
    For instance, VictoriaLogs vendors packages under the `/lib` folder from VictoriaMetrics, and VictoriaTraces vendors the `/lib/logstorage` package from VictoriaLogs.
-   Submit a pull request to the upstream repository first. Afterward, a separate pull request can be opened to update the version of the vendored folder in downstream repository.
+   Submit a pull request to the upstream repository first. Afterward, a separate pull request can be opened to update the version of the vendored folder in the downstream repository.
    * For common packages, the vendored package can be updated with this command: `go get <dependency>@vX.Y.Z`.  
    * For VictoriaMetrics packages, use `go get <dependency>@canonical_commit_hash`. 
    Finally, run `go mod tidy` and `go mod vendor` to update `go.mod`, `go.sum`, and `/vendor`.
 1. Ping reviewers who you think have the best expertise on the matter.
 
-See good example of a [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6487).
+See a good example of a [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6487).
 
 ## Merging Pull Request
 
-The person who merges the Pull Request is responsible for satisfying requirements below:
+The person who merges the Pull Request is responsible for satisfying the requirements below:
 
 1. Make sure that PR satisfies [Pull Request checklist](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist),
    it is approved by at least one reviewer, all CI checks are green.
@@ -97,9 +96,9 @@ The person who merges the Pull Request is responsible for satisfying requirement
 1. If applicable, cherry-pick the change to [LTS release lines](https://docs.victoriametrics.com/victoriametrics/lts-releases/)
    and mention in the PR comment what was or wasn't cherry-picked.
 1. Update related issues with a meaningful message of what has changed and when it will be
-   released. _This helps users to understand the change without reading PR._
+   released. _This helps users to understand the change without reading the PR._
 1. Add label `completed` to related issues.
-1. Do not close related tickets until release is made. If ticket was auto-closed by GitHub or user - re-open it.
+1. Do not close related tickets until the release is made. If the ticket was auto-closed by GitHub or a user - re-open it.
 
 ## KISS principle
 
@@ -115,9 +114,9 @@ We are open to third-party pull requests provided they follow [KISS design princ
 - Minimize the number of moving parts in the distributed system.
 - Avoid automated decisions, which may hurt cluster availability, consistency, performance or debuggability.
 
-Adhering to `KISS` principle, simplifies the resulting code and architecture so it can be reviewed, understood and debugged by a wider audience.
+Adhering to the `KISS` principle, simplifies the resulting code and architecture so it can be reviewed, understood and debugged by a wider audience.
 
-Due to `KISS`, [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) has none of the following "features" popular in distributed computing world:
+Due to `KISS`, [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) has none of the following "features" popular in distributed computing:
 
 - Fragile gossip protocols. See [failed attempt in Thanos](https://github.com/improbable-eng/thanos/blob/030bc345c12c446962225221795f4973848caab5/docs/proposals/completed/201809_gossip-removal.md).
 - Hard-to-understand-and-implement-properly [Paxos protocols](https://www.quora.com/In-distributed-systems-what-is-a-simple-explanation-of-the-Paxos-algorithm).
@@ -126,3 +125,17 @@ Due to `KISS`, [cluster version of VictoriaMetrics](https://docs.victoriametrics
 - Automatic cluster resizing, which may cost you a lot of money if improperly configured.
 - Automatic discovering and addition of new nodes in the cluster, which may mix data between dev and prod clusters :)
 - Automatic leader election, which may result in split brain disaster on network errors.
+
+## Testing
+
+We recommend running the following sequence of checks and tests before submitting a pull request:
+```sh
+# run static checks
+make check-all
+
+# run unit test
+make test-full
+
+# run integration tests
+make apptest
+```

docs/victoriametrics/changelog/CHANGELOG.md

@@ -29,6 +29,8 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * FEATURE: all VictoriaMetrics components: add `-http.header.disableServerHostname` command-line flag for disabling the `X-Server-Hostname` HTTP response header. See [#11067](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11067). Thanks to @zasdaym for contribution.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): log calls to [/api/v1/admin/tsdb/delete_series](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series) API handler. This should help to identify events of metrics deletion from the database. See [#11104](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11104).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `-vm-headers` and `-vm-bearer-token` flags for authenticating requests to the VictoriaMetrics import destination. The flags are available in `opentsdb`, `influx`, `remote-read`, `prometheus`, `mimir`, and `thanos` vmctl sub-commands. See [#8897](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8897).
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add the `last` value to graph legend statistics. See [#10759](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10759).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-promscrape.cluster.shardByLabels` command-line flag for selecting target labels used for sharding scrape targets among `vmagent` instances in cluster mode. See [#11044](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11044).
 
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix issue with producing aggregated samples with identical timestamps between flushes. See [#10808](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10808).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not fail backup list if directory is absent while using `fs://` destination to align with other protocols. See [6c3c548](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/6c3c548ddb0385b749e731f52276f130e2a4e4a8)

docs/victoriametrics/vmagent.md

@@ -797,6 +797,13 @@ For example, the following commands spread scrape targets among a cluster of two
 The `-promscrape.cluster.memberNum` can be set to a StatefulSet pod name when `vmagent` runs in Kubernetes.
 The pod name must end with a number in the range `0 ... promscrape.cluster.membersCount-1`. For example, `-promscrape.cluster.memberNum=vmagent-0`.
 
+By default, targets are sharded among `vmagent` instances by all target labels after relabeling.
+Use `-promscrape.cluster.shardByLabels` to shard targets by specified labels instead.
+For example, `-promscrape.cluster.shardByLabels=service,pod` keeps targets with the same `service` and `pod` label value on the same `vmagent` instance.
+
+If some of the specified labels are present on a target, then only the present labels are used for sharding.
+If none of the specified labels are present, then all target labels are used for sharding.
+
 By default, each scrape target is scraped only by a single `vmagent` instance in the cluster. If there is a need for replicating scrape targets among multiple `vmagent` instances,
 then `-promscrape.cluster.replicationFactor` command-line flag must be set to the desired number of replicas. For example, the following commands
 start a cluster of three `vmagent` instances, where two `vmagent` instances scrape each target:

lib/mergeset/table_search_timing_test.go

@@ -94,7 +94,7 @@ func benchmarkTableSearchKeysExt(b *testing.B, tb *Table, keys [][]byte, stripSu
 				}
 				ts.Seek(searchKey)
 				if !ts.NextItem() {
-					panic(fmt.Errorf("BUG: NextItem must return true for searchKeys[%d]=%q; err=%v", i, searchKey, ts.Error()))
+					panic(fmt.Errorf("BUG: NextItem must return true for searchKeys[%d]=%q; err=%w", i, searchKey, ts.Error()))
 				}
 				if !bytes.HasPrefix(ts.Item, searchKey) {
 					panic(fmt.Errorf("BUG: unexpected item found for searchKey[%d]=%q; got %q; want %q", i, searchKey, ts.Item, key))

lib/promscrape/config.go

@@ -76,6 +76,9 @@ var (
 		"Every %d occurrence in the template is substituted with -promscrape.cluster.memberNum at urls to vmagent instances responsible for scraping the given target "+
 		"at /service-discovery page. For example -promscrape.cluster.memberURLTemplate='http://vmagent-%d:8429/targets'. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more details")
+	clusterShardByLabels = flagutil.NewArrayString("promscrape.cluster.shardByLabels", "Optional list of target labels, which will be used for sharding targets among cluster members "+
+		"if -promscrape.cluster.membersCount is greater than 1. If none of the specified labels are found in a target, then all the target labels will be used for sharding. "+
+		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info")
 	clusterReplicationFactor = flag.Int("promscrape.cluster.replicationFactor", 1, "The number of members in the cluster, which scrape the same targets. "+
 		"If the replication factor is greater than 1, then the deduplication must be enabled at remote storage side. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info")
@@ -86,7 +89,10 @@ var (
 		"Bigger uncompressed responses are rejected. See also max_scrape_size option at https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs")
 )
 
-var clusterMemberID int
+var (
+	clusterMemberID            int
+	clusterShardByLabelsSorted []string
+)
 
 func mustInitClusterMemberID() {
 	s := *clusterMemberNum
@@ -110,6 +116,14 @@ func mustInitClusterMemberID() {
 	clusterMemberID = n
 }
 
+func initClusterShardByLabels() {
+	if len(*clusterShardByLabels) == 0 {
+		return
+	}
+	clusterShardByLabelsSorted = slices.Clone(*clusterShardByLabels)
+	slices.Sort(clusterShardByLabelsSorted)
+}
+
 // Config represents essential parts from Prometheus config defined at https://prometheus.io/docs/prometheus/latest/configuration/configuration/
 type Config struct {
 	Global            GlobalConfig    `yaml:"global,omitempty"`
@@ -1137,13 +1151,29 @@ func (stc *StaticConfig) appendScrapeWork(dst []*ScrapeWork, swc *scrapeWorkConf
 	return dst
 }
 
-func appendScrapeWorkKey(dst []byte, labels *promutil.Labels) []byte {
-	for _, label := range labels.GetLabels() {
-		// Do not use strconv.AppendQuote, since it is slow according to CPU profile.
-		dst = append(dst, label.Name...)
-		dst = append(dst, '=')
-		dst = append(dst, label.Value...)
-		dst = append(dst, ',')
+func appendScrapeWorkKey(dst []byte, labels *promutil.Labels, shardByLabels []string) []byte {
+	originalDstLen := len(dst)
+	for _, targetLabelName := range shardByLabels {
+		for _, label := range labels.GetLabels() {
+			if label.Name == targetLabelName {
+				// Do not use strconv.AppendQuote, since it is slow according to CPU profile.
+				dst = append(dst, label.Name...)
+				dst = append(dst, '=')
+				dst = append(dst, label.Value...)
+				dst = append(dst, ',')
+				break
+			}
+		}
+	}
+	// none of the labels specified in -promscrape.cluster.shardByLabels is present, use all labels for backward compatibility.
+	if len(dst) == originalDstLen {
+		for _, label := range labels.GetLabels() {
+			dst = append(dst, label.Name...)
+			dst = append(dst, '=')
+			dst = append(dst, label.Value...)
+			dst = append(dst, ',')
+		}
+		return dst
 	}
 	return dst
 }
@@ -1195,7 +1225,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1687#issuecomment-940629495
 	if *clusterMembersCount > 1 {
 		bb := scrapeWorkKeyBufPool.Get()
-		bb.B = appendScrapeWorkKey(bb.B[:0], labels)
+		bb.B = appendScrapeWorkKey(bb.B[:0], labels, clusterShardByLabelsSorted)
 		memberNums := getClusterMemberNumsForScrapeWork(bytesutil.ToUnsafeString(bb.B), *clusterMembersCount, *clusterReplicationFactor)
 		scrapeWorkKeyBufPool.Put(bb)
 		if !slices.Contains(memberNums, clusterMemberID) {

lib/promscrape/config_test.go

@@ -148,6 +148,112 @@ func TestGetClusterMemberNumsForScrapeWork(t *testing.T) {
 	f("foo", 3, 2, []int{2, 0})
 }
 
+func TestAppendScrapeWorkKeyShardByLabels(t *testing.T) {
+	f := func(labelsA, labelsB map[string]string, shardByLabels []string, equal bool) {
+		t.Helper()
+		originValue := *clusterShardByLabels
+		*clusterShardByLabels = shardByLabels
+		initClusterShardByLabels()
+		keyA := string(appendScrapeWorkKey(nil, promutil.NewLabelsFromMap(labelsA), clusterShardByLabelsSorted))
+		keyB := string(appendScrapeWorkKey(nil, promutil.NewLabelsFromMap(labelsB), clusterShardByLabelsSorted))
+		if equal && keyA != keyB {
+			t.Fatalf("unexpected different scrape work keys for shardByLabels=%q;\nlabelsA=%v\nlabelsB=%v\nkeyA=%q\nkeyB=%q",
+				shardByLabels, labelsA, labelsB, keyA, keyB)
+		} else if !equal && keyA == keyB {
+			t.Fatalf("unexpected equal scrape work keys for shardByLabels=%q;\nlabelsA=%v\nlabelsB=%v\nkeyA=%q\nkeyB=%q",
+				shardByLabels, labelsA, labelsB, keyA, keyB)
+		}
+		*clusterShardByLabels = originValue
+	}
+
+	// didn't specify -promscrape.cluster.shardByLabels, and all labels are the same
+	f(
+		map[string]string{
+			"a": "aa",
+			"b": "bb",
+			"c": "cc",
+			"d": "dd"},
+		map[string]string{
+			"a": "aa",
+			"b": "bb",
+			"c": "cc",
+			"d": "dd"},
+		[]string{},
+		true,
+	)
+	// match all labels in -promscrape.cluster.shardByLabels, and they're the same
+	f(
+		map[string]string{
+			"a": "aa",
+			"b": "bb",
+			"c": "cc",
+			"d": "dd"},
+		map[string]string{
+			"c": "cc",
+			"a": "aa",
+			"b": "other",
+			"d": "other"},
+		[]string{"c", "a"},
+		true,
+	)
+
+	// match all labels in -promscrape.cluster.shardByLabels, and they're different
+	f(
+		map[string]string{
+			"a": "aa",
+			"b": "bb",
+			"c": "cc",
+			"d": "dd"},
+		map[string]string{
+			"a": "aa",
+			"b": "other",
+			"c": "cc-------",
+			"d": "other"},
+		[]string{"a", "c"},
+		false,
+	)
+
+	// match part of labels in -promscrape.cluster.shardByLabels, and they're the same
+	f(
+		map[string]string{
+			"a": "aa",
+			"c": "cc",
+			"d": "dd"},
+		map[string]string{
+			"a": "aa",
+			"c": "cc",
+			"e": "ee"},
+		[]string{"a", "b", "c"},
+		true,
+	)
+
+	// match part of labels in -promscrape.cluster.shardByLabels, and they're different
+	f(
+		map[string]string{
+			"a": "aa",
+			"c": "cc",
+			"d": "dd"},
+		map[string]string{
+			"a": "aa-------",
+			"c": "cc",
+			"e": "ee"},
+		[]string{"a", "b", "c"},
+		false,
+	)
+	// none of labels in -promscrape.cluster.shardByLabels is matched, so all labels will be used to sharding
+	f(
+		map[string]string{
+			"d": "dd",
+			"e": "ee"},
+		map[string]string{
+			"d": "dd",
+			"e": "ee"},
+		[]string{"a", "b", "c"},
+		true,
+	)
+
+}
+
 func TestLoadStaticConfigs(t *testing.T) {
 	scs, err := loadStaticConfigs("testdata/file_sd.json")
 	if err != nil {

lib/promscrape/scraper.go

@@ -66,6 +66,7 @@ func CheckConfig() error {
 // Scraped data is passed to pushData.
 func Init(pushData func(at *auth.Token, wr *prompb.WriteRequest)) {
 	mustInitClusterMemberID()
+	initClusterShardByLabels()
 	globalStopChan = make(chan struct{})
 	scraperWG.Go(func() {
 		runScraper(*promscrapeConfigFile, pushData, globalStopChan)