lib/fs: retry file deletion on NFS error

This commit adds additional file removal retries.
This is follow-up for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9842.

 According to the stack trace, NFS also could return error for `deleteFilePath`.
This retry is safe, since part already removed from `parts.json` and
performs a skips empty directories on restart.

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060

.github/workflows/build.yml

@@ -66,8 +66,6 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          persist-credentials: false
 
       - name: Setup Go
         id: go

.github/workflows/changelog-linter.yml

@@ -17,7 +17,6 @@ jobs:
         with:
           # needed for proper diff
           fetch-depth: 0
-          persist-credentials: false
 
       - name: 'Validate that changelog changes are under ## tip'
         run: |

.github/workflows/check-commit-signed.yml

@@ -15,7 +15,6 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           fetch-depth: 0  # we need full history for commit verification
-          persist-credentials: false
 
       - name: Check commit signatures
         run: |

.github/workflows/check-licenses.yml

@@ -18,8 +18,6 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          persist-credentials: false
 
       - name: Setup Go
         id: go

.github/workflows/codeql-analysis-go.yml

@@ -32,8 +32,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          persist-credentials: false
 
       - name: Set up Go
         id: go

.github/workflows/docs.yaml

@@ -21,7 +21,6 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           path: __vm
-          persist-credentials: false
 
       - name: Checkout private code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -29,7 +28,6 @@ jobs:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: __vm-docs
-          persist-credentials: true
 
       - name: Import GPG key
         uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd  # v7.0.0

.github/workflows/test.yml

@@ -35,8 +35,6 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          persist-credentials: false
 
       - name: Setup Go
         id: go
@@ -80,8 +78,6 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          persist-credentials: false
 
       - name: Setup Go
         id: go
@@ -107,8 +103,6 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          persist-credentials: false
 
       - name: Setup Go
         id: go

.github/workflows/vmui.yml

@@ -33,8 +33,6 @@ jobs:
     steps:
       - name: Code checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          persist-credentials: false
 
       - name: Cache node_modules
         id: cache

.golangci.yml

@@ -1,18 +1,9 @@
 version: "2"
 linters:
-  enable:
-    - errorlint
   settings:
     errcheck:
       exclude-functions:
         - (net/http.ResponseWriter).Write
-    errorlint:
-      errorf: true
-      # Do not enable `comparison` and `asserts`: they produce false positives,
-      # since many call sites intentionally compare sentinel errors directly (e.g. err == io.EOF)
-      # when the producer is documented to return them unwrapped. See https://github.com/VictoriaMetrics/VictoriaLogs/pull/1490
-      comparison: false
-      asserts: false
   exclusions:
     generated: lax
     presets:

app/vmctl/main.go

@@ -457,7 +457,7 @@ func main() {
 						auth.WithBearer(c.String(vmNativeDstBearerToken)),
 						auth.WithHeaders(c.String(vmNativeDstHeaders)))
 					if err != nil {
-						return fmt.Errorf("error initialize auth config for destination: %s: %w", dstAddr, err)
+						return fmt.Errorf("error initialize auth config for destination: %s: %s", dstAddr, err)
 					}
 
 					// create TLS config
@@ -601,7 +601,7 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 		auth.WithBearer(c.String(vmBearerToken)),
 		auth.WithHeaders(c.String(vmHeaders)))
 	if err != nil {
-		return vm.Config{}, fmt.Errorf("error initialize auth config for destination: %s: %w", addr, err)
+		return vm.Config{}, fmt.Errorf("error initialize auth config for destination: %s: %s", addr, err)
 	}
 
 	return vm.Config{

app/vmctl/utils.go

@@ -74,9 +74,9 @@ func wrapErr(vmErr *vm.ImportError, verbose bool) error {
 		verboseMsg = "(enable `--verbose` output to get more details)"
 	}
 	if vmErr.Err == nil {
-		return fmt.Errorf("%w\n\tLatest delivered batch for timestamps range %d - %d %s\n%s",
+		return fmt.Errorf("%s\n\tLatest delivered batch for timestamps range %d - %d %s\n%s",
 			vmErr.Err, minTS, maxTS, verboseMsg, errTS)
 	}
-	return fmt.Errorf("%w\n\tImporting batch failed for timestamps range %d - %d %s\n%s",
+	return fmt.Errorf("%s\n\tImporting batch failed for timestamps range %d - %d %s\n%s",
 		vmErr.Err, minTS, maxTS, verboseMsg, errTS)
 }

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -5,7 +5,7 @@ import uPlot from "uplot";
 import Button from "../../Main/Button/Button";
 import { CloseIcon, DragIcon } from "../../Main/Icons";
 import { SeriesItemStatsFormatted } from "../../../types";
-import { STATS_ORDER_TOOLTIP } from "../../../constants/graph";
+import { STATS_ORDER } from "../../../constants/graph";
 
 export interface ChartTooltipProps {
   u?: uPlot;
@@ -164,7 +164,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
       </div>
       {statsFormatted && (
         <table className="vm-chart-tooltip-stats">
-          {STATS_ORDER_TOOLTIP.map((key, i) => (
+          {STATS_ORDER.map((key, i) => (
             <div
               className="vm-chart-tooltip-stats-row"
               key={i}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendConfigs/LegendConfigs.tsx

@@ -61,7 +61,7 @@ const LegendConfigs: FC<Props> = ({ data, isCompact }) => {
       label: "Hide Statistics",
       value: hideStats,
       onChange: onChangeStats,
-      info: "If enabled, hides the display of min, median, max, and last values.",
+      info: "If enabled, hides the display of min, median, and max values.",
     }
   ];
 

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendItem/LegendItem.tsx

@@ -5,7 +5,7 @@ import "./style.scss";
 import classNames from "classnames";
 import { getFreeFields } from "./helpers";
 import useCopyToClipboard from "../../../../../hooks/useCopyToClipboard";
-import { STATS_ORDER_LEGEND } from "../../../../../constants/graph";
+import { STATS_ORDER } from "../../../../../constants/graph";
 import { useShowStats } from "../hooks/useShowStats";
 import { useLegendFormat } from "../hooks/useLegendFormat";
 import { getLabelAlias } from "../../../../../utils/metric";
@@ -80,7 +80,7 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields })
       </div>
       {!hideStats && showStats && (
         <div className="vm-legend-item-stats">
-          {STATS_ORDER_LEGEND.map((key, i) => (
+          {STATS_ORDER.map((key, i) => (
             <div
               className="vm-legend-item-stats-row"
               key={i}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendViews/LegendTable.tsx

@@ -4,11 +4,11 @@ import "./style.scss";
 import { LegendItemType } from "../../../../../types";
 import { MouseEvent } from "react";
 import classNames from "classnames";
-import { STATS_ORDER_LEGEND } from "../../../../../constants/graph";
+import { STATS_ORDER } from "../../../../../constants/graph";
 import { useShowStats } from "../hooks/useShowStats";
 import { getValueByPath } from "../../../../../utils/object";
 
-const statsColumns = STATS_ORDER_LEGEND.map(k => ({
+const statsColumns = STATS_ORDER.map(k => ({
   key: `statsFormatted.${k}`,
   title: k
 }));

app/vmui/packages/vmui/src/constants/graph.ts

@@ -26,5 +26,4 @@ export const GRAPH_SIZES: GraphSize[] = [
   },
 ];
 
-export const STATS_ORDER_LEGEND: (keyof SeriesItemStatsFormatted)[] = ["min", "median", "max", "last"];
-export const STATS_ORDER_TOOLTIP: (keyof SeriesItemStatsFormatted)[] = ["min", "median", "max"];
+export const STATS_ORDER: (keyof SeriesItemStatsFormatted)[] = ["min", "median", "max"];

app/vmui/packages/vmui/src/types/uplot.ts

@@ -4,7 +4,6 @@ export interface SeriesItemStatsFormatted {
     min: string,
     max: string,
     median: string,
-    last: string,
 }
 
 export interface SeriesItem extends Series {

app/vmui/packages/vmui/src/utils/uplot/series.ts

@@ -53,7 +53,6 @@ const getSeriesStatistics = (d: MetricResult) => {
       min: formatPrettyNumber(min, min, max),
       max: formatPrettyNumber(max, min, max),
       median: formatPrettyNumber(median, min, max),
-      last: formatPrettyNumber(values.at(-1), min, max),
     },
   };
 };

docs/victoriametrics/changelog/CHANGELOG.md

@@ -29,10 +29,10 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * FEATURE: all VictoriaMetrics components: add `-http.header.disableServerHostname` command-line flag for disabling the `X-Server-Hostname` HTTP response header. See [#11067](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11067). Thanks to @zasdaym for contribution.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): log calls to [/api/v1/admin/tsdb/delete_series](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series) API handler. This should help to identify events of metrics deletion from the database. See [#11104](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11104).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `-vm-headers` and `-vm-bearer-token` flags for authenticating requests to the VictoriaMetrics import destination. The flags are available in `opentsdb`, `influx`, `remote-read`, `prometheus`, `mimir`, and `thanos` vmctl sub-commands. See [#8897](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8897).
-* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add the `last` value to graph legend statistics. See [#10759](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10759).
 
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix issue with producing aggregated samples with identical timestamps between flushes. See [#10808](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10808).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not fail backup list if directory is absent while using `fs://` destination to align with other protocols. See [6c3c548](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/6c3c548ddb0385b749e731f52276f130e2a4e4a8)
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).
 
 ## [v1.145.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.145.0)
 

lib/fs/dir_remover.go

@@ -203,7 +203,12 @@ func tryRemoveDir(dirPath string) bool {
 
 	deleteFilePath := filepath.Join(dirPath, deleteDirFilename)
 	// Remove the deleteDirFilename file, since there are no other entries left in the directory.
-	MustRemovePath(deleteFilePath)
+	if err := os.Remove(deleteFilePath); err != nil {
+		if !isTemporaryNFSError(err) {
+			logger.Fatalf("FATAL: cannot remove %q: %s", deleteFilePath, err)
+		}
+		return false
+	}
 
 	// Sync the directory after the removing deletDirFilename file in order to make sure
 	// all the metadata files are removed at some exotic filesystems such as OSSFS2.
@@ -212,7 +217,12 @@ func tryRemoveDir(dirPath string) bool {
 	MustSyncPath(dirPath)
 
 	// Remove the dirPath itself
-	MustRemovePath(dirPath)
+	if err := os.Remove(dirPath); err != nil {
+		if !isTemporaryNFSError(err) {
+			logger.Fatalf("FATAL: cannot remove %q: %s", dirPath, err)
+		}
+		return false
+	}
 
 	// Do not sync the parent directory for the dirPath - the caller can do this if needed.
 	// It is OK if the dirPath will remain undeleted after unclean shutdown - it will be deleted

lib/mergeset/table_search_timing_test.go

@@ -94,7 +94,7 @@ func benchmarkTableSearchKeysExt(b *testing.B, tb *Table, keys [][]byte, stripSu
 				}
 				ts.Seek(searchKey)
 				if !ts.NextItem() {
-					panic(fmt.Errorf("BUG: NextItem must return true for searchKeys[%d]=%q; err=%w", i, searchKey, ts.Error()))
+					panic(fmt.Errorf("BUG: NextItem must return true for searchKeys[%d]=%q; err=%v", i, searchKey, ts.Error()))
 				}
 				if !bytes.HasPrefix(ts.Item, searchKey) {
 					panic(fmt.Errorf("BUG: unexpected item found for searchKey[%d]=%q; got %q; want %q", i, searchKey, ts.Item, key))