package/release: Add github-verify-release job (#10476)

### Describe Your Changes

The job ensure that:
- the draft release with given `$(TAG)` exists
- the release has excpected `$(GITHUB_ASSETS_COUNT)` number of uploaded
assets
- All the assets were uploaded succesfully.

It also adds helper job `github-get-release` which finds a draft release
by `$(TAG)` and stores into file `/tmp/vm-github-release-$(TAG)` file.

The `github-delete-release1 job is decoupled from the file produced by
`github-create-release job`. So it could be run at any time from any
machine.

### Checklist

The following checks are **mandatory**:

- [ ] My change adheres to [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
- [ ] My change adheres to [VictoriaMetrics development
goals](https://docs.victoriametrics.com/victoriametrics/goals/).

docs/victoriametrics/Release-Guide.md

@@ -152,7 +152,7 @@ and the candidate is deployed to the sandbox environment.
       * To run the command `TAG=v1.xx.y make github-create-release github-upload-assets`, so new release is created
         and all the needed assets are re-uploaded to it.
 
-1. Go to <https://github.com/VictoriaMetrics/VictoriaMetrics/releases> and verify that draft release with the name `TAG` has been created
+1. Run `TAG=v1.xx.y make github-verify-release` to verify that draft release with the name `TAG` has been created
    and this release contains all the needed binaries and checksums.
 1. Update the release description with the content of [CHANGELOG](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md) for this release. **Use "Save Draft" button, do not publish the release yet!**.
 1. Follow the instructions in [LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics-enterprise/blob/enterprise-single-node/Release-Guide.md#lts-release).

package/release/Makefile

@@ -1,6 +1,8 @@
-GITHUB_RELEASE_SPEC_FILE="/tmp/vm-github-release"
+GITHUB_RELEASE_SPEC_FILE="/tmp/vm-github-release-$(TAG)"
 GITHUB_DEBUG_FILE="/tmp/vm-github-debug"
 
+GITHUB_ASSETS_COUNT ?= 112
+
 github-token-check:
 ifndef GITHUB_TOKEN
 	$(error missing GITHUB_TOKEN env var. It must contain github token for VictoriaMetrics project obtained from https://github.com/settings/tokens)
@@ -27,8 +29,8 @@ github-create-release: github-token-check github-tag-check
 			exit 1; \
 		fi
 
-github-upload-assets:
+github-upload-assets: github-get-release
-	@release_id=$$(cat $(GITHUB_RELEASE_SPEC_FILE) | grep '"id"' -m 1 | sed -E 's/.* ([[:digit:]]+)\,/\1/'); \
+	@release_id=$$(jq '.id' "$(GITHUB_RELEASE_SPEC_FILE)"); \
 	$(foreach file, $(wildcard bin/*.zip), FILE=$(file) RELEASE_ID=$${release_id} CONTENT_TYPE="application/zip" $(MAKE) github-upload-asset || exit 1;) \
 	$(foreach file, $(wildcard bin/*.tar.gz), FILE=$(file) RELEASE_ID=$${release_id} CONTENT_TYPE="application/x-gzip" $(MAKE) github-upload-asset || exit 1;) \
 	$(foreach file, $(wildcard bin/*_checksums.txt), FILE=$(file) RELEASE_ID=$${release_id} CONTENT_TYPE="text/plain" $(MAKE) github-upload-asset || exit 1;) 
@@ -55,8 +57,8 @@ endif
 			exit 1; \
 		fi
 
-github-delete-release: github-token-check
+github-delete-release: github-token-check github-tag-check github-get-release
-	@release_id=$$(cat $(GITHUB_RELEASE_SPEC_FILE) | grep '"id"' -m 1 | sed -E 's/.* ([[:digit:]]+)\,/\1/'); \
+	@release_id=$$(jq '.id' "$(GITHUB_RELEASE_SPEC_FILE)"); \
 	result=$$(curl -o $(GITHUB_DEBUG_FILE) -s -w "%{http_code}" \
 		-X DELETE \
 		-H "Accept: application/vnd.github+json" \
@@ -69,3 +71,28 @@ github-delete-release: github-token-check
 			cat $(GITHUB_DEBUG_FILE); \
 			exit 1; \
 		fi
+
+github-verify-release: github-token-check github-tag-check github-get-release
+	@ASSET_COUNT=$$(jq '.assets | length' "$(GITHUB_RELEASE_SPEC_FILE)"); \
+	if [ "$${ASSET_COUNT}" -ne "$(GITHUB_ASSETS_COUNT)" ]; then \
+		echo "Expected $(GITHUB_ASSETS_COUNT) assets, found $${ASSET_COUNT}"; \
+		exit 1; \
+	fi; \
+	FAILED_COUNT=$$(jq '[.assets[] | select(.state != "uploaded")] | length' "$(GITHUB_RELEASE_SPEC_FILE)"); \
+	if [ "$${FAILED_COUNT}" -ne "0" ]; then \
+		echo "Found $${FAILED_COUNT} failed assets"; \
+		exit 1; \
+	fi; \
+	echo "GitHub release $(TAG) has $${ASSET_COUNT} assets, all uploaded successfully"
+
+github-get-release: github-token-check github-tag-check
+	@echo "Fetching GitHub release $(TAG)..."
+	@curl --fail -sSL \
+		-H "Authorization: Bearer $(GITHUB_TOKEN)" \
+		"https://api.github.com/repos/VictoriaMetrics/VictoriaMetrics/releases?per_page=5" \
+		| jq '.[] | select(.draft == true and .tag_name == "$(TAG)")' > "$(GITHUB_RELEASE_SPEC_FILE)"
+	@if [ ! -s "$(GITHUB_RELEASE_SPEC_FILE)" ]; then \
+		echo "Could not find draft release $(TAG)"; \
+		exit 1; \
+	fi
+	@echo "Release data was stored in $(GITHUB_RELEASE_SPEC_FILE)"