Apply suggestions from code review

Co-authored-by: Roman Khavronenko <hagen1778@gmail.com>
Signed-off-by: Zhu Jiekun <jiekun@victoriametrics.com>

Makefile

@@ -471,7 +471,7 @@ test-full-386:
 
 apptest:
 	$(MAKE) victoria-metrics-race vmagent-race vmalert-race vmauth-race vmctl-race vmbackup-race vmrestore-race
-	go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
+	VM_APPTEST=1 go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
 
 apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \

apptest/tests/apptest_test.go

@@ -0,0 +1,40 @@
+package tests
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+func TestMain(m *testing.M) {
+	// check if necessary binaries are there.
+	checkBinaryRequirement("../../bin/victoria-metrics-race")
+	checkBinaryRequirement("../../bin/vmagent-race")
+	checkBinaryRequirement("../../bin/vmauth-race")
+	checkBinaryRequirement("../../bin/vmctl-race")
+	checkBinaryRequirement("../../bin/vmbackup-race")
+	checkBinaryRequirement("../../bin/vmrestore-race")
+
+	// check if the test is run via make command
+	checkIntegrationTestEnv()
+
+	// start the integration test.
+	os.Exit(m.Run())
+}
+
+// checkBinaryRequirement panic if required binary not exist.
+func checkBinaryRequirement(path string) {
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			panic(fmt.Sprintf("apptest failed: %s not found. please run `make apptest` to execute apptest. check how different tests are executed: https://docs.victoriametrics.com/victoriametrics/contributing/#testing", path))
+		}
+	}
+}
+
+func checkIntegrationTestEnv() {
+	if os.Getenv("VM_APPTEST") == "" {
+		logger.Warnf("executing apptest with potential outdated binaries. it's recommended to execute via `make apptest` command. check this doc for more details: https://docs.victoriametrics.com/victoriametrics/contributing/#testing")
+	}
+}

docs/victoriametrics/CONTRIBUTING.md

@@ -68,8 +68,7 @@ Pull requests requirements:
 1. A link to the issue(s) related to the change, if any. Use `Fixes [issue link]` if the PR resolves the issue, or `Related to [issue link]` for reference.
 1. Tests proving that the change is effective. Tests are expected for non-trivial new functionality or non-trivial modifications.
    Bug fixes must include tests unless a maintainer explicitly agrees otherwise.
-   See [this style guide](https://itnext.io/f-tests-as-a-replacement-for-table-driven-tests-in-go-8814a8b19e9e) for tests.
-   To run tests and code checks locally, execute commands `make test-full` and `make check-all`.
+   See [this style guide](https://itnext.io/f-tests-as-a-replacement-for-table-driven-tests-in-go-8814a8b19e9e) for tests. See [this section](#testing) for how to run tests.
 1. Try to not extend the scope of the pull requests outside the issue, do not make unrelated changes.
 1. Update [docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs) if needed. For example, adding a new flag or changing behavior of existing flags or features
    requires reflecting these changes in the documentation. For new features add `{{%/* available_from "#" */%}}` shortcode to the documentation.
@@ -126,3 +125,17 @@ Due to `KISS`, [cluster version of VictoriaMetrics](https://docs.victoriametrics
 - Automatic cluster resizing, which may cost you a lot of money if improperly configured.
 - Automatic discovering and addition of new nodes in the cluster, which may mix data between dev and prod clusters :)
 - Automatic leader election, which may result in split brain disaster on network errors.
+
+## Testing
+
+We recommend running the following sequence of checks and tests before submitting a pull request:
+```sh
+# run static checks
+make check-all
+
+# run unit test
+make test-full
+
+# run integration tests
+make apptest
+```