app/vmauth: allow to serve internal API and different address

vmauth uses 'lib/httpserver' for serving HTTP requests. This server
unconditionally defines built-in routes (such as '/metrics',
'/health', etc). It makes impossible to proxy `HTTP` requests to  backends with the same routes.
Since vmauth's httpserver matches built-in route and return local
response.

 This commit adds new flag `httpInternalListenAddr` with
default empty value. Which removes internal API routes from public
router and exposes it at separate http server.

For example given configuration disables private routes at `0.0.0.0:8427` address and serves it at `0.0.0.0:8426`:

`./bin/vmauth --auth.config=config.yaml --httpListenAddr=:8427 --httpInternalListenAddr=127.0.0.1:8426`

Related issues:
- https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6468
- https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7345

app/vmauth/main.go

@@ -31,7 +31,11 @@ import (
 )
 
 var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
+		"By default, serves internal API and proxy requests. "+
+		" See also -tls, -httpListenAddr.useProxyProtocol and -httpInternalListenAddr.")
+	httpInternalListenAddr = flagutil.NewArrayString("httpInternalListenAddr", "TCP address to listen for incoming internal API http requests. Such as /health, /-/reload, /debug/pprof, etc. "+
+		"If flag is set, vmauth no longer serves internal API at -httpListenAddr.")
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
@@ -91,7 +95,21 @@ func main() {
 	logger.Infof("starting vmauth at %q...", listenAddrs)
 	startTime := time.Now()
 	initAuthConfig()
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	disableInternalRoutes := len(*httpInternalListenAddr) > 0
+	rh := requestHandlerWithInternalRoutes
+	if disableInternalRoutes {
+		rh = requestHandler
+	}
+
+	serveOpts := httpserver.ServeOptions{
+		UseProxyProtocol:     useProxyProtocol,
+		DisableBuiltinRoutes: disableInternalRoutes,
+	}
+	go httpserver.ServeWithOpts(listenAddrs, rh, serveOpts)
+
+	if len(*httpInternalListenAddr) > 0 {
+		go httpserver.Serve(*httpInternalListenAddr, nil, internalRequestHandler)
+	}
 	logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -109,7 +127,7 @@ func main() {
 	logger.Infof("successfully stopped vmauth in %.3f seconds", time.Since(startTime).Seconds())
 }
 
-func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+func internalRequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch r.URL.Path {
 	case "/-/reload":
 		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
@@ -120,6 +138,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusOK)
 		return true
 	}
+	return false
+}
+
+func requestHandlerWithInternalRoutes(w http.ResponseWriter, r *http.Request) bool {
+	if internalRequestHandler(w, r) {
+		return true
+	}
+	return requestHandler(w, r)
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	ats := getAuthTokensFromRequest(r)
 	if len(ats) == 0 {

app/vmauth/main_test.go

@@ -52,7 +52,7 @@ func TestRequestHandler(t *testing.T) {
 		r.Header.Set("Pass-Header", "abc")
 
 		w := &fakeResponseWriter{}
-		if !requestHandler(w, r) {
+		if !requestHandlerWithInternalRoutes(w, r) {
 			t.Fatalf("unexpected false is returned from requestHandler")
 		}
 

apptest/testcase.go

@@ -2,6 +2,8 @@ package apptest
 
 import (
 	"fmt"
+	"os"
+	"path"
 	"testing"
 	"time"
 
@@ -134,6 +136,23 @@ func (c *vmcluster) ForceFlush(t *testing.T) {
 	}
 }
 
+// MustStartVmauth is a test helper function that starts an instance of
+// vmauth and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVmauth(instance string, flags []string, configFileYAML string) *Vmauth {
+	tc.t.Helper()
+
+	configFilePath := path.Join(tc.t.TempDir(), "config.yaml")
+	if err := os.WriteFile(configFilePath, []byte(configFileYAML), os.ModePerm); err != nil {
+		tc.t.Fatalf("cannot init vmauth: config file write failed: %s", err)
+	}
+	app, err := StartVmauth(instance, flags, tc.cli, configFilePath)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(instance, app)
+	return app
+}
+
 // MustStartDefaultCluster starts a typical cluster configuration with default
 // flags.
 func (tc *TestCase) MustStartDefaultCluster() PrometheusWriteQuerier {

apptest/tests/vmauth_routing_test.go

@@ -0,0 +1,182 @@
+package tests
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestVMAuthRouterWithAuth(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	var authorizedRequestsCount, unauthorizedRequestsCount int
+	backendWithAuth := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
+		authorizedRequestsCount++
+	}))
+	defer backendWithAuth.Close()
+	backend := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
+		unauthorizedRequestsCount++
+	}))
+	defer backend.Close()
+
+	authConfig := fmt.Sprintf(`
+users:
+- name: user1
+  username: ba-username
+  password: ba-password
+  url_prefix: %s
+unauthorized_user:
+   url_map:
+   - src_paths:
+     - /backend/health
+     - /backend/ready
+     url_prefix: %s
+  `, backendWithAuth.URL, backend.URL)
+
+	vmauth := tc.MustStartVmauth("vmauth", nil, authConfig)
+
+	makeGetRequestExpectCode := func(prepareRequest func(*http.Request), expectCode int) {
+		t.Helper()
+		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s", vmauth.GetHTTPListenAddr()), nil)
+		if err != nil {
+			t.Fatalf("cannot build http.Request: %s", err)
+		}
+		prepareRequest(req)
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatalf("cannot make http.Get request for target=%q: %s", req.URL, err)
+		}
+		responseText, err := io.ReadAll(resp.Body)
+		if err != nil {
+			t.Fatalf("cannot read response body: %s", err)
+		}
+		resp.Body.Close()
+		if resp.StatusCode != expectCode {
+			t.Fatalf("unexpected http response code: %d, want: %d, response text: %s", resp.StatusCode, expectCode, responseText)
+		}
+	}
+	assertBackendsRequestsCount := func(expectAuthorized, expectUnauthorized int) {
+		t.Helper()
+		if expectAuthorized != authorizedRequestsCount {
+			t.Fatalf("expected to have %d authorized proxied requests, got: %d", expectAuthorized, authorizedRequestsCount)
+		}
+
+		if expectUnauthorized != unauthorizedRequestsCount {
+			t.Fatalf("expected to have %d unauthorized proxied requests, got: %d", expectUnauthorized, unauthorizedRequestsCount)
+		}
+
+	}
+
+	makeGetRequestExpectCode(func(r *http.Request) {
+		r.URL.Path = "/backend/api"
+		r.URL.User = url.UserPassword("ba-username", "ba-password")
+	}, http.StatusOK)
+	assertBackendsRequestsCount(1, 0)
+
+	makeGetRequestExpectCode(func(r *http.Request) {
+		r.URL.Path = "/backend/health"
+	}, http.StatusOK)
+	assertBackendsRequestsCount(1, 1)
+
+	// remove unauthorized section and proxy only specified path for authorized
+	vmauth.UpdateConfiguration(t, fmt.Sprintf(`
+users:
+- name: user1
+  username: ba-username
+  password: ba-password
+  url_map:
+  - src_paths:
+    - /backend/health
+    url_prefix: %s
+`, backendWithAuth.URL))
+
+	// ensure unauthorized requests no longer served
+	makeGetRequestExpectCode(func(r *http.Request) {
+		r.URL.Path = "/backend/health"
+	}, http.StatusUnauthorized)
+	assertBackendsRequestsCount(1, 1)
+
+	makeGetRequestExpectCode(func(r *http.Request) {
+		r.URL.User = url.UserPassword("ba-username", "ba-password")
+		r.URL.Path = "/backend/health"
+	}, http.StatusOK)
+	assertBackendsRequestsCount(2, 1)
+
+	// url path is missing at proxy configuration
+	makeGetRequestExpectCode(func(r *http.Request) {
+		r.URL.User = url.UserPassword("ba-username", "ba-password")
+		r.URL.Path = "/backend"
+	}, http.StatusBadRequest)
+	assertBackendsRequestsCount(2, 1)
+
+}
+
+func TestVMAuthRouterWithInternalAddr(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	var proxiedRequestsCount int
+	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		proxiedRequestsCount++
+	}))
+	defer backend.Close()
+
+	authConfig := fmt.Sprintf(`
+unauthorized_user:
+   url_prefix: %s
+  `, backend.URL)
+
+	const (
+		// it's not possible to use random ports
+		// since it makes test flaky
+		listenPortPublic  = "50127"
+		listenPortPrivate = "50126"
+	)
+
+	vmauthFlags := []string{
+		fmt.Sprintf("-httpListenAddr=127.0.0.1:%s", listenPortPublic),
+		fmt.Sprintf("-httpInternalListenAddr=127.0.0.1:%s", listenPortPrivate),
+		"-flagsAuthKey=protected",
+	}
+	vmauth := tc.MustStartVmauth("vmauth", vmauthFlags, authConfig)
+
+	makeGetRequestExpectCode := func(targetURL string, expectCode int) {
+		t.Helper()
+		resp, err := http.Get(targetURL)
+		if err != nil {
+			t.Fatalf("cannot make http.Get request for target=%q: %s", targetURL, err)
+		}
+		responseText, err := io.ReadAll(resp.Body)
+		if err != nil {
+			t.Fatalf("cannot read response body: %s", err)
+		}
+		resp.Body.Close()
+		if resp.StatusCode != expectCode {
+			t.Fatalf("unexpected http response code: %d, want: %d, response text: %s", resp.StatusCode, expectCode, responseText)
+		}
+	}
+	assertBackendRequestsCount := func(expected int) {
+		t.Helper()
+		if proxiedRequestsCount != expected {
+			t.Fatalf("expected to have %d proxied requests, got: %d", expected, proxiedRequestsCount)
+		}
+	}
+	// built-in http server must reject request, since it protected with authKey
+	makeGetRequestExpectCode(fmt.Sprintf("http://127.0.0.1:%s/flags", listenPortPrivate), http.StatusUnauthorized)
+	assertBackendRequestsCount(0)
+
+	makeGetRequestExpectCode(fmt.Sprintf("http://127.0.0.1:%s/flags", listenPortPublic), http.StatusOK)
+	assertBackendRequestsCount(1)
+
+	// reload config and ensure that vmauth no longer proxies requests to the backend
+	vmauth.UpdateConfiguration(t, "")
+	makeGetRequestExpectCode(fmt.Sprintf("http://127.0.0.1:%s/flags", listenPortPrivate), http.StatusUnauthorized)
+	assertBackendRequestsCount(1)
+}

apptest/vmauth.go

@@ -0,0 +1,82 @@
+package apptest
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"syscall"
+	"testing"
+	"time"
+)
+
+var httpBuilitinListenAddrRE = regexp.MustCompile(`pprof handlers are exposed at http://(.*:\d{1,5})/debug/pprof/`)
+
+// Vmauth holds the state of a vmauth app and provides vmauth-specific
+// functions.
+type Vmauth struct {
+	*app
+	*ServesMetrics
+
+	httpListenAddr string
+	configFilePath string
+	cli            *Client
+}
+
+// StartVmauth starts an instance of vmauth with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr)
+func StartVmauth(instance string, flags []string, cli *Client, configFilePath string) (*Vmauth, error) {
+	extractREs := []*regexp.Regexp{
+		httpBuilitinListenAddrRE,
+	}
+
+	app, stderrExtracts, err := startApp(instance, "../../bin/vmauth", flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-httpListenAddr": "127.0.0.1:0",
+			"-auth.config":    configFilePath,
+		},
+		extractREs: extractREs,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Vmauth{
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
+			cli:        cli,
+		},
+		httpListenAddr: stderrExtracts[0],
+		configFilePath: configFilePath,
+		cli:            cli,
+	}, nil
+}
+
+// UpdateConfiguration performs configuration file reload for app and waits for configuration apply
+//
+// Due to second prescision of config reload metric, config cannot be reloaded more than 1 time in a second
+func (app *Vmauth) UpdateConfiguration(t *testing.T, configFileYAML string) {
+	t.Helper()
+	ct := int(time.Now().Unix())
+	if err := os.WriteFile(app.configFilePath, []byte(configFileYAML), os.ModePerm); err != nil {
+		t.Fatalf("unexpected error at UpdateConfiguration, cannot write configFile content: %s", err)
+	}
+	if err := app.process.Signal(syscall.SIGHUP); err != nil {
+		t.Fatalf("unexpected signal error: %s", err)
+	}
+	for range 10 {
+		ts := app.GetIntMetric(t, "vmauth_config_last_reload_success_timestamp_seconds")
+		if ts < ct {
+			time.Sleep(time.Millisecond * 100)
+			continue
+		}
+		return
+	}
+	t.Fatalf("timeout waiting for config reload success")
+}
+
+// GetHTTPListenAddr returns listen http addr
+func (app *Vmauth) GetHTTPListenAddr() string {
+	return app.httpListenAddr
+}

docs/changelog/CHANGELOG.md

@@ -20,6 +20,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 
 * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): improve startup times when opening a storage with the [retention](https://docs.victoriametrics.com/#retention) exceeding a few months.
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add the ability to switch the heatmap to a line chart. Now, vmui would suggest to switch to line graph display if heatmap can't be properly rendered. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8057).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): add `-httpInternalListenAddr` cmd-line flag to serve internal HTTP routes `/metrics`, `/flags`, etc. It allows properly route requests to backends with the same service routes as vmauth. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6468) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7345) for details.
 * FEATURE: expose `/-/healthy` and `/-/ready` endpoints as Prometheus does on vmselect endpoints, thus achieving full Prometheus compatibility for external dependencies.
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): properly perform graceful shutdown for kafka client.

docs/vmauth.md

@@ -1086,6 +1086,8 @@ It is recommended protecting the following endpoints with authKeys:
 * `/metrics` with `-metricsAuthKey` command-line flag, so unauthorized users couldn't access [vmauth metrics](#monitoring).
 * `/debug/pprof` with `-pprofAuthKey` command-line flag, so unauthorized users couldn't access [profiling information](#profiling).
 
+As an alternative, it's possible to serve internal API routes at the different listen address with command-line flag `-httpInternalListenAddr=127.0.0.1:8426`. {{% available_from "#" %}}
+
 `vmauth` also supports the ability to restrict access by IP - see [these docs](#ip-filters). See also [concurrency limiting docs](#concurrency-limiting).
 
 ## Automatic issuing of TLS certificates
@@ -1290,8 +1292,14 @@ See the docs at https://docs.victoriametrics.com/vmauth/ .
      HTTP request header to use for obtaining authorization tokens. By default auth tokens are read from Authorization request header
      Supports an array of values separated by comma or specified via multiple flags.
      Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
+  -httpInternalListenAddr array
+     TCP address to listen for incoming internal API http requests. Such as /health, /-/reload, /debug/pprof, etc.
+     If flag is set, vmauth no longer serves internal API at -httpListenAddr.
+     Supports an array of values separated by comma or specified via multiple flags.
+     Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
   -httpListenAddr array
-     TCP address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol
+     TCP address to listen for incoming http requests. By default, serves internal API and proxy requests.
+     See also -tls, -httpListenAddr.useProxyProtocol and -httpInternalListenAddr.
      Supports an array of values separated by comma or specified via multiple flags.
      Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
   -httpListenAddr.useProxyProtocol array

lib/httpserver/httpserver.go

@@ -83,6 +83,19 @@ type server struct {
 // In such cases the caller must serve the request.
 type RequestHandler func(w http.ResponseWriter, r *http.Request) bool
 
+// ServeOptions defiens optional parameters for http server
+type ServeOptions struct {
+	// UseProxyProtocol if is set to true for the corresponding addr, then the incoming connections are accepted via proxy protocol.
+	// See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
+	UseProxyProtocol *flagutil.ArrayBool
+	// DisableBuiltinRoutes whether not to serve built-in routes for the given server, such as:
+	// /health, /debug/pprof and few others
+	// In addition basic auth check and authKey checks will be disabled for the given addr
+	//
+	// Mostly required by http proxy servers, which peforms own authorization and requests routing
+	DisableBuiltinRoutes bool
+}
+
 // Serve starts an http server on the given addrs with the given optional rh.
 //
 // By default all the responses are transparently compressed, since egress traffic is usually expensive.
@@ -97,23 +110,47 @@ func Serve(addrs []string, useProxyProtocol *flagutil.ArrayBool, rh RequestHandl
 			return false
 		}
 	}
+	opts := ServeOptions{
+		UseProxyProtocol: useProxyProtocol,
+	}
 	for idx, addr := range addrs {
 		if addr == "" {
 			continue
 		}
-		useProxyProto := false
-		if useProxyProtocol != nil {
-			useProxyProto = useProxyProtocol.GetOptionalArg(idx)
-		}
-		go serve(addr, useProxyProto, rh, idx)
+		go serve(addr, rh, idx, opts)
 	}
 }
 
-func serve(addr string, useProxyProtocol bool, rh RequestHandler, idx int) {
+// ServeWithOpts starts an http server on the given addrs with the given optional request handlers.
+//
+// By default all the responses are transparently compressed, since egress traffic is usually expensive.
+//
+// The compression can be disabled by specifying -http.disableResponseCompression command-line flag.
+// TODO: rename to Serve and update Serve usage with new signature
+func ServeWithOpts(addrs []string, rh RequestHandler, opts ServeOptions) {
+	if rh == nil {
+		rh = func(_ http.ResponseWriter, _ *http.Request) bool {
+			return false
+		}
+	}
+	for idx, addr := range addrs {
+		if addr == "" {
+			continue
+		}
+		go serve(addr, rh, idx, opts)
+	}
+}
+
+func serve(addr string, rh RequestHandler, idx int, opts ServeOptions) {
 	scheme := "http"
 	if tlsEnable.GetOptionalArg(idx) {
 		scheme = "https"
 	}
+	useProxyProto := false
+	if opts.UseProxyProtocol != nil {
+		useProxyProto = opts.UseProxyProtocol.GetOptionalArg(idx)
+	}
+
 	var tlsConfig *tls.Config
 	if tlsEnable.GetOptionalArg(idx) {
 		certFile := tlsCertFile.GetOptionalArg(idx)
@@ -125,20 +162,22 @@ func serve(addr string, useProxyProtocol bool, rh RequestHandler, idx int) {
 		}
 		tlsConfig = tc
 	}
-	ln, err := netutil.NewTCPListener(scheme, addr, useProxyProtocol, tlsConfig)
+	ln, err := netutil.NewTCPListener(scheme, addr, useProxyProto, tlsConfig)
 	if err != nil {
 		logger.Fatalf("cannot start http server at %s: %s", addr, err)
-	} else {
-		logger.Infof("started server at %s://%s/", scheme, ln.Addr())
+	}
+	logger.Infof("started server at %s://%s/", scheme, ln.Addr())
+	if !opts.DisableBuiltinRoutes {
 		logger.Infof("pprof handlers are exposed at %s://%s/debug/pprof/", scheme, ln.Addr())
 	}
-	serveWithListener(addr, ln, rh)
+
+	serveWithListener(addr, ln, rh, opts.DisableBuiltinRoutes)
 }
 
-func serveWithListener(addr string, ln net.Listener, rh RequestHandler) {
+func serveWithListener(addr string, ln net.Listener, rh RequestHandler, disableBuiltinRoutes bool) {
 	var s server
+
 	s.s = &http.Server{
-		Handler: gzipHandler(&s, rh),
 
 		// Disable http/2, since it doesn't give any advantages for VictoriaMetrics services.
 		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
@@ -162,6 +201,19 @@ func serveWithListener(addr string, ln net.Listener, rh RequestHandler) {
 			return context.WithValue(ctx, connDeadlineTimeKey, &deadline)
 		}
 	}
+	rhw := rh
+	if !disableBuiltinRoutes {
+		rhw = func(w http.ResponseWriter, r *http.Request) bool {
+			return builtinRoutesHandler(&s, r, w, rh)
+		}
+	}
+	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handlerWrapper(w, r, rhw)
+	})
+	if !*disableResponseCompression {
+		h = gzipHandlerWrapper(h)
+	}
+	s.s.Handler = h
 
 	serversLock.Lock()
 	servers[addr] = &s
@@ -244,16 +296,6 @@ func stop(addr string) error {
 	return nil
 }
 
-func gzipHandler(s *server, rh RequestHandler) http.HandlerFunc {
-	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		handlerWrapper(s, w, r, rh)
-	})
-	if *disableResponseCompression {
-		return h
-	}
-	return gzipHandlerWrapper(h)
-}
-
 var gzipHandlerWrapper = func() func(http.Handler) http.HandlerFunc {
 	hw, err := gzhttp.NewWrapper(gzhttp.CompressionLevel(1))
 	if err != nil {
@@ -278,7 +320,7 @@ var hostname = func() string {
 	return h
 }()
 
-func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh RequestHandler) {
+func handlerWrapper(w http.ResponseWriter, r *http.Request, rh RequestHandler) {
 	// All the VictoriaMetrics code assumes that panic stops the process.
 	// Unfortunately, the standard net/http.Server recovers from panics in request handlers,
 	// so VictoriaMetrics state can become inconsistent after the recovered panic.
@@ -310,12 +352,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 		h.Set("Connection", "close")
 	}
 	path := r.URL.Path
-	if strings.HasSuffix(path, "/favicon.ico") {
-		w.Header().Set("Cache-Control", "max-age=3600")
-		faviconRequests.Inc()
-		w.Write(faviconData)
-		return
-	}
+
 	prefix := GetPathPrefix()
 	if prefix != "" {
 		// Trim -http.pathPrefix from path
@@ -336,13 +373,40 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 		path = path[len(prefix)-1:]
 		r.URL.Path = path
 	}
+
+	w = &responseWriterWithAbort{
+		ResponseWriter: w,
+	}
+	if rh(w, r) {
+		return
+	}
+
+	Errorf(w, r, "unsupported path requested: %q", r.URL.Path)
+	unsupportedRequestErrors.Inc()
+}
+
+func builtinRoutesHandler(s *server, r *http.Request, w http.ResponseWriter, rh RequestHandler) bool {
+	if !isProtectedByAuthFlag(r.URL.Path) && !CheckBasicAuth(w, r) {
+		return true
+	}
+
+	h := w.Header()
+
+	path := r.URL.Path
+	if strings.HasSuffix(path, "/favicon.ico") {
+		w.Header().Set("Cache-Control", "max-age=3600")
+		faviconRequests.Inc()
+		w.Write(faviconData)
+		return true
+	}
+
 	switch r.URL.Path {
 	case "/health":
 		h.Set("Content-Type", "text/plain; charset=utf-8")
 		deadline := s.shutdownDelayDeadline.Load()
 		if deadline <= 0 {
 			w.Write([]byte("OK"))
-			return
+			return true
 		}
 		// Return non-OK response during grace period before shutting down the server.
 		// Load balancers must notify these responses and re-route new requests to other servers.
@@ -353,7 +417,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 		}
 		errMsg := fmt.Sprintf("The server is in delayed shutdown mode, which will end in %.3fs", d.Seconds())
 		http.Error(w, errMsg, http.StatusServiceUnavailable)
-		return
+		return true
 	case "/ping":
 		// This is needed for compatibility with InfluxDB agents.
 		// See https://docs.influxdata.com/influxdb/v1.7/tools/api/#ping-http-endpoint
@@ -362,64 +426,54 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 			status = http.StatusOK
 		}
 		w.WriteHeader(status)
-		return
+		return true
 	case "/metrics":
 		metricsRequests.Inc()
 		if !CheckAuthFlag(w, r, metricsAuthKey) {
-			return
+			return true
 		}
 		startTime := time.Now()
 		h.Set("Content-Type", "text/plain; charset=utf-8")
 		appmetrics.WritePrometheusMetrics(w)
 		metricsHandlerDuration.UpdateDuration(startTime)
-		return
+		return true
 	case "/flags":
 		if !CheckAuthFlag(w, r, flagsAuthKey) {
-			return
+			return true
 		}
 		h.Set("Content-Type", "text/plain; charset=utf-8")
 		flagutil.WriteFlags(w)
-		return
+		return true
 	case "/-/healthy":
 		// This is needed for Prometheus compatibility
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1833
 		fmt.Fprintf(w, "VictoriaMetrics is Healthy.\n")
-		return
+		return true
 	case "/-/ready":
 		// This is needed for Prometheus compatibility
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1833
 		fmt.Fprintf(w, "VictoriaMetrics is Ready.\n")
-		return
+		return true
 	case "/robots.txt":
 		// This prevents search engines from indexing contents
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4128
 		fmt.Fprintf(w, "User-agent: *\nDisallow: /\n")
-		return
+		return true
 	default:
 		if strings.HasPrefix(r.URL.Path, "/debug/pprof/") {
 			pprofRequests.Inc()
 			if !CheckAuthFlag(w, r, pprofAuthKey) {
-				return
+				return true
 			}
 			pprofHandler(r.URL.Path[len("/debug/pprof/"):], w, r)
-			return
+			return true
 		}
 
 		if !isProtectedByAuthFlag(r.URL.Path) && !CheckBasicAuth(w, r) {
-			return
+			return true
 		}
-
-		w = &responseWriterWithAbort{
-			ResponseWriter: w,
-		}
-		if rh(w, r) {
-			return
-		}
-
-		Errorf(w, r, "unsupported path requested: %q", r.URL.Path)
-		unsupportedRequestErrors.Inc()
-		return
 	}
+	return rh(w, r)
 }
 
 func isProtectedByAuthFlag(path string) bool {

lib/httpserver/httpserver_test.go

@@ -162,8 +162,11 @@ func TestHandlerWrapper(t *testing.T) {
 
 	srv := &server{s: &http.Server{}}
 	w := &httptest.ResponseRecorder{}
-	handlerWrapper(srv, w, req, func(_ http.ResponseWriter, _ *http.Request) bool {
-		return true
+
+	handlerWrapper(w, req, func(w http.ResponseWriter, r *http.Request) bool {
+		return builtinRoutesHandler(srv, r, w, func(_ http.ResponseWriter, _ *http.Request) bool {
+			return true
+		})
 	})
 
 	h := w.Header()