rms/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2026-05-17 08:36:55 +03:00
Code Issues Packages Projects Releases Wiki Activity

http server: limit max concurrent requests (#4185)

Browse Source 
* lib/httpserver: introduce `-http.maxConcurrentRequests` command-line flag

Introduce `-http.maxConcurrentRequests` command-line flag to protect
VM components from resource exhaustion during unexpected spikes of HTTP requests.
By default, the new flag's value is set to 0 which means no limits are applied.

Signed-off-by: hagen1778 <roman@victoriametrics.com>

* lib/httpserver: mention http.maxConcurrentRequests in docs

Signed-off-by: hagen1778 <roman@victoriametrics.com>

---------

Signed-off-by: hagen1778 <roman@victoriametrics.com>
This commit is contained in:
Roman Khavronenko
2023-04-24 14:52:06 +02:00
committed by GitHub
parent 31e174977e
commit 77f76371d0
20 changed files with 66 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
lib/httpserver/httpserver.go
View File

@@ -51,6 +51,10 @@ var (
shutdownDelay = flag.Duration("http.shutdownDelay", 0, `Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers`)
idleConnTimeout = flag.Duration("http.idleConnTimeout", time.Minute, "Timeout for incoming idle http connections")
connTimeout = flag.Duration("http.connTimeout", 2*time.Minute, `Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem`)
maxConcurrentRequests = flag.Int("http.maxConcurrentRequests", 0, "The maximum number of concurrent HTTP requests. "+
"Use this flag as a safety measure to prevent from overloading during attacks or thundering herd problem."+
"Value should depend on the amount of free memory and number of free file descriptors. The more memory/descriptors is available, the more concurrent requests can be served."+
"If set to zero - no limits are applied.")
)
var (
@@ -60,6 +64,7 @@ var (
type server struct {
shutdownDelayDeadline int64
concurrencyLimiter chan struct{}
s *http.Server
}
@@ -71,9 +76,9 @@ type server struct {
// In such cases the caller must serve the request.
type RequestHandler func(w http.ResponseWriter, r *http.Request) bool
// Serve starts an http server on the given addr with the given optional rh.
// Serve starts a http server on the given addr with the given optional rh.
//
// By default all the responses are transparently compressed, since egress traffic is usually expensive.
// By default, all the responses are transparently compressed, since egress traffic is usually expensive.
//
// The compression is also disabled if -http.disableResponseCompression flag is set.
//
@@ -136,6 +141,7 @@ func serveWithListener(addr string, ln net.Listener, rh RequestHandler) {
return context.WithValue(ctx, connDeadlineTimeKey, &deadline)
},
}
s.concurrencyLimiter = make(chan struct{}, *maxConcurrentRequests)
serversLock.Lock()
servers[addr] = &s
serversLock.Unlock()
@@ -341,6 +347,19 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
if !CheckBasicAuth(w, r) {
return
}
if *maxConcurrentRequests > 0 {
select {
case s.concurrencyLimiter <- struct{}{}:
default:
Errorf(w, r, "couldn't start processing the request at path %q, "+
"since -http.maxConcurrentRequests=%d concurrent requests are executed.", r.URL.Path, *maxConcurrentRequests)
limitReachedRequestErrors.Inc()
return
}
defer func() { <-s.concurrencyLimiter }()
}
if rh(w, r) {
return
}
@@ -429,7 +448,8 @@ var (
pprofDefaultRequests = metrics.NewCounter(`vm_http_requests_total{path="/debug/pprof/default"}`)
faviconRequests = metrics.NewCounter(`vm_http_requests_total{path="/favicon.ico"}`)
unsupportedRequestErrors = metrics.NewCounter(`vm_http_request_errors_total{path="*", reason="unsupported"}`)
unsupportedRequestErrors = metrics.NewCounter(`vm_http_request_errors_total{path="*", reason="unsupported"}`)
limitReachedRequestErrors = metrics.NewCounter(`vm_http_request_errors_total{path="*", reason="concurrency limit"}`)
requestsTotal = metrics.NewCounter(`vm_http_requests_all_total`)
)