From 5bfa8b83f361368de2187b7ba201c60a8ebb8640 Mon Sep 17 00:00:00 2001 From: Ignasi Barrera Date: Mon, 12 Oct 2026 18:52:32 +0200 Subject: [PATCH 1/2] server: configurable gRPC max concurrent streams While Redis was unresponsive, every rate limit call blocked inside the service, and with no limit on concurrent streams the blocked calls, and the goroutines and memory serving them, kept piling up for as long as Redis stayed away. Add GRPC_MAX_CONCURRENT_STREAMS (default 0, the gRPC default) to bound the concurrent calls per gRPC connection, so excess calls wait on the client side instead. Signed-off-by: Ignasi Barrera --- README.md | 5 +++++ src/server/server_impl.go | 3 +++ src/settings/settings.go | 3 +++ src/settings/settings_test.go | 7 +++++++ 4 files changed, 18 insertions(+) diff --git a/README.md b/README.md index 3a2de3e04..52ed1361e 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ - [xDS Management Server Based Configuration Loading](#xds-management-server-based-configuration-loading) - [Log Format](#log-format) - [GRPC Keepalive](#grpc-keepalive) + - [GRPC Concurrency](#grpc-concurrency) - [Health-check](#health-check) - [Health-check configurations](#health-check-configurations) - [GRPC server](#grpc-server) @@ -930,6 +931,10 @@ The behavior can be fixed by configuring the following env variables for the rat - `GRPC_MAX_CONNECTION_AGE`: a duration for the maximum amount of time a connection may exist before it will be closed by sending a GoAway. A random jitter of +/-10% will be added to MaxConnectionAge to spread out connection storms. - `GRPC_MAX_CONNECTION_AGE_GRACE`: an additive period after MaxConnectionAge after which the connection will be forcibly closed. +## GRPC Concurrency + +- `GRPC_MAX_CONCURRENT_STREAMS`: maximum number of concurrent calls on each gRPC connection. Calls beyond it wait on the client side (HTTP/2 flow control) instead of piling up in the rate limit service while it is slow, e.g. because Redis is. Size it above the normal number of concurrent calls per connection (calls per second per connection × latency). `0` keeps the gRPC default (unbounded). Default: `0` + ## Health-check Health check status is determined internally by individual components. diff --git a/src/server/server_impl.go b/src/server/server_impl.go index 3b7a93266..966f004be 100644 --- a/src/server/server_impl.go +++ b/src/server/server_impl.go @@ -255,6 +255,9 @@ func newServer(s settings.Settings, name string, statsManager stats.Manager, loc grpc.UnaryInterceptor(s.GrpcUnaryInterceptor), grpc.StatsHandler(otelgrpc.NewServerHandler()), } + if s.GrpcMaxConcurrentStreams > 0 { + grpcOptions = append(grpcOptions, grpc.MaxConcurrentStreams(s.GrpcMaxConcurrentStreams)) + } if s.GrpcServerUseTLS { grpcServerTlsConfig := s.GrpcServerTlsConfig ret.grpcCertProvider = provider.NewCertProvider(s, ret.store, s.GrpcServerTlsCert, s.GrpcServerTlsKey) diff --git a/src/settings/settings.go b/src/settings/settings.go index d11b56dc8..ea5950243 100644 --- a/src/settings/settings.go +++ b/src/settings/settings.go @@ -33,6 +33,9 @@ type Settings struct { GrpcMaxConnectionAge time.Duration `envconfig:"GRPC_MAX_CONNECTION_AGE" default:"24h" description:"Duration a connection may exist before it will be closed by sending a GoAway."` // GrpcMaxConnectionAgeGrace is an additive period after MaxConnectionAge after which the connection will be forcibly closed. GrpcMaxConnectionAgeGrace time.Duration `envconfig:"GRPC_MAX_CONNECTION_AGE_GRACE" default:"1h" description:"Period after MaxConnectionAge after which the connection will be forcibly closed."` + // GrpcMaxConcurrentStreams bounds the number of concurrent calls (streams) + // on each gRPC connection. 0 keeps the gRPC default (unbounded). + GrpcMaxConcurrentStreams uint32 `envconfig:"GRPC_MAX_CONCURRENT_STREAMS" default:"0"` // GrpcServerUseTLS enables gprc connections to server over TLS GrpcServerUseTLS bool `envconfig:"GRPC_SERVER_USE_TLS" default:"false"` // Allow to set the server certificate and key for TLS connections. diff --git a/src/settings/settings_test.go b/src/settings/settings_test.go index c6ab2fd0c..7368e7735 100644 --- a/src/settings/settings_test.go +++ b/src/settings/settings_test.go @@ -143,3 +143,10 @@ func TestRedisPoolOnEmptyBehavior_IndependentConfiguration(t *testing.T) { // Per-second pool configured differently assert.Equal(t, "CREATE", settings.RedisPerSecondPoolOnEmptyBehavior) } + +func TestGrpcMaxConcurrentStreams(t *testing.T) { + assert.Equal(t, uint32(0), NewSettings().GrpcMaxConcurrentStreams) + + t.Setenv("GRPC_MAX_CONCURRENT_STREAMS", "128") + assert.Equal(t, uint32(128), NewSettings().GrpcMaxConcurrentStreams) +} From 157db82c3b59bb26b4344656f423e149d51647f9 Mon Sep 17 00:00:00 2001 From: Prabhjot Singh <86856448+psbrar99@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:56:22 -0600 Subject: [PATCH 2/2] Update Redis connection timeout descriptions Clarified timeout settings for Redis connection establishment. Signed-off-by: Prabhjot Singh <86856448+psbrar99@users.noreply.github.com> --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 52ed1361e..85ad7474f 100644 --- a/README.md +++ b/README.md @@ -1351,10 +1351,10 @@ The deployment type can be specified with the `REDIS_TYPE` / `REDIS_PERSECOND_TY ### Connection Timeout -Controls the maximum duration for Redis connection establishment, read operations, and write operations. +Controls the maximum duration for Redis connection establishment. -1. `REDIS_TIMEOUT`: sets the timeout for Redis connection and I/O operations. Default: `10s` -1. `REDIS_PERSECOND_TIMEOUT`: sets the timeout for per-second Redis connection and I/O operations. Default: `10s` +1. `REDIS_TIMEOUT`: sets the timeout for Redis connection establishment. Default: `10s` +1. `REDIS_PERSECOND_TIMEOUT`: sets the timeout for per-second Redis connection establishment. Default: `10s` ### Pool On-Empty Behavior