From db57a7a10ebb4217786cb44077a5842fd7b22c00 Mon Sep 17 00:00:00 2001 From: Chuang Wang Date: Mon, 26 Oct 2026 10:53:52 -0700 Subject: [PATCH] atenet-egress: set TCP_NOTSENT_LOWAT on its sockets An upload to a destination that reads slowly can put the tunnel's 1 MiB HTTP/2 stream window into the send queue of the gateway's socket to the destination, charged to the gateway's container. Set TCP_NOTSENT_LOWAT to 16 KiB on the egress listener and on the four dynamic forward proxy clusters, so the backlog stays behind Envoy's buffer limit and the tunnel's flow control. In the Envoy benchmark a stalled tunnel alone on its connection dropped from 1420 to 48 KiB of socket memory, with throughput within run-to-run noise. Fixed buffer sizes are left out: a fixed SO_RCVBUF cut a tunnel's throughput by 90% at 10 ms round trip. Signed-off-by: Chuang Wang --- .../egresssocketoptions_manifest_test.go | 104 ++++++++++++++++++ manifests/ate-install/atenet-egress.yaml | 22 ++++ 2 files changed, 126 insertions(+) create mode 100644 cmd/atenet/internal/router/egresssocketoptions_manifest_test.go diff --git a/cmd/atenet/internal/router/egresssocketoptions_manifest_test.go b/cmd/atenet/internal/router/egresssocketoptions_manifest_test.go new file mode 100644 index 0000000000..1eb71c7c21 --- /dev/null +++ b/cmd/atenet/internal/router/egresssocketoptions_manifest_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "testing" + + "sigs.k8s.io/yaml" +) + +// socketOption is an Envoy SocketOption as the manifest writes it. +type socketOption struct { + Level int64 `json:"level"` + Name int64 `json:"name"` + IntValue int64 `json:"int_value"` + State string `json:"state"` +} + +// notSentLowat is the TCP_NOTSENT_LOWAT value in opts, or 0, checking that it +// is set before the socket binds. +func notSentLowat(t *testing.T, where string, opts []socketOption) int64 { + t.Helper() + for _, o := range opts { + // IPPROTO_TCP and TCP_NOTSENT_LOWAT. + if o.Level == 6 && o.Name == 25 { + if o.State != "STATE_PREBIND" { + t.Errorf("%s sets TCP_NOTSENT_LOWAT in state %q, want STATE_PREBIND", where, o.State) + } + return o.IntValue + } + } + return 0 +} + +// TestEgressManifestSetsTCPNotSentLowat checks that the egress gateway sets +// TCP_NOTSENT_LOWAT on every socket it holds: the egress listener's, from +// atunnel, and every cluster's that dials destinations. A slow reader then +// backs data up behind Envoy's buffer limit rather than in a kernel send +// queue charged to the gateway's container. See +// docs/network-socket-memory.md. +func TestEgressManifestSetsTCPNotSentLowat(t *testing.T) { + var bootstrap struct { + StaticResources struct { + Listeners []struct { + Name string `json:"name"` + SocketOptions []socketOption `json:"socket_options"` + } `json:"listeners"` + Clusters []struct { + Name string `json:"name"` + ClusterType *struct { + Name string `json:"name"` + } `json:"cluster_type"` + UpstreamBindConfig *struct { + SocketOptions []socketOption `json:"socket_options"` + } `json:"upstream_bind_config"` + } `json:"clusters"` + } `json:"static_resources"` + } + if err := yaml.Unmarshal([]byte(envoyConfig(t)), &bootstrap); err != nil { + t.Fatalf("parsing envoy.yaml: %v", err) + } + listenerFound := false + for _, l := range bootstrap.StaticResources.Listeners { + if l.Name != "egress" { + continue + } + listenerFound = true + if got := notSentLowat(t, "listener egress", l.SocketOptions); got != 16384 { + t.Errorf("listener egress sets TCP_NOTSENT_LOWAT to %d, want 16384", got) + } + } + if !listenerFound { + t.Error("found no egress listener; the manifest changed shape and this test is checking nothing") + } + found := 0 + for _, c := range bootstrap.StaticResources.Clusters { + if c.ClusterType == nil || c.ClusterType.Name != "envoy.clusters.dynamic_forward_proxy" { + continue + } + found++ + var opts []socketOption + if c.UpstreamBindConfig != nil { + opts = c.UpstreamBindConfig.SocketOptions + } + if got := notSentLowat(t, "cluster "+c.Name, opts); got != 16384 { + t.Errorf("cluster %s sets TCP_NOTSENT_LOWAT to %d, want 16384", c.Name, got) + } + } + if found == 0 { + t.Fatal("found no dynamic_forward_proxy cluster; the manifest changed shape and this test is checking nothing") + } +} diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index f5d2f5ddc1..e106179025 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -77,6 +77,10 @@ data: socket_address: { address: "::", ipv4_compat: true, port_value: 443 } # Limit single I/O read to 64 Kb to prevent overly active actors from starving other actors. per_connection_buffer_limit_bytes: 65535 + # TCP_NOTSENT_LOWAT on the sockets from atunnel, as on the forward + # proxy clusters below. See docs/network-socket-memory.md. + socket_options: + - { description: TCP_NOTSENT_LOWAT, level: 6, name: 25, int_value: 16384, state: STATE_PREBIND } filter_chains: # Named so ext_proc can read it back as xds.filter_chain_name. Must # match EgressFilterChainName in @@ -990,6 +994,12 @@ data: - name: egress_forward_proxy lb_policy: CLUSTER_PROVIDED connect_timeout: 5s + # TCP_NOTSENT_LOWAT: accept writes only while fewer than 16 KiB wait + # unsent, so data for a slow destination stays upstream instead of in + # the kernel. See docs/network-socket-memory.md. + upstream_bind_config: + socket_options: + - { description: TCP_NOTSENT_LOWAT, level: 6, name: 25, int_value: 16384, state: STATE_PREBIND } # Fits 1000 destinations x 32 worker threads x 2 requests at once per # destination per thread = 64000 connections. circuit_breakers: @@ -1039,6 +1049,10 @@ data: - name: egress_forward_proxy_grpc lb_policy: CLUSTER_PROVIDED connect_timeout: 5s + # TCP_NOTSENT_LOWAT, as on egress_forward_proxy. + upstream_bind_config: + socket_options: + - { description: TCP_NOTSENT_LOWAT, level: 6, name: 25, int_value: 16384, state: STATE_PREBIND } cluster_type: name: envoy.clusters.dynamic_forward_proxy typed_config: @@ -1075,6 +1089,10 @@ data: - name: egress_forward_proxy_cleartext lb_policy: CLUSTER_PROVIDED connect_timeout: 5s + # TCP_NOTSENT_LOWAT, as on egress_forward_proxy. + upstream_bind_config: + socket_options: + - { description: TCP_NOTSENT_LOWAT, level: 6, name: 25, int_value: 16384, state: STATE_PREBIND } # Sized like egress_forward_proxy, for the same reason. circuit_breakers: thresholds: @@ -1112,6 +1130,10 @@ data: - name: egress_forward_proxy_passthrough lb_policy: CLUSTER_PROVIDED connect_timeout: 5s + # TCP_NOTSENT_LOWAT, as on egress_forward_proxy. + upstream_bind_config: + socket_options: + - { description: TCP_NOTSENT_LOWAT, level: 6, name: 25, int_value: 16384, state: STATE_PREBIND } cluster_type: name: envoy.clusters.dynamic_forward_proxy typed_config: