From 04518e02b0d5713bbf223289564e50c40a8cf870 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sun, 15 Nov 2026 00:39:30 +0100 Subject: [PATCH] Limit the size and count of HTTP/1.1 response headers Nothing bounded the response header section: a server could send a header line that never ends, or headers without end, and the client kept buffering. The parser now caps the section at 1 MiB and 1000 fields, trailers included, and returns headers_too_large or too_many_headers past them. The max_header_size and max_headers request options change the caps. An incomplete header line is no longer rescanned from its start on every read. --- guides/http_guide.md | 13 +++ include/hackney_lib.hrl | 5 + src/hackney.erl | 21 +++- src/hackney_conn.erl | 29 ++++- src/hackney_http.erl | 66 +++++++++-- test/hackney_header_limits_tests.erl | 168 +++++++++++++++++++++++++++ 6 files changed, 281 insertions(+), 21 deletions(-) create mode 100644 test/hackney_header_limits_tests.erl diff --git a/guides/http_guide.md b/guides/http_guide.md index 06e916ae..b6300281 100644 --- a/guides/http_guide.md +++ b/guides/http_guide.md @@ -175,6 +175,19 @@ large downloads use the streaming API instead. hackney:get(URL, [], <<>>, [{max_body_size, 10 * 1024 * 1024}]). ``` +### Response Header Limits + +The header section of an HTTP/1.1 response, trailers included, is capped at +1 MiB and 1000 header fields. Past the size cap the request fails with +`{error, headers_too_large}`; past the count, with +`{error, too_many_headers}`. Set `max_header_size` or `max_headers` to change +them, or `infinity` to disable either. + +```erlang +hackney:get(URL, [], <<>>, [{max_header_size, 4 * 1024 * 1024}, + {max_headers, 5000}]). +``` + ### Stream Response Body (Async Mode) For incremental body streaming, use async mode: diff --git a/include/hackney_lib.hrl b/include/hackney_lib.hrl index 9308bf22..4012b9f2 100644 --- a/include/hackney_lib.hrl +++ b/include/hackney_lib.hrl @@ -18,6 +18,11 @@ type = auto :: atom(), max_line_length = 4096 :: integer(), max_empty_lines = 10 :: integer(), + max_header_size = 1048576 :: non_neg_integer() | infinity, + max_headers = 1000 :: non_neg_integer() | infinity, + header_size = 0 :: non_neg_integer(), + header_count = 0 :: non_neg_integer(), + scan_from = 0 :: non_neg_integer(), empty_lines = 0 :: integer(), state = on_first_line :: atom(), buffer = <<>> :: binary(), diff --git a/src/hackney.erl b/src/hackney.erl index 83e73231..50293976 100644 --- a/src/hackney.erl +++ b/src/hackney.erl @@ -663,6 +663,12 @@ request(Method, URL, Headers, Body) -> %% - max_body_size: Cap in bytes on a response body returned whole, after %% decompression (default 512 MiB, `infinity' disables it). Past it the %% request fails with {error, body_too_large}; stream larger bodies. +%% - max_header_size: Cap in bytes on an HTTP/1.1 response header section, +%% trailers included (default 1 MiB, `infinity' disables it). Past it the +%% request fails with {error, headers_too_large}. +%% - max_headers: Cap on the number of HTTP/1.1 response header fields, +%% trailers included (default 1000, `infinity' disables it). Past it the +%% request fails with {error, too_many_headers}. %% %% Returns: %% - {ok, Status, Headers, Body}: Success with response body @@ -1521,7 +1527,9 @@ sync_request_with_redirect(ConnPid, Method, Path, Headers, Body, WithBody, Optio MaxBodySize -> [{max_body_size, MaxBodySize} | ReqOpts0] end, case hackney_conn:send_request_headers(ConnPid, Method, Path, HeadersList, - with_checkout_timeout(ReqOpts, Options)) of + with_checkout_timeout( + with_header_limits(ReqOpts, Options), + Options)) of ok -> {ok, ConnPid}; {error, Reason} -> {error, Reason} end; @@ -1559,7 +1567,8 @@ sync_request_with_redirect_body(ConnPid, Method, Path, HeadersList, FinalBody, MaxBodySize -> [{max_body_size, MaxBodySize} | ReqOpts3] end, case hackney_conn:request(ConnPid, Method, Path, HeadersList, FinalBody, infinity, - with_checkout_timeout(ReqOpts, Options)) of + with_checkout_timeout(with_header_limits(ReqOpts, Options), + Options)) of %% HTTP/2 returns body directly - handle 4-tuple first {ok, Status, RespHeaders, RespBody} when Status >= 301, Status =< 303; Status =:= 307; Status =:= 308 -> %% HTTP/2 redirect status @@ -1779,7 +1788,8 @@ async_request(ConnPid, Method, Path, Headers, Body, AsyncMode, StreamTo, FollowR %% is handled atomically inside hackney_conn:do_request_async case hackney_conn:request_async(ConnPid, Method, Path, HeadersList, FinalBody, AsyncMode, StreamTo, FollowRedirect, - with_checkout_timeout(ReqOpts, Options)) of + with_checkout_timeout( + with_header_limits(ReqOpts, Options), Options)) of {ok, Ref} -> {ok, Ref}; {error, Reason} -> @@ -2339,6 +2349,11 @@ parse_proxy_url(Url) when is_binary(Url) -> _:_ -> {error, invalid_proxy_url} end. +%% @private Forward the HTTP/1.1 response header limits to the connection. +with_header_limits(ReqOpts, Options) -> + [{K, V} || K <- [max_header_size, max_headers], + V <- [proplists:get_value(K, Options)], V =/= undefined] ++ ReqOpts. + %% @private An HTTP/2 or HTTP/3 request refused at the server's limit on %% concurrent streams waits on the connection as long as a pool checkout %% would (hackney_conn:stream_call/3). diff --git a/src/hackney_conn.erl b/src/hackney_conn.erl index 49e2a24c..11c51ecb 100644 --- a/src/hackney_conn.erl +++ b/src/hackney_conn.erl @@ -193,6 +193,8 @@ inform_fun :: fun((integer(), binary(), list()) -> any()) | undefined, auto_decompress = false :: boolean(), max_body_size = ?DEFAULT_MAX_BODY_SIZE :: non_neg_integer() | infinity, + %% Per-request max_header_size / max_headers, as parser options. + header_limits = [] :: list(), %% Pool integration pool_pid :: pid() | undefined, %% If set, connection is from a pool @@ -1316,17 +1318,20 @@ connected({call, From}, {request, Method, Path, Headers, Body, ReqOpts}, Data) - auto_decompress = AutoDecompress, body_length = undefined, max_body_size = max_body_size(ReqOpts), + header_limits = header_limits(ReqOpts), recv_timeout = RecvTimeout }, {next_state, sending, NewData, [{next_event, internal, {send_request, Method, Path, Headers, Body}}]}; connected({call, From}, {request_async, Method, Path, Headers, Body, AsyncMode, StreamTo}, Data) -> %% Start a new async request (no redirect following) - do_request_async(From, Method, Path, Headers, Body, AsyncMode, StreamTo, false, Data); + do_request_async(From, Method, Path, Headers, Body, AsyncMode, StreamTo, false, + Data#conn_data{header_limits = []}); connected({call, From}, {request_async, Method, Path, Headers, Body, AsyncMode, StreamTo, FollowRedirect}, Data) -> %% Start a new async request with redirect option (HTTP/1.1) - do_request_async(From, Method, Path, Headers, Body, AsyncMode, StreamTo, FollowRedirect, Data); + do_request_async(From, Method, Path, Headers, Body, AsyncMode, StreamTo, FollowRedirect, + Data#conn_data{header_limits = []}); connected({call, From}, {request_async, Method, Path, Headers, Body, AsyncMode, StreamTo, FollowRedirect, ReqOpts}, #conn_data{protocol = http2} = Data) -> %% HTTP/2 async request with ReqOpts (fix for issue #832) @@ -1345,7 +1350,8 @@ connected({call, From}, {request_async, Method, Path, Headers, Body, AsyncMode, connected({call, From}, {request_async, Method, Path, Headers, Body, AsyncMode, StreamTo, FollowRedirect, ReqOpts}, Data) -> %% HTTP/1.1 async request with ReqOpts (fix for issue #832) RecvTimeout = proplists:get_value(recv_timeout, ReqOpts, Data#conn_data.recv_timeout), - NewData = Data#conn_data{recv_timeout = RecvTimeout}, + NewData = Data#conn_data{recv_timeout = RecvTimeout, + header_limits = header_limits(ReqOpts)}, do_request_async(From, Method, Path, Headers, Body, AsyncMode, StreamTo, FollowRedirect, NewData); connected({call, From}, {send_headers, Method, Path, Headers, ReqOpts}, #conn_data{protocol = http3} = Data) -> @@ -1395,6 +1401,7 @@ connected({call, From}, {send_headers, Method, Path, Headers, ReqOpts}, Data) -> async_ref = undefined, stream_to = undefined, max_body_size = max_body_size(ReqOpts), + header_limits = header_limits(ReqOpts), body_length = undefined }, %% Transition to streaming_body state @@ -1847,7 +1854,7 @@ receiving(internal, {do_recv_response, include_pid}, Data) -> receiving(internal, do_recv_response_async, Data) -> %% Receive and parse response status and headers (async mode) - Parser = hackney_http:parser([response]), + Parser = response_parser(Data), DataWithParser = Data#conn_data{parser = Parser}, #conn_data{async_ref = Ref, stream_to = StreamTo, async = AsyncMode, follow_redirect = FollowRedirect, method = Method} = Data, @@ -2478,7 +2485,7 @@ finish_async_streaming(#conn_data{transport = Transport, socket = Socket, %% @private Handle receiving response status and headers %% IncludePid determines whether to include pid in the response do_recv_response_impl(Data, IncludePid) -> - Parser = hackney_http:parser([response]), + Parser = response_parser(Data), DataWithParser = Data#conn_data{parser = Parser}, case recv_status_and_headers(DataWithParser) of {ok, Status, Headers, NewData} -> @@ -2870,7 +2877,7 @@ recv_status_and_headers_loop(Data) -> %% Reset parser for next response, preserving any buffered data %% The old parser's buffer may contain the start of the next response OldBuffer = hackney_http:get(OldParser, buffer), - NewParser = hackney_http:parser([response]), + NewParser = response_parser(Data), recv_status_and_headers_loop(NewData#conn_data{parser = NewParser, buffer = OldBuffer}); Other -> Other @@ -3001,6 +3008,16 @@ max_body_size(ReqOpts) -> Max -> Max end. +%% @private Per-request limits on the response header section, passed to +%% the parser. Unset options keep the parser defaults. +header_limits(ReqOpts) -> + [{K, V} || K <- [max_header_size, max_headers], + V <- [proplists:get_value(K, ReqOpts)], V =/= undefined]. + +%% @private A response parser with the request's header limits. +response_parser(#conn_data{header_limits = Limits}) -> + hackney_http:parser([response | Limits]). + body_within_limit(_Size, infinity) -> true; body_within_limit(Size, Max) -> Size =< Max. diff --git a/src/hackney_http.erl b/src/hackney_http.erl index 8f3a4145..d5c7c3dd 100644 --- a/src/hackney_http.erl +++ b/src/hackney_http.erl @@ -92,7 +92,9 @@ -type parser_option() :: request | response | auto | {max_empty_lines, integer()} -| {max_line_length, integer()}. +| {max_line_length, integer()} +| {max_header_size, non_neg_integer() | infinity} +| {max_headers, non_neg_integer() | infinity}. -type parser_options() :: [parser_option()]. @@ -116,6 +118,12 @@ parser() -> %% before we give up. %%
  • `{max_lines_empty, Max}': the maximum number of empty line we %% accept before the first line happen
  • +%%
  • `{max_header_size, Bytes}': the maximum size of the header section, +%% trailers included (default 1 MiB, or `infinity'). Past it the parser +%% returns `{error, headers_too_large}'.
  • +%%
  • `{max_headers, Count}': the maximum number of header fields, +%% trailers included (default 1000, or `infinity'). Past it the parser +%% returns `{error, too_many_headers}'.
  • %% -spec parser(parser_options()) -> parser(). parser(Options) -> @@ -298,8 +306,24 @@ parse_headers(#hparser{}=St) -> parse_header(St). -parse_header(#hparser{}=St) -> - parse_header_sep([<<"\r\n">>, <<"\n">>], St). +parse_header(#hparser{buffer=Buf, scan_from=From}=St) -> + %% Until a line ends, only scan the bytes that arrived since the last + %% call: rescanning the whole buffer on every read is quadratic in the + %% length of a header line. + Size = byte_size(Buf), + case binary:match(Buf, <<"\n">>, [{scope, {From, Size - From}}]) of + nomatch -> + header_more(St#hparser{scan_from=Size}); + _ -> + parse_header_sep([<<"\r\n">>, <<"\n">>], St#hparser{scan_from=0}) + end. + +%% A header line is still incomplete: wait for more, within max_header_size. +header_more(#hparser{buffer=Buf, header_size=HSize, max_header_size=Max}=St) -> + case HSize + byte_size(Buf) > Max of + true -> {error, headers_too_large}; + false -> {more, St} + end. parse_header_sep([], St) -> {more, St}; @@ -328,13 +352,24 @@ parse_header_(Sep, #hparser{buffer=Buf}=St) -> {more, St} end. -parse_header(Line, St) -> - {Key, Value} = case binary:split(Line, <<":">>) of - [K] -> {K, <<>>}; - [K, V] -> {K, hackney_bstr:trim(V)} - end, - St1 = update_parser_header(Key, Value, St), - {header, {Key, Value}, St1}. +parse_header(Line, #hparser{header_size=HSize, header_count=Count, + max_header_size=MaxSize, max_headers=MaxCount}=St) -> + HSize1 = HSize + byte_size(Line) + 2, + Count1 = Count + 1, + if + HSize1 > MaxSize -> + {error, headers_too_large}; + Count1 > MaxCount -> + {error, too_many_headers}; + true -> + {Key, Value} = case binary:split(Line, <<":">>) of + [K] -> {K, <<>>}; + [K, V] -> {K, hackney_bstr:trim(V)} + end, + St1 = update_parser_header(Key, Value, + St#hparser{header_size=HSize1, header_count=Count1}), + {header, {Key, Value}, St1} + end. %% Fast path for common headers using case-insensitive pattern matching %% This avoids calling to_lower/trim for every header @@ -410,14 +445,17 @@ parse_trailers(St) -> {ok, _Trailers, #hparser{buffer=Rest1}} -> {done, Rest1}; {more, St2} -> - {more, St2} + {more, St2}; + {error, _} = Error -> + Error end. parse_trailers(St, Acc) -> case parse_headers(St) of {header, Header, St2} -> parse_trailers(St2, [Header | Acc]); {headers_complete, St2} -> {ok, lists:reverse(Acc), St2}; - {more, St2} -> {more, St2} + {more, St2} -> {more, St2}; + {error, _} = Error -> Error end. parse_body(#hparser{body_state=waiting, method= <<"HEAD">>, buffer=Buffer}) -> @@ -642,6 +680,10 @@ parse_options([{max_line_length, MaxLength} | Rest], St) -> parse_options(Rest, St#hparser{max_line_length=MaxLength}); parse_options([{max_empty_lines, MaxEmptyLines} | Rest], St) -> parse_options(Rest, St#hparser{max_empty_lines=MaxEmptyLines}); +parse_options([{max_header_size, Max} | Rest], St) -> + parse_options(Rest, St#hparser{max_header_size=Max}); +parse_options([{max_headers, Max} | Rest], St) -> + parse_options(Rest, St#hparser{max_headers=Max}); parse_options([_ | Rest], St) -> parse_options(Rest, St). diff --git a/test/hackney_header_limits_tests.erl b/test/hackney_header_limits_tests.erl new file mode 100644 index 00000000..8eff6662 --- /dev/null +++ b/test/hackney_header_limits_tests.erl @@ -0,0 +1,168 @@ +%%% -*- erlang -*- +%%% +%%% This file is part of hackney released under the Apache 2 license. +%%% See the NOTICE for more information. +%%% +%%% Limits on the HTTP/1.1 response header section: max_header_size and +%%% max_headers, in the parser and through hackney:request/5. + +-module(hackney_header_limits_tests). + +-include_lib("eunit/include/eunit.hrl"). + +%% Parser + +defaults_accept_normal_headers_test() -> + Hs = [{<<"X-", (integer_to_binary(N))/binary>>, <<"v">>} || N <- lists:seq(1, 50)], + ?assertMatch({ok, Hs}, parse([], [header_block(Hs)])). + +unterminated_line_too_large_test() -> + Line = binary:copy(<<"a">>, 2000), + ?assertEqual({error, headers_too_large}, + parse([{max_header_size, 1024}], [<<"X-A: ">>, Line])), + %% same bytes, in small reads + ?assertEqual({error, headers_too_large}, + parse([{max_header_size, 1024}], split_every(<<"X-A: ", Line/binary>>, 7))). + +complete_lines_add_up_test() -> + Hs = [{<<"X-", (integer_to_binary(N))/binary>>, binary:copy(<<"v">>, 100)} + || N <- lists:seq(1, 20)], + ?assertEqual({error, headers_too_large}, + parse([{max_header_size, 1024}], [header_block(Hs)])), + ?assertMatch({ok, _}, parse([{max_header_size, 4096}], [header_block(Hs)])). + +too_many_headers_test() -> + Hs = [{<<"X-Pad">>, <<"v">>} || _ <- lists:seq(1, 11)], + ?assertEqual({error, too_many_headers}, + parse([{max_headers, 10}], [header_block(Hs)])), + ?assertMatch({ok, _}, parse([{max_headers, 11}], [header_block(Hs)])). + +default_header_count_test() -> + Hs = [{<<"X-Pad">>, <<"v">>} || _ <- lists:seq(1, 1001)], + ?assertEqual({error, too_many_headers}, parse([], [header_block(Hs)])). + +infinity_disables_test() -> + Line = binary:copy(<<"a">>, 2 * 1024 * 1024), + Hs = [{<<"X-A">>, Line} | [{<<"X-Pad">>, <<"v">>} || _ <- lists:seq(1, 1500)]], + ?assertMatch({ok, _}, parse([{max_header_size, infinity}, {max_headers, infinity}], + [header_block(Hs)])), + ?assertEqual({error, headers_too_large}, parse([], [header_block(Hs)])). + +%% Reading a long header line in many small reads must not rescan the +%% whole buffer each time (that was quadratic). +long_line_small_reads_test_() -> + {timeout, 60, fun() -> + Line = binary:copy(<<"a">>, 8 * 1024 * 1024), + Segs = split_every(<<"X-A: ", Line/binary, "\r\n\r\n">>, 4096), + T0 = erlang:monotonic_time(millisecond), + ?assertMatch({ok, [{<<"X-A">>, _}]}, + parse([{max_header_size, infinity}], Segs)), + ?assert(erlang:monotonic_time(millisecond) - T0 < 2000) + end}. + +folded_and_lf_only_headers_test() -> + Block = <<"Content-Type: multipart/related;\r\n\tboundary=x\r\nOther: y\r\n\r\n">>, + ?assertEqual({ok, [{<<"Content-Type">>, <<"multipart/related; boundary=x">>}, + {<<"Other">>, <<"y">>}]}, + parse([], split_every(Block, 3))), + ?assertEqual({ok, [{<<"A">>, <<"1">>}, {<<"B">>, <<"2">>}]}, + parse([], split_every(<<"A: 1\nB: 2\n\n">>, 2))). + +trailers_count_test() -> + P0 = hackney_http:parser([response, {max_headers, 3}]), + {response, _, 200, _, P1} = hackney_http:execute(P0, <<"HTTP/1.1 200 OK\r\n">>), + {header, _, P2} = hackney_http:execute(P1, <<"Transfer-Encoding: chunked\r\n">>), + {headers_complete, P3} = hackney_http:execute(P2, <<"\r\n">>), + ?assertEqual({error, too_many_headers}, + hackney_http:execute(P3, <<"0\r\nT1: a\r\nT2: b\r\nT3: c\r\n\r\n">>)). + +%% hackney:request/5 + +request_test_() -> + {setup, + fun() -> {ok, _} = application:ensure_all_started(hackney), ok end, + fun(_) -> ok end, + [{"unterminated header line", fun t_request_unterminated/0}, + {"too many headers", fun t_request_too_many/0}, + {"per-request override", fun t_request_override/0}, + {"async request", fun t_request_async/0}]}. + +t_request_unterminated() -> + Port = serve(fun(S) -> + ok = gen_tcp:send(S, <<"HTTP/1.1 200 OK\r\nX-A: ">>), + Chunk = binary:copy(<<"a">>, 65536), + [ok = gen_tcp:send(S, Chunk) || _ <- lists:seq(1, 32)] + end), + ?assertEqual({error, headers_too_large}, get(Port, [])). + +t_request_too_many() -> + Hs = [{<<"X-Pad">>, <<"v">>} || _ <- lists:seq(1, 1200)], + Port = serve(fun(S) -> + ok = gen_tcp:send(S, [<<"HTTP/1.1 200 OK\r\n">>, header_block(Hs)]) + end), + ?assertEqual({error, too_many_headers}, get(Port, [])). + +t_request_override() -> + Hs = [{<<"X-Pad">>, <<"v">>} || _ <- lists:seq(1, 1200)], + Reply = [<<"HTTP/1.1 200 OK\r\n">>, + header_block([{<<"Content-Length">>, <<"2">>} | Hs]), <<"ok">>], + Port = serve(fun(S) -> ok = gen_tcp:send(S, Reply) end), + ?assertMatch({ok, 200, _, <<"ok">>}, get(Port, [{max_headers, 2000}])), + Port2 = serve(fun(S) -> ok = gen_tcp:send(S, Reply) end), + ?assertEqual({error, too_many_headers}, get(Port2, [{max_headers, 100}])). + +t_request_async() -> + Hs = [{<<"X-Pad">>, <<"v">>} || _ <- lists:seq(1, 20)], + Port = serve(fun(S) -> + ok = gen_tcp:send(S, [<<"HTTP/1.1 200 OK\r\n">>, header_block(Hs)]) + end), + {ok, Ref} = get(Port, [async, {max_headers, 10}]), + receive + {hackney_response, Ref, Msg} -> ?assertEqual({error, too_many_headers}, Msg) + after 5000 -> ?assert(false) + end. + +%% helpers + +get(Port, Opts) -> + hackney:request(get, <<"http://127.0.0.1:", (integer_to_binary(Port))/binary, "/">>, + [], <<>>, [{pool, false}, {recv_timeout, 5000} | Opts]). + +%% One-shot server: reads the request, runs Reply on the socket, then holds +%% it open until the client goes away. +serve(Reply) -> + {ok, L} = gen_tcp:listen(0, [binary, {active, false}, {reuseaddr, true}]), + {ok, Port} = inet:port(L), + spawn(fun() -> + {ok, S} = gen_tcp:accept(L, 5000), + {ok, _} = gen_tcp:recv(S, 0, 5000), + catch Reply(S), + _ = gen_tcp:recv(S, 0, 10000), + gen_tcp:close(S), + gen_tcp:close(L) + end), + Port. + +header_block(Hs) -> + iolist_to_binary([[[K, <<": ">>, V, <<"\r\n">>] || {K, V} <- Hs], <<"\r\n">>]). + +%% Parse a header section fed as separate reads, the way hackney_conn does: +%% drain with execute/1 until the parser asks for more, then feed the next. +parse(Opts, Segments) -> + P0 = hackney_http:parser([response | Opts]), + {response, _, 200, _, P1} = hackney_http:execute(P0, <<"HTTP/1.1 200 OK\r\n">>), + feed(P1, Segments, []). + +feed(P, Segments, Acc) -> + step(hackney_http:execute(P), Segments, Acc). + +step({header, H, P}, Segs, Acc) -> feed(P, Segs, [H | Acc]); +step({headers_complete, _}, _Segs, Acc) -> {ok, lists:reverse(Acc)}; +step({error, _} = E, _Segs, _Acc) -> E; +step({more, _}, [], _Acc) -> incomplete; +step({more, P}, [Seg | Rest], Acc) -> step(hackney_http:execute(P, Seg), Rest, Acc). + +split_every(Bin, N) when byte_size(Bin) =< N -> [Bin]; +split_every(Bin, N) -> + <> = Bin, + [H | split_every(T, N)].