From 62f5638872b49807c88e68040c56d447abe4d4f5 Mon Sep 17 00:00:00 2001 From: JK <19386515+Johnny-Kao@users.noreply.github.com> Date: Thu, 1 Oct 2026 01:21:11 +0900 Subject: [PATCH 1/2] Avoid redundant wire header normalization --- h11/_headers.py | 89 +++++++++++++++++++++++++++----------------- h11/_readers.py | 40 +++++++++++++++----- h11/tests/test_io.py | 10 +++++ 3 files changed, 95 insertions(+), 44 deletions(-) diff --git a/h11/_headers.py b/h11/_headers.py index 31da3e2..9e5e4e7 100644 --- a/h11/_headers.py +++ b/h11/_headers.py @@ -1,5 +1,15 @@ import re -from typing import AnyStr, cast, List, overload, Sequence, Tuple, TYPE_CHECKING, Union +from typing import ( + AnyStr, + cast, + List, + Optional, + overload, + Sequence, + Tuple, + TYPE_CHECKING, + Union, +) from ._abnf import field_name, field_value from ._util import bytesify, LocalProtocolError, validate @@ -132,6 +142,41 @@ def raw_items(self) -> List[Tuple[bytes, bytes]]: ] +def _normalize_special_header( + name: bytes, + value: bytes, + seen_content_length: Optional[bytes], + saw_transfer_encoding: bool, +) -> Tuple[bytes, Optional[bytes], bool, bool]: + if name == b"content-length": + lengths = {length.strip() for length in value.split(b",")} + if len(lengths) != 1: + raise LocalProtocolError("conflicting Content-Length headers") + value = lengths.pop() + validate(_content_length_re, value, "bad Content-Length") + if len(value) > CONTENT_LENGTH_MAX_DIGITS: + raise LocalProtocolError("bad Content-Length") + if seen_content_length is None: + seen_content_length = value + elif seen_content_length != value: + raise LocalProtocolError("conflicting Content-Length headers") + else: + return value, seen_content_length, saw_transfer_encoding, False + elif name == b"transfer-encoding": + if saw_transfer_encoding: + raise LocalProtocolError( + "multiple Transfer-Encoding headers", error_status_hint=501 + ) + value = value.lower() + if value != b"chunked": + raise LocalProtocolError( + "Only Transfer-Encoding: chunked is supported", + error_status_hint=501, + ) + saw_transfer_encoding = True + return value, seen_content_length, saw_transfer_encoding, True + + @overload def normalize_and_validate(headers: Headers, _parsed: Literal[True]) -> Headers: ... @@ -169,39 +214,15 @@ def normalize_and_validate( raw_name = name name = name.lower() - if name == b"content-length": - lengths = {length.strip() for length in value.split(b",")} - if len(lengths) != 1: - raise LocalProtocolError("conflicting Content-Length headers") - value = lengths.pop() - validate(_content_length_re, value, "bad Content-Length") - if len(value) > CONTENT_LENGTH_MAX_DIGITS: - raise LocalProtocolError("bad Content-Length") - if seen_content_length is None: - seen_content_length = value - new_headers.append((raw_name, name, value)) - elif seen_content_length != value: - raise LocalProtocolError("conflicting Content-Length headers") - elif name == b"transfer-encoding": - # "A server that receives a request message with a transfer coding - # it does not understand SHOULD respond with 501 (Not - # Implemented)." - # https://tools.ietf.org/html/rfc7230#section-3.3.1 - if saw_transfer_encoding: - raise LocalProtocolError( - "multiple Transfer-Encoding headers", error_status_hint=501 - ) - # "All transfer-coding names are case-insensitive" - # -- https://tools.ietf.org/html/rfc7230#section-4 - value = value.lower() - if value != b"chunked": - raise LocalProtocolError( - "Only Transfer-Encoding: chunked is supported", - error_status_hint=501, - ) - saw_transfer_encoding = True - new_headers.append((raw_name, name, value)) - else: + ( + value, + seen_content_length, + saw_transfer_encoding, + keep, + ) = _normalize_special_header( + name, value, seen_content_length, saw_transfer_encoding + ) + if keep: new_headers.append((raw_name, name, value)) return Headers(new_headers) diff --git a/h11/_readers.py b/h11/_readers.py index 576804c..51eb8e1 100644 --- a/h11/_readers.py +++ b/h11/_readers.py @@ -21,6 +21,7 @@ from ._abnf import chunk_header, header_field, request_line, status_line from ._events import Data, EndOfMessage, InformationalResponse, Request, Response +from ._headers import _normalize_special_header, Headers from ._receivebuffer import ReceiveBuffer from ._state import ( CLIENT, @@ -61,12 +62,33 @@ def _obsolete_line_fold(lines: Iterable[bytes]) -> Iterable[bytes]: yield last -def _decode_header_lines( - lines: Iterable[bytes], -) -> Iterable[Tuple[bytes, bytes]]: +def _decode_header_lines(lines: Iterable[bytes]) -> Headers: + full_items = [] for line in _obsolete_line_fold(lines): - matches = validate(header_field_re, line, "illegal header line: {!r}", line) - yield (matches["field_name"], matches["field_value"]) + match = header_field_re.fullmatch(line) + if match is None: + raise LocalProtocolError(f"illegal header line: {line!r}") + raw_name = match["field_name"] + full_items.append((raw_name, raw_name.lower(), match["field_value"])) + + seen_content_length = None + saw_transfer_encoding = False + write_index = 0 + for raw_name, name, value in full_items: + ( + value, + seen_content_length, + saw_transfer_encoding, + keep, + ) = _normalize_special_header( + name, value, seen_content_length, saw_transfer_encoding + ) + if keep: + full_items[write_index] = (raw_name, name, value) + write_index += 1 + + del full_items[write_index:] + return Headers(full_items) request_line_re = re.compile(request_line.encode("ascii")) @@ -83,9 +105,7 @@ def maybe_read_from_IDLE_client(buf: ReceiveBuffer) -> Optional[Request]: matches = validate( request_line_re, lines[0], "illegal request line: {!r}", lines[0] ) - return Request( - headers=list(_decode_header_lines(lines[1:])), _parsed=True, **matches - ) + return Request(headers=_decode_header_lines(lines[1:]), _parsed=True, **matches) status_line_re = re.compile(status_line.encode("ascii")) @@ -111,7 +131,7 @@ def maybe_read_from_SEND_RESPONSE_server( InformationalResponse if status_code < 200 else Response ) return class_( - headers=list(_decode_header_lines(lines[1:])), + headers=_decode_header_lines(lines[1:]), _parsed=True, status_code=status_code, reason=reason, @@ -158,7 +178,7 @@ def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: lines = buf.maybe_extract_lines() if lines is None: return None - return EndOfMessage(headers=list(_decode_header_lines(lines))) + return EndOfMessage(headers=_decode_header_lines(lines)) if self._bytes_to_discard: data = buf.maybe_extract_at_most(len(self._bytes_to_discard)) if data is None: diff --git a/h11/tests/test_io.py b/h11/tests/test_io.py index 407e044..fb3fc16 100644 --- a/h11/tests/test_io.py +++ b/h11/tests/test_io.py @@ -547,6 +547,16 @@ def test_reject_garbage_in_header_line() -> None: ) +def test_header_syntax_error_precedes_semantic_error() -> None: + buf = makebuf( + b"GET / HTTP/1.1\r\n" b"Host: example.com \n" b"Content-Length:\n" b"0\r\n\r\n" + ) + reader = READERS[CLIENT, IDLE] + assert callable(reader) + with pytest.raises(LocalProtocolError, match="illegal header line"): + reader(buf) + + def test_reject_non_vchar_in_path() -> None: for bad_char in b"\x00\x20\x7f\xee": message = bytearray(b"HEAD /") From ec2e2dbbdc61587911dda3dd5f868f14ab0733f4 Mon Sep 17 00:00:00 2001 From: JK <19386515+Johnny-Kao@users.noreply.github.com> Date: Thu, 1 Oct 2026 01:22:42 +0900 Subject: [PATCH 2/2] Add news fragment for wire header optimization --- newsfragments/207.misc.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/207.misc.rst diff --git a/newsfragments/207.misc.rst b/newsfragments/207.misc.rst new file mode 100644 index 0000000..262793b --- /dev/null +++ b/newsfragments/207.misc.rst @@ -0,0 +1 @@ +Reduce temporary allocations and redundant header normalization when parsing headers from the wire.