From ae8365c1fe49678ab689bdb76e12bb684f7e571e Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 16 Oct 2026 13:43:51 +0200 Subject: [PATCH] feat: add a `user_agent_suffix` option --- docs/02_concepts/10_custom_http_clients.mdx | 2 + src/apify_client/_apify_client.py | 12 ++++ src/apify_client/http_clients/_base.py | 5 ++ src/apify_client/http_clients/_httpx2.py | 6 ++ src/apify_client/http_clients/_impit.py | 6 ++ tests/unit/test_client_headers.py | 77 +++++++++++++++++++++ 6 files changed, 108 insertions(+) diff --git a/docs/02_concepts/10_custom_http_clients.mdx b/docs/02_concepts/10_custom_http_clients.mdx index b7e7f08b..ad05f9f1 100644 --- a/docs/02_concepts/10_custom_http_clients.mdx +++ b/docs/02_concepts/10_custom_http_clients.mdx @@ -45,6 +45,8 @@ You can configure the default client through the +To identify a tool built on top of the client, pass `user_agent_suffix`. The client appends it to its default `User-Agent` header after a semicolon and a space. Pass a list, such as `['MyTool/1.0', 'MyPlugin/2.0']`, to append several parts. The built-in HTTP clients accept the same option, so with `with_custom_http_client`, pass it to the HTTP client's constructor. + ## Built-in HTTPX2 client The package also provides `Httpx2HttpClient` and `Httpx2HttpClientAsync`. They use the same request preparation, compression, retry policy, timeout tiers and growth, error handling, logging, and statistics as the default Impit clients, with [HTTPX2](https://github.com/pydantic/httpx2) as the transport. diff --git a/src/apify_client/_apify_client.py b/src/apify_client/_apify_client.py index 73beadef..08664bbb 100644 --- a/src/apify_client/_apify_client.py +++ b/src/apify_client/_apify_client.py @@ -126,6 +126,7 @@ def __init__( timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, headers: dict[str, str] | None = None, + user_agent_suffix: str | list[str] | None = None, compression: HttpCompressionAlgorithm | HttpCompressor = 'gzip', ) -> None: """Initialize the Apify API client. @@ -148,6 +149,9 @@ def __init__( timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. headers: Additional HTTP headers to include in all API requests. + user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `, for + example to identify the tool built on top of the client. With `with_custom_http_client`, pass it to + the HTTP client's constructor. compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm, or an `HttpCompressor` instance for finer-grained control. """ @@ -212,6 +216,7 @@ def __init__( self._timeout_long = timeout_long self._timeout_max = timeout_max self._headers = headers + self._user_agent_suffix = user_agent_suffix self._http_compressor = resolve_compressor(compression) @classmethod @@ -283,6 +288,7 @@ def http_client(self) -> HttpClient: min_delay_between_retries=self._min_delay_between_retries, statistics=self._statistics, headers=self._headers, + user_agent_suffix=self._user_agent_suffix, http_compressor=self._http_compressor, ) @@ -490,6 +496,7 @@ def __init__( timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, headers: dict[str, str] | None = None, + user_agent_suffix: str | list[str] | None = None, compression: HttpCompressionAlgorithm | HttpCompressor = 'gzip', ) -> None: """Initialize the Apify API client. @@ -512,6 +519,9 @@ def __init__( timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. headers: Additional HTTP headers to include in all API requests. + user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `, for + example to identify the tool built on top of the client. With `with_custom_http_client`, pass it to + the HTTP client's constructor. compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm, or an `HttpCompressor` instance for finer-grained control. """ @@ -576,6 +586,7 @@ def __init__( self._timeout_long = timeout_long self._timeout_max = timeout_max self._headers = headers + self._user_agent_suffix = user_agent_suffix self._http_compressor = resolve_compressor(compression) @classmethod @@ -647,6 +658,7 @@ def http_client(self) -> HttpClientAsync: min_delay_between_retries=self._min_delay_between_retries, statistics=self._statistics, headers=self._headers, + user_agent_suffix=self._user_agent_suffix, http_compressor=self._http_compressor, ) return self._http_client diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index 5ffa6551..e58f2d05 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -126,6 +126,7 @@ def __init__( min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, statistics: ClientStatistics | None = None, headers: dict[str, str] | None = None, + user_agent_suffix: str | list[str] | None = None, http_compressor: HttpCompressor | None = None, ) -> None: """Initialize the HTTP client base. @@ -140,6 +141,7 @@ def __init__( min_delay_between_retries: Minimum delay between retries. statistics: Statistics tracker for API calls. Created automatically if not provided. headers: Additional HTTP headers to include in all requests. + user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `. http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. """ self._http_compressor = http_compressor if http_compressor is not None else GzipHttpCompressor() @@ -163,6 +165,9 @@ def __init__( client_version = metadata.version('apify-client') user_agent = f'ApifyClient/{client_version} ({sys.platform}; Python/{python_version}); isAtHome/{is_at_home}' + if user_agent_suffix: + suffix_parts = [user_agent_suffix] if isinstance(user_agent_suffix, str) else user_agent_suffix + user_agent += f'; {"; ".join(suffix_parts)}' default_headers['User-Agent'] = user_agent if token is not None: diff --git a/src/apify_client/http_clients/_httpx2.py b/src/apify_client/http_clients/_httpx2.py index 48eac9d9..8a235fa9 100644 --- a/src/apify_client/http_clients/_httpx2.py +++ b/src/apify_client/http_clients/_httpx2.py @@ -65,6 +65,7 @@ def __init__( min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, statistics: ClientStatistics | None = None, headers: dict[str, str] | None = None, + user_agent_suffix: str | list[str] | None = None, http_compressor: HttpCompressor | None = None, ) -> None: """Initialize the HTTPX2-based synchronous HTTP client. @@ -79,6 +80,7 @@ def __init__( min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). statistics: Statistics tracker for API calls. Created automatically if not provided. headers: Additional HTTP headers to include in all requests. + user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `. http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. """ super().__init__( @@ -91,6 +93,7 @@ def __init__( min_delay_between_retries=min_delay_between_retries, statistics=statistics, headers=headers, + user_agent_suffix=user_agent_suffix, http_compressor=http_compressor, ) @@ -169,6 +172,7 @@ def __init__( min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, statistics: ClientStatistics | None = None, headers: dict[str, str] | None = None, + user_agent_suffix: str | list[str] | None = None, http_compressor: HttpCompressor | None = None, ) -> None: """Initialize the HTTPX2-based asynchronous HTTP client. @@ -183,6 +187,7 @@ def __init__( min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). statistics: Statistics tracker for API calls. Created automatically if not provided. headers: Additional HTTP headers to include in all requests. + user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `. http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. """ super().__init__( @@ -195,6 +200,7 @@ def __init__( min_delay_between_retries=min_delay_between_retries, statistics=statistics, headers=headers, + user_agent_suffix=user_agent_suffix, http_compressor=http_compressor, ) diff --git a/src/apify_client/http_clients/_impit.py b/src/apify_client/http_clients/_impit.py index 5d01b2c2..fb192e30 100644 --- a/src/apify_client/http_clients/_impit.py +++ b/src/apify_client/http_clients/_impit.py @@ -61,6 +61,7 @@ def __init__( min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, statistics: ClientStatistics | None = None, headers: dict[str, str] | None = None, + user_agent_suffix: str | list[str] | None = None, http_compressor: HttpCompressor | None = None, ) -> None: """Initialize the Impit-based synchronous HTTP client. @@ -75,6 +76,7 @@ def __init__( min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). statistics: Statistics tracker for API calls. Created automatically if not provided. headers: Additional HTTP headers to include in all requests. + user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `. http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. """ super().__init__( @@ -87,6 +89,7 @@ def __init__( min_delay_between_retries=min_delay_between_retries, statistics=statistics, headers=headers, + user_agent_suffix=user_agent_suffix, http_compressor=http_compressor, ) @@ -161,6 +164,7 @@ def __init__( min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, statistics: ClientStatistics | None = None, headers: dict[str, str] | None = None, + user_agent_suffix: str | list[str] | None = None, http_compressor: HttpCompressor | None = None, ) -> None: """Initialize the Impit-based asynchronous HTTP client. @@ -175,6 +179,7 @@ def __init__( min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). statistics: Statistics tracker for API calls. Created automatically if not provided. headers: Additional HTTP headers to include in all requests. + user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `. http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. """ super().__init__( @@ -187,6 +192,7 @@ def __init__( min_delay_between_retries=min_delay_between_retries, statistics=statistics, headers=headers, + user_agent_suffix=user_agent_suffix, http_compressor=http_compressor, ) diff --git a/tests/unit/test_client_headers.py b/tests/unit/test_client_headers.py index 8706c31b..d4aeb49b 100644 --- a/tests/unit/test_client_headers.py +++ b/tests/unit/test_client_headers.py @@ -7,8 +7,10 @@ from typing import TYPE_CHECKING import httpx2 +import pytest from werkzeug import Request, Response +from apify_client import ApifyClient, ApifyClientAsync from apify_client.http_clients import Httpx2HttpClient, Httpx2HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync if TYPE_CHECKING: @@ -149,6 +151,81 @@ def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings +user_agent_suffix_cases = [ + pytest.param(None, '', id='no suffix'), + pytest.param('MyTool/1.0', '; MyTool/1.0', id='string suffix'), + pytest.param(['SDK/3.0.0', 'Crawlee/1.0.0'], '; SDK/3.0.0; Crawlee/1.0.0', id='list suffix'), +] + + +@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases) +async def test_user_agent_suffix_async( + httpserver: HTTPServer, + http_client_async_class: type[HttpClientAsync], + user_agent_suffix: str | list[str] | None, + expected_suffix: str, +) -> None: + """The HTTP client appends `user_agent_suffix` to the default `User-Agent` header.""" + client = http_client_async_class(user_agent_suffix=user_agent_suffix) + httpserver.expect_request('/').respond_with_handler(_header_handler) + + response = await client.call(method='GET', url=httpserver.url_for('/')) + + request_headers = json.loads(response.text)['received_headers'] + assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}' + + +@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases) +def test_user_agent_suffix_sync( + httpserver: HTTPServer, + http_client_class: type[HttpClient], + user_agent_suffix: str | list[str] | None, + expected_suffix: str, +) -> None: + """The HTTP client appends `user_agent_suffix` to the default `User-Agent` header.""" + client = http_client_class(user_agent_suffix=user_agent_suffix) + httpserver.expect_request('/').respond_with_handler(_header_handler) + + response = client.call(method='GET', url=httpserver.url_for('/')) + + request_headers = json.loads(response.text)['received_headers'] + assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}' + + +@pytest.mark.usefixtures('http_client_classes') +@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases) +async def test_apify_client_user_agent_suffix_async( + httpserver: HTTPServer, + user_agent_suffix: str | list[str] | None, + expected_suffix: str, +) -> None: + """`ApifyClientAsync` passes `user_agent_suffix` to its default HTTP client.""" + client = ApifyClientAsync(token='placeholder_token', user_agent_suffix=user_agent_suffix) + httpserver.expect_request('/').respond_with_handler(_header_handler) + + response = await client.http_client.call(method='GET', url=httpserver.url_for('/')) + + request_headers = json.loads(response.text)['received_headers'] + assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}' + + +@pytest.mark.usefixtures('http_client_classes') +@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases) +def test_apify_client_user_agent_suffix_sync( + httpserver: HTTPServer, + user_agent_suffix: str | list[str] | None, + expected_suffix: str, +) -> None: + """`ApifyClient` passes `user_agent_suffix` to its default HTTP client.""" + client = ApifyClient(token='placeholder_token', user_agent_suffix=user_agent_suffix) + httpserver.expect_request('/').respond_with_handler(_header_handler) + + response = client.http_client.call(method='GET', url=httpserver.url_for('/')) + + request_headers = json.loads(response.text)['received_headers'] + assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}' + + async def test_per_request_headers_override_defaults_async( httpserver: HTTPServer, http_client_async_class: type[HttpClientAsync] ) -> None: