Skip to content

feat: add a user_agent_suffix option #1145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vdusek merged 1 commit into from
Oct 16, 2026
Merged

feat: add a user_agent_suffix option #1145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/02_concepts/10_custom_http_clients.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ You can configure the default client through the <ApiLink to="class/ApifyClient"
</TabItem>
</Tabs>

To identify a tool built on top of the client, pass `user_agent_suffix`. The client appends it to its default `User-Agent` header after a semicolon and a space. Pass a list, such as `['MyTool/1.0', 'MyPlugin/2.0']`, to append several parts. The built-in HTTP clients accept the same option, so with `with_custom_http_client`, pass it to the HTTP client's constructor.

## Built-in HTTPX2 client

The package also provides <ApiLink to="class/Httpx2HttpClient">`Httpx2HttpClient`</ApiLink> and <ApiLink to="class/Httpx2HttpClientAsync">`Httpx2HttpClientAsync`</ApiLink>. They use the same request preparation, compression, retry policy, timeout tiers and growth, error handling, logging, and statistics as the default Impit clients, with [HTTPX2](https://github.com/pydantic/httpx2) as the transport.
Expand Down
12 changes: 12 additions & 0 deletions src/apify_client/_apify_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ def __init__(
timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
headers: dict[str, str] | None = None,
user_agent_suffix: str | list[str] | None = None,
compression: HttpCompressionAlgorithm | HttpCompressor = 'gzip',
) -> None:
"""Initialize the Apify API client.
Expand All @@ -148,6 +149,9 @@ def __init__(
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
headers: Additional HTTP headers to include in all API requests.
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `, for
example to identify the tool built on top of the client. With `with_custom_http_client`, pass it to
the HTTP client's constructor.
compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm,
or an `HttpCompressor` instance for finer-grained control.
"""
Expand Down Expand Up @@ -212,6 +216,7 @@ def __init__(
self._timeout_long = timeout_long
self._timeout_max = timeout_max
self._headers = headers
self._user_agent_suffix = user_agent_suffix
self._http_compressor = resolve_compressor(compression)

@classmethod
Expand Down Expand Up @@ -283,6 +288,7 @@ def http_client(self) -> HttpClient:
min_delay_between_retries=self._min_delay_between_retries,
statistics=self._statistics,
headers=self._headers,
user_agent_suffix=self._user_agent_suffix,
http_compressor=self._http_compressor,
)

Expand Down Expand Up @@ -490,6 +496,7 @@ def __init__(
timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
headers: dict[str, str] | None = None,
user_agent_suffix: str | list[str] | None = None,
compression: HttpCompressionAlgorithm | HttpCompressor = 'gzip',
) -> None:
"""Initialize the Apify API client.
Expand All @@ -512,6 +519,9 @@ def __init__(
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
headers: Additional HTTP headers to include in all API requests.
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `, for
example to identify the tool built on top of the client. With `with_custom_http_client`, pass it to
the HTTP client's constructor.
compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm,
or an `HttpCompressor` instance for finer-grained control.
"""
Expand Down Expand Up @@ -576,6 +586,7 @@ def __init__(
self._timeout_long = timeout_long
self._timeout_max = timeout_max
self._headers = headers
self._user_agent_suffix = user_agent_suffix
self._http_compressor = resolve_compressor(compression)

@classmethod
Expand Down Expand Up @@ -647,6 +658,7 @@ def http_client(self) -> HttpClientAsync:
min_delay_between_retries=self._min_delay_between_retries,
statistics=self._statistics,
headers=self._headers,
user_agent_suffix=self._user_agent_suffix,
http_compressor=self._http_compressor,
)
return self._http_client
Expand Down
5 changes: 5 additions & 0 deletions src/apify_client/http_clients/_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ def __init__(
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
statistics: ClientStatistics | None = None,
headers: dict[str, str] | None = None,
user_agent_suffix: str | list[str] | None = None,
http_compressor: HttpCompressor | None = None,
) -> None:
"""Initialize the HTTP client base.
Expand All @@ -140,6 +141,7 @@ def __init__(
min_delay_between_retries: Minimum delay between retries.
statistics: Statistics tracker for API calls. Created automatically if not provided.
headers: Additional HTTP headers to include in all requests.
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `.
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
"""
self._http_compressor = http_compressor if http_compressor is not None else GzipHttpCompressor()
Expand All @@ -163,6 +165,9 @@ def __init__(
client_version = metadata.version('apify-client')

user_agent = f'ApifyClient/{client_version} ({sys.platform}; Python/{python_version}); isAtHome/{is_at_home}'
if user_agent_suffix:
suffix_parts = [user_agent_suffix] if isinstance(user_agent_suffix, str) else user_agent_suffix
user_agent += f'; {"; ".join(suffix_parts)}'
default_headers['User-Agent'] = user_agent

if token is not None:
Expand Down
6 changes: 6 additions & 0 deletions src/apify_client/http_clients/_httpx2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ def __init__(
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
statistics: ClientStatistics | None = None,
headers: dict[str, str] | None = None,
user_agent_suffix: str | list[str] | None = None,
http_compressor: HttpCompressor | None = None,
) -> None:
"""Initialize the HTTPX2-based synchronous HTTP client.
Expand All @@ -79,6 +80,7 @@ def __init__(
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
statistics: Statistics tracker for API calls. Created automatically if not provided.
headers: Additional HTTP headers to include in all requests.
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `.
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
"""
super().__init__(
Expand All @@ -91,6 +93,7 @@ def __init__(
min_delay_between_retries=min_delay_between_retries,
statistics=statistics,
headers=headers,
user_agent_suffix=user_agent_suffix,
http_compressor=http_compressor,
)

Expand Down Expand Up @@ -169,6 +172,7 @@ def __init__(
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
statistics: ClientStatistics | None = None,
headers: dict[str, str] | None = None,
user_agent_suffix: str | list[str] | None = None,
http_compressor: HttpCompressor | None = None,
) -> None:
"""Initialize the HTTPX2-based asynchronous HTTP client.
Expand All @@ -183,6 +187,7 @@ def __init__(
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
statistics: Statistics tracker for API calls. Created automatically if not provided.
headers: Additional HTTP headers to include in all requests.
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `.
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
"""
super().__init__(
Expand All @@ -195,6 +200,7 @@ def __init__(
min_delay_between_retries=min_delay_between_retries,
statistics=statistics,
headers=headers,
user_agent_suffix=user_agent_suffix,
http_compressor=http_compressor,
)

Expand Down
6 changes: 6 additions & 0 deletions src/apify_client/http_clients/_impit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ def __init__(
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
statistics: ClientStatistics | None = None,
headers: dict[str, str] | None = None,
user_agent_suffix: str | list[str] | None = None,
http_compressor: HttpCompressor | None = None,
) -> None:
"""Initialize the Impit-based synchronous HTTP client.
Expand All @@ -75,6 +76,7 @@ def __init__(
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
statistics: Statistics tracker for API calls. Created automatically if not provided.
headers: Additional HTTP headers to include in all requests.
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `.
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
"""
super().__init__(
Expand All @@ -87,6 +89,7 @@ def __init__(
min_delay_between_retries=min_delay_between_retries,
statistics=statistics,
headers=headers,
user_agent_suffix=user_agent_suffix,
http_compressor=http_compressor,
)

Expand Down Expand Up @@ -161,6 +164,7 @@ def __init__(
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
statistics: ClientStatistics | None = None,
headers: dict[str, str] | None = None,
user_agent_suffix: str | list[str] | None = None,
http_compressor: HttpCompressor | None = None,
) -> None:
"""Initialize the Impit-based asynchronous HTTP client.
Expand All @@ -175,6 +179,7 @@ def __init__(
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
statistics: Statistics tracker for API calls. Created automatically if not provided.
headers: Additional HTTP headers to include in all requests.
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `.
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
"""
super().__init__(
Expand All @@ -187,6 +192,7 @@ def __init__(
min_delay_between_retries=min_delay_between_retries,
statistics=statistics,
headers=headers,
user_agent_suffix=user_agent_suffix,
http_compressor=http_compressor,
)

Expand Down
77 changes: 77 additions & 0 deletions tests/unit/test_client_headers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@
from typing import TYPE_CHECKING

import httpx2
import pytest
from werkzeug import Request, Response

from apify_client import ApifyClient, ApifyClientAsync
from apify_client.http_clients import Httpx2HttpClient, Httpx2HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync

if TYPE_CHECKING:
Expand Down Expand Up @@ -149,6 +151,81 @@ def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient
assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings


user_agent_suffix_cases = [
pytest.param(None, '', id='no suffix'),
pytest.param('MyTool/1.0', '; MyTool/1.0', id='string suffix'),
pytest.param(['SDK/3.0.0', 'Crawlee/1.0.0'], '; SDK/3.0.0; Crawlee/1.0.0', id='list suffix'),

@Pijukatel Pijukatel Oct 16, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this just a test, or a hint that we are going to set those suffixes in our tools built on top of the client?

@vdusek vdusek Oct 16, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We surely can.

In JS, SDK and the MCP server use it. The SDK sets SDK/<version> and Crawlee/<version>.

In Python, the LangChain and Haystack integrations do the same, but through client internals. LangChain uses client.http_client._headers and Haystack uses client.http_client.httpx_client. Strands sends its own custom x-apify-integration-platform header.

]


@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases)
async def test_user_agent_suffix_async(
httpserver: HTTPServer,
http_client_async_class: type[HttpClientAsync],
user_agent_suffix: str | list[str] | None,
expected_suffix: str,
) -> None:
"""The HTTP client appends `user_agent_suffix` to the default `User-Agent` header."""
client = http_client_async_class(user_agent_suffix=user_agent_suffix)
httpserver.expect_request('/').respond_with_handler(_header_handler)

response = await client.call(method='GET', url=httpserver.url_for('/'))

request_headers = json.loads(response.text)['received_headers']
assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}'


@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases)
def test_user_agent_suffix_sync(
httpserver: HTTPServer,
http_client_class: type[HttpClient],
user_agent_suffix: str | list[str] | None,
expected_suffix: str,
) -> None:
"""The HTTP client appends `user_agent_suffix` to the default `User-Agent` header."""
client = http_client_class(user_agent_suffix=user_agent_suffix)
httpserver.expect_request('/').respond_with_handler(_header_handler)

response = client.call(method='GET', url=httpserver.url_for('/'))

request_headers = json.loads(response.text)['received_headers']
assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}'


@pytest.mark.usefixtures('http_client_classes')
@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases)
async def test_apify_client_user_agent_suffix_async(
httpserver: HTTPServer,
user_agent_suffix: str | list[str] | None,
expected_suffix: str,
) -> None:
"""`ApifyClientAsync` passes `user_agent_suffix` to its default HTTP client."""
client = ApifyClientAsync(token='placeholder_token', user_agent_suffix=user_agent_suffix)
httpserver.expect_request('/').respond_with_handler(_header_handler)

response = await client.http_client.call(method='GET', url=httpserver.url_for('/'))

request_headers = json.loads(response.text)['received_headers']
assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}'


@pytest.mark.usefixtures('http_client_classes')
@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases)
def test_apify_client_user_agent_suffix_sync(
httpserver: HTTPServer,
user_agent_suffix: str | list[str] | None,
expected_suffix: str,
) -> None:
"""`ApifyClient` passes `user_agent_suffix` to its default HTTP client."""
client = ApifyClient(token='placeholder_token', user_agent_suffix=user_agent_suffix)
httpserver.expect_request('/').respond_with_handler(_header_handler)

response = client.http_client.call(method='GET', url=httpserver.url_for('/'))

request_headers = json.loads(response.text)['received_headers']
assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}'


async def test_per_request_headers_override_defaults_async(
httpserver: HTTPServer, http_client_async_class: type[HttpClientAsync]
) -> None:
Expand Down
Loading