diff --git a/docs/02_concepts/08_pagination.mdx b/docs/02_concepts/08_pagination.mdx index 86951f66..99b343df 100644 --- a/docs/02_concepts/08_pagination.mdx +++ b/docs/02_concepts/08_pagination.mdx @@ -53,6 +53,10 @@ to retrieve every item matching the filters. For dataset, key-value store and re matching helpers are `iterate_items`, `iterate_keys` and `iterate_requests`. They handle pagination automatically, so you don't need to manage offsets, limits or cursors yourself. +To control how many items each API call fetches, pass `chunk_size`. It caps the size of every page, and a page +never asks for more items than `limit` has left. Without `chunk_size`, collection clients let the API pick the +page size, while the storage helpers fetch 1,000 items per call. + The example below iterates over every Actor owned by the current user using a collection client's `iterate` method: diff --git a/src/apify_client/_resource_clients/actor_collection.py b/src/apify_client/_resource_clients/actor_collection.py index d640dfa3..e7241e0c 100644 --- a/src/apify_client/_resource_clients/actor_collection.py +++ b/src/apify_client/_resource_clients/actor_collection.py @@ -81,6 +81,7 @@ def iterate( offset: int | None = None, desc: bool | None = None, sort_by: Literal['createdAt', 'stats.lastRunStartedAt'] | None = 'createdAt', + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> Iterator[ActorListItem]: """Iterate over the Actors the user has created or used. @@ -96,6 +97,7 @@ def iterate( offset: What Actor to include as first when retrieving the list. desc: Whether to sort the Actors in descending order based on their creation date. sort_by: Field to sort the results by. + chunk_size: Maximum number of Actors requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -105,7 +107,7 @@ def iterate( def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfActors: return self.list(my=my, limit=limit, offset=offset, desc=desc, sort_by=sort_by, timeout=timeout) - return get_items_iterator(_callback, limit=limit, offset=offset) + return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size) def create( self, @@ -273,6 +275,7 @@ def iterate( offset: int | None = None, desc: bool | None = None, sort_by: Literal['createdAt', 'stats.lastRunStartedAt'] | None = 'createdAt', + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> AsyncIterator[ActorListItem]: """Iterate over the Actors the user has created or used. @@ -288,6 +291,7 @@ def iterate( offset: What Actor to include as first when retrieving the list. desc: Whether to sort the Actors in descending order based on their creation date. sort_by: Field to sort the results by. + chunk_size: Maximum number of Actors requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -297,7 +301,7 @@ def iterate( async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfActors: return await self.list(my=my, limit=limit, offset=offset, desc=desc, sort_by=sort_by, timeout=timeout) - return get_items_iterator_async(_callback, limit=limit, offset=offset) + return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size) async def create( self, diff --git a/src/apify_client/_resource_clients/build_collection.py b/src/apify_client/_resource_clients/build_collection.py index 06d085cd..eea37af3 100644 --- a/src/apify_client/_resource_clients/build_collection.py +++ b/src/apify_client/_resource_clients/build_collection.py @@ -67,6 +67,7 @@ def iterate( limit: int | None = None, offset: int | None = None, desc: bool | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> Iterator[BuildListItem]: """Iterate over all Actor builds. @@ -81,6 +82,7 @@ def iterate( limit: How many builds to retrieve. offset: What build to include as first when retrieving the list. desc: Whether to sort the builds in descending order based on their start date. + chunk_size: Maximum number of builds requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -90,7 +92,7 @@ def iterate( def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfBuilds: return self.list(limit=limit, offset=offset, desc=desc, timeout=timeout) - return get_items_iterator(_callback, limit=limit, offset=offset) + return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size) @docs_group('Resource clients') @@ -146,6 +148,7 @@ def iterate( limit: int | None = None, offset: int | None = None, desc: bool | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> AsyncIterator[BuildListItem]: """Iterate over all Actor builds. @@ -160,6 +163,7 @@ def iterate( limit: How many builds to retrieve. offset: What build to include as first when retrieving the list. desc: Whether to sort the builds in descending order based on their start date. + chunk_size: Maximum number of builds requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -169,4 +173,4 @@ def iterate( async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfBuilds: return await self.list(limit=limit, offset=offset, desc=desc, timeout=timeout) - return get_items_iterator_async(_callback, limit=limit, offset=offset) + return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size) diff --git a/src/apify_client/_resource_clients/dataset_collection.py b/src/apify_client/_resource_clients/dataset_collection.py index 794b9ade..6533f811 100644 --- a/src/apify_client/_resource_clients/dataset_collection.py +++ b/src/apify_client/_resource_clients/dataset_collection.py @@ -83,6 +83,7 @@ def iterate( offset: int | None = None, desc: bool | None = None, ownership: StorageOwnership | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> Iterator[DatasetListItem]: """Iterate over the available datasets. @@ -99,6 +100,7 @@ def iterate( desc: Whether to sort the datasets in descending order based on their modification date. ownership: Filter by ownership. 'ownedByMe' returns only user's own datasets, 'sharedWithMe' returns only datasets shared with the user. + chunk_size: Maximum number of datasets requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -110,7 +112,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfD unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout ) - return get_items_iterator(_callback, limit=limit, offset=offset) + return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size) def get_or_create( self, @@ -198,6 +200,7 @@ def iterate( offset: int | None = None, desc: bool | None = None, ownership: StorageOwnership | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> AsyncIterator[DatasetListItem]: """Iterate over the available datasets. @@ -214,6 +217,7 @@ def iterate( desc: Whether to sort the datasets in descending order based on their modification date. ownership: Filter by ownership. 'ownedByMe' returns only user's own datasets, 'sharedWithMe' returns only datasets shared with the user. + chunk_size: Maximum number of datasets requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -225,7 +229,7 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout ) - return get_items_iterator_async(_callback, limit=limit, offset=offset) + return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size) async def get_or_create( self, diff --git a/src/apify_client/_resource_clients/key_value_store_collection.py b/src/apify_client/_resource_clients/key_value_store_collection.py index 752fd866..ef5a7b6c 100644 --- a/src/apify_client/_resource_clients/key_value_store_collection.py +++ b/src/apify_client/_resource_clients/key_value_store_collection.py @@ -82,6 +82,7 @@ def iterate( offset: int | None = None, desc: bool | None = None, ownership: StorageOwnership | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> Iterator[KeyValueStoreResource]: """Iterate over the available key-value stores. @@ -98,6 +99,7 @@ def iterate( desc: Whether to sort the key-value stores in descending order based on their modification date. ownership: Filter by ownership. 'ownedByMe' returns only user's own key-value stores, 'sharedWithMe' returns only key-value stores shared with the user. + chunk_size: Maximum number of key-value stores requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -109,7 +111,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfK unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout ) - return get_items_iterator(_callback, limit=limit, offset=offset) + return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size) def get_or_create( self, @@ -197,6 +199,7 @@ def iterate( offset: int | None = None, desc: bool | None = None, ownership: StorageOwnership | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> AsyncIterator[KeyValueStoreResource]: """Iterate over the available key-value stores. @@ -213,6 +216,7 @@ def iterate( desc: Whether to sort the key-value stores in descending order based on their modification date. ownership: Filter by ownership. 'ownedByMe' returns only user's own key-value stores, 'sharedWithMe' returns only key-value stores shared with the user. + chunk_size: Maximum number of key-value stores requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -224,7 +228,7 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout ) - return get_items_iterator_async(_callback, limit=limit, offset=offset) + return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size) async def get_or_create( self, diff --git a/src/apify_client/_resource_clients/request_queue_collection.py b/src/apify_client/_resource_clients/request_queue_collection.py index 678d26b1..3844c20f 100644 --- a/src/apify_client/_resource_clients/request_queue_collection.py +++ b/src/apify_client/_resource_clients/request_queue_collection.py @@ -83,6 +83,7 @@ def iterate( offset: int | None = None, desc: bool | None = None, ownership: StorageOwnership | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> Iterator[RequestQueueListItem]: """Iterate over the available request queues. @@ -99,6 +100,7 @@ def iterate( desc: Whether to sort the request queues in descending order based on their modification date. ownership: Filter by ownership. 'ownedByMe' returns only user's own request queues, 'sharedWithMe' returns only request queues shared with the user. + chunk_size: Maximum number of request queues requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -110,7 +112,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfR unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout ) - return get_items_iterator(_callback, limit=limit, offset=offset) + return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size) def get_or_create( self, @@ -196,6 +198,7 @@ def iterate( offset: int | None = None, desc: bool | None = None, ownership: StorageOwnership | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> AsyncIterator[RequestQueueListItem]: """Iterate over the available request queues. @@ -212,6 +215,7 @@ def iterate( desc: Whether to sort the request queues in descending order based on their modification date. ownership: Filter by ownership. 'ownedByMe' returns only user's own request queues, 'sharedWithMe' returns only request queues shared with the user. + chunk_size: Maximum number of request queues requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -223,7 +227,7 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout ) - return get_items_iterator_async(_callback, limit=limit, offset=offset) + return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size) async def get_or_create( self, diff --git a/src/apify_client/_resource_clients/run_collection.py b/src/apify_client/_resource_clients/run_collection.py index 03224011..527904f1 100644 --- a/src/apify_client/_resource_clients/run_collection.py +++ b/src/apify_client/_resource_clients/run_collection.py @@ -88,6 +88,7 @@ def iterate( status: ActorJobStatus | list[ActorJobStatus] | None = None, # ty: ignore[invalid-type-form] started_before: str | datetime | None = None, started_after: str | datetime | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> Iterator[RunListItem]: """Iterate over all Actor runs. @@ -105,6 +106,7 @@ def iterate( status: Retrieve only runs with the provided statuses. started_before: Only return runs started before this date (inclusive). started_after: Only return runs started after this date (inclusive). + chunk_size: Maximum number of runs requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -122,7 +124,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfR timeout=timeout, ) - return get_items_iterator(_callback, limit=limit, offset=offset) + return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size) @docs_group('Resource clients') @@ -197,6 +199,7 @@ def iterate( status: ActorJobStatus | list[ActorJobStatus] | None = None, # ty: ignore[invalid-type-form] started_before: str | datetime | None = None, started_after: str | datetime | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> AsyncIterator[RunListItem]: """Iterate over all Actor runs. @@ -214,6 +217,7 @@ def iterate( status: Retrieve only runs with the provided statuses. started_before: Only return runs started before this date (inclusive). started_after: Only return runs started after this date (inclusive). + chunk_size: Maximum number of runs requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -231,4 +235,4 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L timeout=timeout, ) - return get_items_iterator_async(_callback, limit=limit, offset=offset) + return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size) diff --git a/src/apify_client/_resource_clients/schedule_collection.py b/src/apify_client/_resource_clients/schedule_collection.py index 8ba347f0..7e7afbc8 100644 --- a/src/apify_client/_resource_clients/schedule_collection.py +++ b/src/apify_client/_resource_clients/schedule_collection.py @@ -69,6 +69,7 @@ def iterate( limit: int | None = None, offset: int | None = None, desc: bool | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> Iterator[ScheduleListItem]: """Iterate over the available schedules. @@ -82,6 +83,7 @@ def iterate( limit: How many schedules to retrieve. offset: What schedules to include as first when retrieving the list. desc: Whether to sort the schedules in descending order based on their modification date. + chunk_size: Maximum number of schedules requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -91,7 +93,7 @@ def iterate( def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfSchedules: return self.list(limit=limit, offset=offset, desc=desc, timeout=timeout) - return get_items_iterator(_callback, limit=limit, offset=offset) + return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size) def create( self, @@ -192,6 +194,7 @@ def iterate( limit: int | None = None, offset: int | None = None, desc: bool | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> AsyncIterator[ScheduleListItem]: """Iterate over the available schedules. @@ -205,6 +208,7 @@ def iterate( limit: How many schedules to retrieve. offset: What schedules to include as first when retrieving the list. desc: Whether to sort the schedules in descending order based on their modification date. + chunk_size: Maximum number of schedules requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -214,7 +218,7 @@ def iterate( async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfSchedules: return await self.list(limit=limit, offset=offset, desc=desc, timeout=timeout) - return get_items_iterator_async(_callback, limit=limit, offset=offset) + return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size) async def create( self, diff --git a/src/apify_client/_resource_clients/store_collection.py b/src/apify_client/_resource_clients/store_collection.py index c421251e..7e5470e6 100644 --- a/src/apify_client/_resource_clients/store_collection.py +++ b/src/apify_client/_resource_clients/store_collection.py @@ -85,6 +85,7 @@ def iterate( category: str | None = None, username: str | None = None, pricing_model: str | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> Iterator[StoreActor]: """Iterate over Actors in Apify store. @@ -103,6 +104,7 @@ def iterate( category: Filter by this category. username: Filter by this username. pricing_model: Filter by this pricing model. + chunk_size: Maximum number of Actors requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -121,7 +123,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfS timeout=timeout, ) - return get_items_iterator(_callback, limit=limit, offset=offset) + return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size) @docs_group('Resource clients') @@ -195,6 +197,7 @@ def iterate( category: str | None = None, username: str | None = None, pricing_model: str | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> AsyncIterator[StoreActor]: """Iterate over Actors in Apify store. @@ -213,6 +216,7 @@ def iterate( category: Filter by this category. username: Filter by this username. pricing_model: Filter by this pricing model. + chunk_size: Maximum number of Actors requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -231,4 +235,4 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L timeout=timeout, ) - return get_items_iterator_async(_callback, limit=limit, offset=offset) + return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size) diff --git a/src/apify_client/_resource_clients/task_collection.py b/src/apify_client/_resource_clients/task_collection.py index 6db4e222..24d73750 100644 --- a/src/apify_client/_resource_clients/task_collection.py +++ b/src/apify_client/_resource_clients/task_collection.py @@ -76,6 +76,7 @@ def iterate( limit: int | None = None, offset: int | None = None, desc: bool | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> Iterator[TaskListItem]: """Iterate over the available tasks. @@ -89,6 +90,7 @@ def iterate( limit: How many tasks to list. offset: What task to include as first when retrieving the list. desc: Whether to sort the tasks in descending order based on their creation date. + chunk_size: Maximum number of tasks requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -98,7 +100,7 @@ def iterate( def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfTasks: return self.list(limit=limit, offset=offset, desc=desc, timeout=timeout) - return get_items_iterator(_callback, limit=limit, offset=offset) + return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size) def create( self, @@ -249,6 +251,7 @@ def iterate( limit: int | None = None, offset: int | None = None, desc: bool | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> AsyncIterator[TaskListItem]: """Iterate over the available tasks. @@ -262,6 +265,7 @@ def iterate( limit: How many tasks to list. offset: What task to include as first when retrieving the list. desc: Whether to sort the tasks in descending order based on their creation date. + chunk_size: Maximum number of tasks requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -271,7 +275,7 @@ def iterate( async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfTasks: return await self.list(limit=limit, offset=offset, desc=desc, timeout=timeout) - return get_items_iterator_async(_callback, limit=limit, offset=offset) + return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size) async def create( self, diff --git a/src/apify_client/_resource_clients/webhook_collection.py b/src/apify_client/_resource_clients/webhook_collection.py index 1693ee9f..7cf06933 100644 --- a/src/apify_client/_resource_clients/webhook_collection.py +++ b/src/apify_client/_resource_clients/webhook_collection.py @@ -70,6 +70,7 @@ def iterate( limit: int | None = None, offset: int | None = None, desc: bool | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> Iterator[WebhookListItem]: """Iterate over the available webhooks. @@ -83,6 +84,7 @@ def iterate( limit: How many webhooks to retrieve. offset: What webhook to include as first when retrieving the list. desc: Whether to sort the webhooks in descending order based on their date of creation. + chunk_size: Maximum number of webhooks requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -92,7 +94,7 @@ def iterate( def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfWebhooks: return self.list(limit=limit, offset=offset, desc=desc, timeout=timeout) - return get_items_iterator(_callback, limit=limit, offset=offset) + return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size) def create( self, @@ -203,6 +205,7 @@ def iterate( limit: int | None = None, offset: int | None = None, desc: bool | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> AsyncIterator[WebhookListItem]: """Iterate over the available webhooks. @@ -216,6 +219,7 @@ def iterate( limit: How many webhooks to retrieve. offset: What webhook to include as first when retrieving the list. desc: Whether to sort the webhooks in descending order based on their date of creation. + chunk_size: Maximum number of webhooks requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -225,7 +229,7 @@ def iterate( async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfWebhooks: return await self.list(limit=limit, offset=offset, desc=desc, timeout=timeout) - return get_items_iterator_async(_callback, limit=limit, offset=offset) + return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size) async def create( self, diff --git a/src/apify_client/_resource_clients/webhook_dispatch_collection.py b/src/apify_client/_resource_clients/webhook_dispatch_collection.py index 805ea5de..385554b0 100644 --- a/src/apify_client/_resource_clients/webhook_dispatch_collection.py +++ b/src/apify_client/_resource_clients/webhook_dispatch_collection.py @@ -63,6 +63,7 @@ def iterate( limit: int | None = None, offset: int | None = None, desc: bool | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> Iterator[WebhookDispatch]: """Iterate over all webhook dispatches of a user. @@ -76,6 +77,7 @@ def iterate( limit: How many webhook dispatches to retrieve. offset: What webhook dispatch to include as first when retrieving the list. desc: Whether to sort the webhook dispatches in descending order based on the date of their creation. + chunk_size: Maximum number of webhook dispatches requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -85,7 +87,7 @@ def iterate( def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfWebhookDispatches: return self.list(limit=limit, offset=offset, desc=desc, timeout=timeout) - return get_items_iterator(_callback, limit=limit, offset=offset) + return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size) @docs_group('Resource clients') @@ -137,6 +139,7 @@ def iterate( limit: int | None = None, offset: int | None = None, desc: bool | None = None, + chunk_size: int | None = None, timeout: Timeout = 'medium', ) -> AsyncIterator[WebhookDispatch]: """Iterate over all webhook dispatches of a user. @@ -150,6 +153,7 @@ def iterate( limit: How many webhook dispatches to retrieve. offset: What webhook dispatch to include as first when retrieving the list. desc: Whether to sort the webhook dispatches in descending order based on the date of their creation. + chunk_size: Maximum number of webhook dispatches requested per API call when iterating across pages. timeout: Timeout for the API HTTP request. Yields: @@ -159,4 +163,4 @@ def iterate( async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfWebhookDispatches: return await self.list(limit=limit, offset=offset, desc=desc, timeout=timeout) - return get_items_iterator_async(_callback, limit=limit, offset=offset) + return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size) diff --git a/tests/unit/test_client_pagination.py b/tests/unit/test_client_pagination.py index 356649b4..8236e06c 100644 --- a/tests/unit/test_client_pagination.py +++ b/tests/unit/test_client_pagination.py @@ -445,7 +445,13 @@ def __hash__(self) -> int: 'chunk_size', {'chunk_size': 100, 'limit': 250}, create_items(0, 250), - STORAGE_CLIENTS, + OPTIONS_CLIENTS, + ), + _PaginationCase( + 'Offset, limit, chunk_size', + {'offset': 1000, 'limit': 250, 'chunk_size': 100}, + create_items(1000, 1250), + COLLECTION_CLIENTS | DATASET_CLIENTS, ), _PaginationCase( 'Offset, limit, descending, chunk_size', @@ -665,6 +671,62 @@ async def test_rq_list_requests_iterable_async( assert returned_items == expected_items +COLLECTION_CHUNK_SIZE_PARAMS = [ + pytest.param( + client_name, + inputs, + expected_pages, + id=f'{client_name}:{case_id}', + ) + for case_id, inputs, expected_pages in ( + ('no chunk_size', {}, [(0, 0), (1000, 0), (2000, 0), (2500, 0)]), + ( + 'chunk_size', + {'chunk_size': 400}, + [(0, 400), (400, 400), (800, 400), (1200, 400), (1600, 400), (2000, 400), (2400, 400), (2500, 400)], + ), + ('chunk_size and limit', {'chunk_size': 400, 'limit': 1000}, [(0, 400), (400, 400), (800, 200)]), + ('chunk_size, limit and offset', {'chunk_size': 400, 'limit': 500, 'offset': 300}, [(300, 400), (700, 100)]), + ('chunk_size above limit', {'chunk_size': 400, 'limit': 150, 'offset': 10}, [(10, 150)]), + ) + for client_name in sorted(COLLECTION_CLIENTS) +] + + +def requested_pages(httpserver: HTTPServer) -> list[tuple[int, int]]: + """Return the `(offset, limit)` query params of every request the server received.""" + return [ + (_parse_int_param(request.args.get('offset')), _parse_int_param(request.args.get('limit'))) + for request, _ in httpserver.log + ] + + +@pytest.mark.parametrize(('client_name', 'inputs', 'expected_pages'), COLLECTION_CHUNK_SIZE_PARAMS) +def test_collection_iterate_chunk_size_caps_page_limit( + pagination_server: HTTPServer, + client_name: str, + inputs: dict, + expected_pages: list[tuple[int, int]], +) -> None: + """A sync collection `iterate()` sends `chunk_size` as each page's `limit`, clamped to the rows still left.""" + client: CollectionClient = _CLIENT_FACTORIES[client_name](_make_sync_client(pagination_server)) + list(client.iterate(**inputs)) + assert requested_pages(pagination_server) == expected_pages + + +@pytest.mark.parametrize(('client_name', 'inputs', 'expected_pages'), COLLECTION_CHUNK_SIZE_PARAMS) +async def test_collection_iterate_chunk_size_caps_page_limit_async( + pagination_server: HTTPServer, + client_name: str, + inputs: dict, + expected_pages: list[tuple[int, int]], +) -> None: + """An async collection `iterate()` sends `chunk_size` as each page's `limit`, clamped to the rows still left.""" + client: CollectionClientAsync = _CLIENT_FACTORIES[client_name](_make_async_client(pagination_server)) + _ = [item async for item in client.iterate(**inputs)] + assert requested_pages(pagination_server) == expected_pages + + class FakeOffsetPage: """Offset-paginated page whose `count` (rows scanned) and `len(items)` can differ in either direction."""