Skip to content

feat: add chunk_size to collection iterate() methods #1146

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vdusek wants to merge 2 commits into
base: master
Choose a base branch
from
Open

feat: add chunk_size to collection iterate() methods #1146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e0bcb2a
feat: add `chunk_size` to collection `iterate()` methods
vdusek Oct 16, 2026
7d74a4f
test: cover iterating without `chunk_size`
vdusek Oct 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/02_concepts/08_pagination.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ to retrieve every item matching the filters. For dataset, key-value store and re
matching helpers are `iterate_items`, `iterate_keys` and `iterate_requests`. They handle pagination
automatically, so you don't need to manage offsets, limits or cursors yourself.

To control how many items each API call fetches, pass `chunk_size`. It caps the size of every page, and a page
never asks for more items than `limit` has left. Without `chunk_size`, collection clients let the API pick the
page size, while the storage helpers fetch 1,000 items per call.

The example below iterates over every Actor owned by the current user using a collection client's `iterate`
method:

Expand Down
8 changes: 6 additions & 2 deletions src/apify_client/_resource_clients/actor_collection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ def iterate(
offset: int | None = None,
desc: bool | None = None,
sort_by: Literal['createdAt', 'stats.lastRunStartedAt'] | None = 'createdAt',
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> Iterator[ActorListItem]:
"""Iterate over the Actors the user has created or used.
Expand All @@ -96,6 +97,7 @@ def iterate(
offset: What Actor to include as first when retrieving the list.
desc: Whether to sort the Actors in descending order based on their creation date.
sort_by: Field to sort the results by.
chunk_size: Maximum number of Actors requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -105,7 +107,7 @@ def iterate(
def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfActors:
return self.list(my=my, limit=limit, offset=offset, desc=desc, sort_by=sort_by, timeout=timeout)

return get_items_iterator(_callback, limit=limit, offset=offset)
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

def create(
self,
Expand Down Expand Up @@ -273,6 +275,7 @@ def iterate(
offset: int | None = None,
desc: bool | None = None,
sort_by: Literal['createdAt', 'stats.lastRunStartedAt'] | None = 'createdAt',
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> AsyncIterator[ActorListItem]:
"""Iterate over the Actors the user has created or used.
Expand All @@ -288,6 +291,7 @@ def iterate(
offset: What Actor to include as first when retrieving the list.
desc: Whether to sort the Actors in descending order based on their creation date.
sort_by: Field to sort the results by.
chunk_size: Maximum number of Actors requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -297,7 +301,7 @@ def iterate(
async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfActors:
return await self.list(my=my, limit=limit, offset=offset, desc=desc, sort_by=sort_by, timeout=timeout)

return get_items_iterator_async(_callback, limit=limit, offset=offset)
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

async def create(
self,
Expand Down
8 changes: 6 additions & 2 deletions src/apify_client/_resource_clients/build_collection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ def iterate(
limit: int | None = None,
offset: int | None = None,
desc: bool | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> Iterator[BuildListItem]:
"""Iterate over all Actor builds.
Expand All @@ -81,6 +82,7 @@ def iterate(
limit: How many builds to retrieve.
offset: What build to include as first when retrieving the list.
desc: Whether to sort the builds in descending order based on their start date.
chunk_size: Maximum number of builds requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -90,7 +92,7 @@ def iterate(
def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfBuilds:
return self.list(limit=limit, offset=offset, desc=desc, timeout=timeout)

return get_items_iterator(_callback, limit=limit, offset=offset)
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)


@docs_group('Resource clients')
Expand Down Expand Up @@ -146,6 +148,7 @@ def iterate(
limit: int | None = None,
offset: int | None = None,
desc: bool | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> AsyncIterator[BuildListItem]:
"""Iterate over all Actor builds.
Expand All @@ -160,6 +163,7 @@ def iterate(
limit: How many builds to retrieve.
offset: What build to include as first when retrieving the list.
desc: Whether to sort the builds in descending order based on their start date.
chunk_size: Maximum number of builds requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -169,4 +173,4 @@ def iterate(
async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfBuilds:
return await self.list(limit=limit, offset=offset, desc=desc, timeout=timeout)

return get_items_iterator_async(_callback, limit=limit, offset=offset)
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
8 changes: 6 additions & 2 deletions src/apify_client/_resource_clients/dataset_collection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ def iterate(
offset: int | None = None,
desc: bool | None = None,
ownership: StorageOwnership | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> Iterator[DatasetListItem]:
"""Iterate over the available datasets.
Expand All @@ -99,6 +100,7 @@ def iterate(
desc: Whether to sort the datasets in descending order based on their modification date.
ownership: Filter by ownership. 'ownedByMe' returns only user's own datasets,
'sharedWithMe' returns only datasets shared with the user.
chunk_size: Maximum number of datasets requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -110,7 +112,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfD
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
)

return get_items_iterator(_callback, limit=limit, offset=offset)
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

def get_or_create(
self,
Expand Down Expand Up @@ -198,6 +200,7 @@ def iterate(
offset: int | None = None,
desc: bool | None = None,
ownership: StorageOwnership | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> AsyncIterator[DatasetListItem]:
"""Iterate over the available datasets.
Expand All @@ -214,6 +217,7 @@ def iterate(
desc: Whether to sort the datasets in descending order based on their modification date.
ownership: Filter by ownership. 'ownedByMe' returns only user's own datasets,
'sharedWithMe' returns only datasets shared with the user.
chunk_size: Maximum number of datasets requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -225,7 +229,7 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
)

return get_items_iterator_async(_callback, limit=limit, offset=offset)
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

async def get_or_create(
self,
Expand Down
8 changes: 6 additions & 2 deletions src/apify_client/_resource_clients/key_value_store_collection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ def iterate(
offset: int | None = None,
desc: bool | None = None,
ownership: StorageOwnership | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> Iterator[KeyValueStoreResource]:
"""Iterate over the available key-value stores.
Expand All @@ -98,6 +99,7 @@ def iterate(
desc: Whether to sort the key-value stores in descending order based on their modification date.
ownership: Filter by ownership. 'ownedByMe' returns only user's own key-value stores,
'sharedWithMe' returns only key-value stores shared with the user.
chunk_size: Maximum number of key-value stores requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -109,7 +111,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfK
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
)

return get_items_iterator(_callback, limit=limit, offset=offset)
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

def get_or_create(
self,
Expand Down Expand Up @@ -197,6 +199,7 @@ def iterate(
offset: int | None = None,
desc: bool | None = None,
ownership: StorageOwnership | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> AsyncIterator[KeyValueStoreResource]:
"""Iterate over the available key-value stores.
Expand All @@ -213,6 +216,7 @@ def iterate(
desc: Whether to sort the key-value stores in descending order based on their modification date.
ownership: Filter by ownership. 'ownedByMe' returns only user's own key-value stores,
'sharedWithMe' returns only key-value stores shared with the user.
chunk_size: Maximum number of key-value stores requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -224,7 +228,7 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
)

return get_items_iterator_async(_callback, limit=limit, offset=offset)
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

async def get_or_create(
self,
Expand Down
8 changes: 6 additions & 2 deletions src/apify_client/_resource_clients/request_queue_collection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ def iterate(
offset: int | None = None,
desc: bool | None = None,
ownership: StorageOwnership | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> Iterator[RequestQueueListItem]:
"""Iterate over the available request queues.
Expand All @@ -99,6 +100,7 @@ def iterate(
desc: Whether to sort the request queues in descending order based on their modification date.
ownership: Filter by ownership. 'ownedByMe' returns only user's own request queues,
'sharedWithMe' returns only request queues shared with the user.
chunk_size: Maximum number of request queues requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -110,7 +112,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfR
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
)

return get_items_iterator(_callback, limit=limit, offset=offset)
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

def get_or_create(
self,
Expand Down Expand Up @@ -196,6 +198,7 @@ def iterate(
offset: int | None = None,
desc: bool | None = None,
ownership: StorageOwnership | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> AsyncIterator[RequestQueueListItem]:
"""Iterate over the available request queues.
Expand All @@ -212,6 +215,7 @@ def iterate(
desc: Whether to sort the request queues in descending order based on their modification date.
ownership: Filter by ownership. 'ownedByMe' returns only user's own request queues,
'sharedWithMe' returns only request queues shared with the user.
chunk_size: Maximum number of request queues requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -223,7 +227,7 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
)

return get_items_iterator_async(_callback, limit=limit, offset=offset)
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

async def get_or_create(
self,
Expand Down
8 changes: 6 additions & 2 deletions src/apify_client/_resource_clients/run_collection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ def iterate(
status: ActorJobStatus | list[ActorJobStatus] | None = None, # ty: ignore[invalid-type-form]
started_before: str | datetime | None = None,
started_after: str | datetime | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> Iterator[RunListItem]:
"""Iterate over all Actor runs.
Expand All @@ -105,6 +106,7 @@ def iterate(
status: Retrieve only runs with the provided statuses.
started_before: Only return runs started before this date (inclusive).
started_after: Only return runs started after this date (inclusive).
chunk_size: Maximum number of runs requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -122,7 +124,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfR
timeout=timeout,
)

return get_items_iterator(_callback, limit=limit, offset=offset)
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)


@docs_group('Resource clients')
Expand Down Expand Up @@ -197,6 +199,7 @@ def iterate(
status: ActorJobStatus | list[ActorJobStatus] | None = None, # ty: ignore[invalid-type-form]
started_before: str | datetime | None = None,
started_after: str | datetime | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> AsyncIterator[RunListItem]:
"""Iterate over all Actor runs.
Expand All @@ -214,6 +217,7 @@ def iterate(
status: Retrieve only runs with the provided statuses.
started_before: Only return runs started before this date (inclusive).
started_after: Only return runs started after this date (inclusive).
chunk_size: Maximum number of runs requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -231,4 +235,4 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L
timeout=timeout,
)

return get_items_iterator_async(_callback, limit=limit, offset=offset)
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
8 changes: 6 additions & 2 deletions src/apify_client/_resource_clients/schedule_collection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ def iterate(
limit: int | None = None,
offset: int | None = None,
desc: bool | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> Iterator[ScheduleListItem]:
"""Iterate over the available schedules.
Expand All @@ -82,6 +83,7 @@ def iterate(
limit: How many schedules to retrieve.
offset: What schedules to include as first when retrieving the list.
desc: Whether to sort the schedules in descending order based on their modification date.
chunk_size: Maximum number of schedules requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -91,7 +93,7 @@ def iterate(
def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfSchedules:
return self.list(limit=limit, offset=offset, desc=desc, timeout=timeout)

return get_items_iterator(_callback, limit=limit, offset=offset)
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

def create(
self,
Expand Down Expand Up @@ -192,6 +194,7 @@ def iterate(
limit: int | None = None,
offset: int | None = None,
desc: bool | None = None,
chunk_size: int | None = None,
timeout: Timeout = 'medium',
) -> AsyncIterator[ScheduleListItem]:
"""Iterate over the available schedules.
Expand All @@ -205,6 +208,7 @@ def iterate(
limit: How many schedules to retrieve.
offset: What schedules to include as first when retrieving the list.
desc: Whether to sort the schedules in descending order based on their modification date.
chunk_size: Maximum number of schedules requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.

Yields:
Expand All @@ -214,7 +218,7 @@ def iterate(
async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfSchedules:
return await self.list(limit=limit, offset=offset, desc=desc, timeout=timeout)

return get_items_iterator_async(_callback, limit=limit, offset=offset)
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

async def create(
self,
Expand Down
Loading
Loading