diff --git a/CHANGELOG.md b/CHANGELOG.md index 76c6f81..c5319b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,73 @@ # Changelog +## 0.3.3 + +- Added `failOnEmptyTestSuite="true"` to `phpunit.xml.dist` so a suite matching zero tests fails + instead of passing green. +- Replaced magic literals with named constants: `HttpClientCore::attemptTimeout()` now scales the + per-attempt timeout by a dedicated `TIMEOUT_BACKOFF_FACTOR` constant, and `Json::decode()` uses a + named `MAX_JSON_DEPTH` constant. +- Corrected the `RunClient::get()` and `BuildClient::get()` doc comments to explain that the + `waitForFinishSecs` value is clamped client-side to the request-timeout budget and additionally + capped at 60s by the server. +- Reworded the docs namespace table so the `Options` row no longer implies its example list is + exhaustive. + +## 0.3.2 + +- Fixed `RunClient::metamorph()` to normalize a slash-form `targetActorId` (e.g. `username/actor-name`) + to the URL-safe `username~actor-name` form before sending it, matching the reference JS client. +- Documented the expected `YYYY-MM-DD` date format for `me()->monthlyUsage()` in the docs. +- Expanded the docs namespace table with the commonly-used option classes so their `use` namespace + is discoverable. + +## 0.3.1 + +- Fixed `KeyValueStoreClient::iterateKeys()` so a `limit` of `0` (like `null`) iterates the whole + store instead of stopping after a single page; a positive `limit` still caps the total keys + yielded across all pages. +- Documented the `bool $forefront` parameter on the request-queue `addRequest`/`updateRequest`/ + `prolongRequestLock`/`deleteRequestLock` methods and the `?bool $gracefully` parameter on + `run()->abort()`, and added behavior descriptions for `recordExists`, `setRecordJson`, + `deleteRecord`, `getRecordPublicUrl`, `createKeysPublicUrl`, and `createItemsPublicUrl`. +- Corrected the README error-handling description so the "4xx are thrown" rule notes its exception: + a 404 on a single-resource fetch returns `null` from `get()` and is a no-op for `delete()`. +- Added the optional `baseUrl` argument to the README configuration snippet and documented that + `paginateRequests()` yields `RequestQueueRequest` instances. +- Fixed `RequestQueueClient::paginateRequests()` so a `limit` of `0` (like `null`) iterates all + requests instead of yielding a single page, matching `iterateKeys` and the offset paginator. +- Documented that combining item-dropping dataset filters (`skipEmpty`, and `clean` which implies + it) with multi-page `iterateItems()` can repeat or skip items, mirroring the reference JS client's + offset advancement. + +## 0.3.0 + +- Added lazy iteration helpers matching the reference client, which iterates every collection: an + `iterate()` generator on the Actor, Actor-version, Actor-env-var, build, run, dataset, + key-value-store, request-queue, schedule, task, webhook (account-wide and nested) and + webhook-dispatch collections; `DatasetClient::iterateItems()` for dataset items; and + `KeyValueStoreClient::iterateKeys()` for store keys (cursor-based). Each fetches pages on demand. +- Iteration `limit` semantics: for the offset/limit iterators, the options' `limit` now caps the + total number of items yielded across all pages (unset = all) and the per-page size is a separate + `$chunkSize` argument. `StoreCollectionClient::iterate()` follows the same rule (previously its + `limit` was used as the page size); `StoreListOptions::withOffset()` is replaced by + `withPagination()`. +- `KeyValueStoreClient::iterateKeys()` follows the store's cursor pagination + (`exclusiveStartKey`/`nextExclusiveStartKey`) and stops on the total-item cap or an untruncated page. +- Documented every new iteration method with runnable examples and clarified the request-queue + method list, the key-value-store record snippet, and when `TransportException` surfaces versus + `ApifyApiException`. + +## 0.2.2 + +- `batchAddRequests` now validates every request's individual payload size up front, before any + HTTP call, so an oversized request anywhere in a large batch is rejected without POSTing earlier + chunks (previously later chunks could partially mutate the queue before the error was raised). + +## 0.2.1 + +- Synced to Apify OpenAPI spec `v2-2026-07-10T105921Z`. No public interface changes. + ## 0.2.0 - Synced to Apify OpenAPI spec `v2-2026-07-08T143931Z`. No public interface changes. diff --git a/README.md b/README.md index eb8fdce..a0decb5 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,9 @@ $client = new ApifyClient('my-api-token'); // pass a value (e.g. 120) to bound the wait, or null to wait indefinitely (as here). $run = $client->actor('apify/hello-world')->call(null, null, null); -// Read items from the run's default dataset. -$items = $client->dataset($run->getDefaultDatasetId())->listItems(); +// Read items from the run's default dataset. getDefaultDatasetId() is ?string, so cast it +// to satisfy dataset(string $id). +$items = $client->dataset((string) $run->getDefaultDatasetId())->listItems(); echo 'Item count: ' . $items->getCount() . PHP_EOL; ``` @@ -50,6 +51,7 @@ The constructor accepts named arguments for non-default settings: ```php $configured = new ApifyClient( token: 'my-api-token', + baseUrl: 'https://api.apify.com', maxRetries: 5, minDelayBetweenRetriesMillis: 1000, timeoutSecs: 120, @@ -70,7 +72,10 @@ $configured = new ApifyClient( | `httpClient` | Guzzle | The replaceable transport (`Apify\Client\Http\HttpClientInterface`). | Requests are retried on network errors, HTTP 429 (rate limit) and 5xx responses, with exponential -backoff and jitter. 4xx responses (other than 429) are thrown immediately as `ApifyApiException`. +backoff and jitter. Other 4xx responses are thrown immediately as `ApifyApiException`, with one +exception: a resource-not-found 404 (the API's `record-not-found` / `record-or-token-not-found` +error type) on a single-resource fetch is not thrown — `get()` returns `null` and `delete()` is +treated as a successful no-op (see [Error handling](#error-handling)). ### Replaceable HTTP transport @@ -115,7 +120,11 @@ try { | `getData(): ?array` | Additional structured error data provided by the API, if any. | Transport-level failures (network errors, timeouts) are retried internally; only if every retry is -exhausted does the underlying error surface. Requests are retried on network errors, HTTP 429 and 5xx. +exhausted does the underlying error surface, as an `Apify\Client\Exception\TransportException` +(a `RuntimeException`; `isTimeout()` reports whether a request timed out). In short: +`ApifyApiException` means the server returned an error response (a 4xx/5xx with a body), whereas +`TransportException` means the request never produced a usable response (network failure or timeout) +after all retries. Requests are retried on network errors, HTTP 429 and 5xx. ## Versioning diff --git a/docs/README.md b/docs/README.md index f480b7e..f63782c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -23,7 +23,7 @@ Every class is under the `Apify\Client\` PSR-4 root. Use these when writing `use |---|---|---| | `Apify\Client\` | The entry point and version constants. | `ApifyClient`, `Version` | | `Apify\Client\Model\` | Response models returned by the API. | `RequestQueueRequest`, `ActorEnvVar`, `Dataset`, `ActorRun`, `PaginationList` | -| `Apify\Client\Options\` | Option objects (the `*Options` classes) **and** enums. | `ActorListOptions`, `DatasetListItemsOptions`, `PaginateRequestsOptions`, `RequestQueueClientOptions`, `DownloadItemsFormat` | +| `Apify\Client\Options\` | Option objects (all the `*Options` classes) **and** enums. | e.g. `ActorListOptions`, `ActorStartOptions`, `TaskStartOptions`, `RunListOptions`, `RunResurrectOptions`, `StorageListOptions`, `StoreListOptions`, `DatasetListItemsOptions`, `ListKeysOptions`, `GetRecordOptions`, `ListRequestsOptions`, `BatchAddRequestsOptions`, `PaginateRequestsOptions`, `LogOptions`, `DownloadItemsFormat` — see [options reference](options.md) for the full list | | `Apify\Client\Http\` | The replaceable transport and its adapters. | `HttpClientInterface`, `GuzzleHttpClient`, `Psr18HttpClient` | | `Apify\Client\Exception\` | Exceptions thrown by the client. | `ApifyApiException`, `TransportException` | @@ -42,6 +42,44 @@ PSR-7 `Psr\Http\Message\StreamInterface` (from the `psr/http-message` package), Methods that fetch a single resource return `null` when the resource does not exist, rather than throwing. API failures are thrown as `ApifyApiException` (see [error handling](../README.md#error-handling)). +## ApifyClient methods + +`ApifyClient` is the entry point: construct one, then call an accessor to get a sub-client for a +specific resource or collection. Single-resource accessors take an ID (or, where the API allows it, +a name) and return that resource's client; collection accessors take no arguments and return a +collection client for listing and creating. Method detail lives on the linked [resource +pages](#resource-pages); the signatures below are the entry points. + +| Method | Returns | Notes | +|---|---|---| +| `actor(string $id): ActorClient` | Actor client | Single Actor, by ID or `username/name`. | +| `actors(): ActorCollectionClient` | Actor collection | List and create Actors. | +| `build(string $id): BuildClient` | Build client | Single Actor build. | +| `builds(): BuildCollectionClient` | Build collection | List builds across Actors. | +| `run(string $id): RunClient` | Run client | Single Actor run. | +| `runs(): RunCollectionClient` | Run collection | List runs across Actors. | +| `dataset(string $id): DatasetClient` | Dataset client | Single dataset, by ID or name. | +| `datasets(): DatasetCollectionClient` | Dataset collection | List and create datasets. | +| `keyValueStore(string $id): KeyValueStoreClient` | Key-value store client | Single store, by ID or name. | +| `keyValueStores(): KeyValueStoreCollectionClient` | Key-value store collection | List and create stores. | +| `requestQueue(string $id, ?RequestQueueClientOptions $options = null): RequestQueueClient` | Request queue client | Single queue, by ID or name; optional client options (`clientKey`, per-request `timeoutSecs`). | +| `requestQueues(): RequestQueueCollectionClient` | Request queue collection | List and create queues. | +| `task(string $id): TaskClient` | Task client | Single task. | +| `tasks(): TaskCollectionClient` | Task collection | List and create tasks. | +| `schedule(string $id): ScheduleClient` | Schedule client | Single schedule. | +| `schedules(): ScheduleCollectionClient` | Schedule collection | List and create schedules. | +| `webhook(string $id): WebhookClient` | Webhook client | Single webhook. | +| `webhooks(): WebhookCollectionClient` | Webhook collection | List and create webhooks. | +| `webhookDispatch(string $id): WebhookDispatchClient` | Webhook dispatch client | Single webhook dispatch. | +| `webhookDispatches(): WebhookDispatchCollectionClient` | Webhook dispatch collection | List webhook dispatches. | +| `store(): StoreCollectionClient` | Store collection | Browse the public Apify Store. | +| `log(string $buildOrRunId): LogClient` | Log client | Log for a build or run, by ID. | +| `me(): UserClient` | User client | The authenticated user (`users/me`). | +| `user(string $id): UserClient` | User client | A public user profile, by ID. | +| `setStatusMessage(string $message, bool $isTerminal = false): ActorRun` | Updated run | Set the current run's status message; see [Setting single-resource status](#setting-single-resource-status). | +| `getUserAgent(): string` | User-Agent string | The `User-Agent` the client sends. | +| `getApiBaseUrl(): string` | Base URL | The resolved API base URL (with `/v2`). | + ## Models and unmodeled data (`toArray`) Response models expose the commonly-used fields as typed getters (e.g. `$actor->getId()`). The @@ -58,12 +96,14 @@ $actions = $schedule?->toArray()['actions'] ?? null; ## Raw JSON values A few methods return data whose shape is not modelled and is instead returned as a decoded -associative array (or accept an arbitrary value serialized to JSON): +JSON value — typically an associative array, though `getInput()` is typed `mixed` and returns +whatever JSON value was stored (or accept an arbitrary value serialized to JSON): - Read: `me()->monthlyUsage(...)`, `me()->limits()`, `task($id)->getInput()`, `build($id)->getOpenApiDefinition()`, `dataset($id)->getStatistics()`, and the raw request-queue - operations (`listRequests`, `listAndLockHead`, `prolongRequestLock`, `unlockRequests`, - `batchDeleteRequests`). + operations that return a response body (`listRequests`, `listAndLockHead`, `prolongRequestLock`, + `unlockRequests`, `batchDeleteRequests`). Note that `deleteRequestLock` returns `void` (it releases + a lock and has no meaningful body), so it is not in this list. - Write: definition/`update`/`create` arguments accept any JSON-serializable value — typically an associative array. diff --git a/docs/actors.md b/docs/actors.md index 9699591..33136ea 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -4,12 +4,17 @@ Snippets assume `$client = new ApifyClient('my-api-token');` and imported types. ## Actor collection — `$client->actors()` -- `list(?ActorListOptions $options): PaginationList` — list the account's Actors. +- `list(?ActorListOptions $options = null): PaginationList` — list the account's Actors. +- `iterate(?ActorListOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all matching Actors, paging on demand. The options' `limit` caps the total number yielded across all pages (unset = all); `$chunkSize` is the per-page size. - `create(mixed $actor): Actor` — create a new Actor from a JSON-serializable definition. ```php $page = $client->actors()->list(new ActorListOptions(my: true, limit: 10)); +foreach ($client->actors()->iterate(new ActorListOptions(my: true), 100) as $actor) { + echo $actor->getName() . PHP_EOL; +} + $actor = $client->actors()->create([ 'name' => 'my-actor', 'isPublic' => false, @@ -47,7 +52,7 @@ $lastSucceeded = $client->actor('apify/hello-world')->lastRun(new LastRunOptions ## Actor versions — `$client->actor($id)->versions()` / `->version($n)` -- Collection: `list(?ListOptions): PaginationList`, `create(mixed $version): ActorVersion`. +- Collection: `list(?ListOptions): PaginationList`, `iterate(?ListOptions $options = null, ?int $chunkSize = null): iterable`, `create(mixed $version): ActorVersion`. - Single: `get(): ?ActorVersion`, `update(mixed $newFields): ActorVersion`, `delete(): void`. ```php @@ -60,9 +65,12 @@ $version = $client->actor('me~my-actor')->versions()->create([ ## Environment variables — `->version($n)->envVars()` / `->envVar($name)` -- Collection: `list(): PaginationList`, `create(ActorEnvVar $envVar): ActorEnvVar`. +- Collection: `list(): PaginationList`, `iterate(?int $chunkSize = null): iterable`, `create(ActorEnvVar $envVar): ActorEnvVar`. - Single: `get(): ?ActorEnvVar`, `update(ActorEnvVar $envVar): ActorEnvVar`, `delete(): void`. +`iterate()` on the environment-variable collection takes only the optional `$chunkSize` (per-page +size); the endpoint has no filters, mirroring the reference client's parameterless iterator. + ```php $client->actor('me~my-actor')->version('0.0')->envVars()->create(new ActorEnvVar('API_KEY', 'secret', isSecret: true)); ``` diff --git a/docs/builds.md b/docs/builds.md index 2c09c04..4f250b2 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -4,10 +4,15 @@ Snippets assume `$client = new ApifyClient('my-api-token');` and imported types. ## Build collection — `$client->builds()` -- `list(?ListOptions $options): PaginationList` — list the account's builds. +- `list(?ListOptions $options = null): PaginationList` — list the account's builds. +- `iterate(?ListOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all builds, paging on demand. The options' `limit` caps the total number yielded across all pages (unset = all); `$chunkSize` is the per-page size. ```php $page = $client->builds()->list(new ListOptions(limit: 20, desc: true)); + +foreach ($client->builds()->iterate(new ListOptions(desc: true), 50) as $build) { + echo $build->getId() . PHP_EOL; +} ``` An Actor's builds are available at `$client->actor($id)->builds()`. diff --git a/docs/examples.md b/docs/examples.md index c8c6f0d..d984364 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -3,9 +3,11 @@ Each snippet below assumes a configured `$client` and that the types it uses are imported with the appropriate `use` statements (see [Namespaces](README.md#namespaces)); the first [complete program](#a-complete-standalone-program) shows the full scaffolding the shorter snippets -omit for brevity. The same programs live under [`tests/Examples/`](../tests/Examples) and are executed -end-to-end against the live API by the `Test examples` CI step (see `ExamplesTest`), so they are -guaranteed to stay runnable. +omit for brevity. The complete programs on this page live under +[`tests/Examples/`](../tests/Examples) and are executed end-to-end against the live API by the +`Test examples` CI step (see `ExamplesTest`), so those programs are guaranteed to stay runnable. +Inline snippets on the other documentation pages are not executed: they are only syntax-checked with +`php -l` by `DocSnippetsTest`, which catches parse errors but does not resolve classes or check types. ## A complete, standalone program @@ -30,8 +32,9 @@ try { // Run a public store Actor and wait up to 120s for it to finish. $run = $client->actor('apify/hello-world')->call(null, null, 120); - // Read the items the run produced into its default dataset. - $items = $client->dataset($run->getDefaultDatasetId())->listItems(); + // Read the items the run produced into its default dataset. getDefaultDatasetId() is + // ?string, so cast it to satisfy dataset(string $id). + $items = $client->dataset((string) $run->getDefaultDatasetId())->listItems(); echo 'Item count: ' . $items->getCount() . PHP_EOL; } catch (ApifyApiException $e) { echo 'API error ' . $e->getStatusCode() . ': ' . $e->getApiMessage() . PHP_EOL; @@ -42,7 +45,8 @@ try { ```php $run = $client->actor('apify/hello-world')->call(null, null, 120); -$items = $client->dataset($run->getDefaultDatasetId())->listItems(); +// getDefaultDatasetId() is ?string, so cast it to satisfy dataset(string $id). +$items = $client->dataset((string) $run->getDefaultDatasetId())->listItems(); echo 'Item count: ' . $items->getCount() . PHP_EOL; ``` @@ -120,7 +124,8 @@ if ($last !== null) { ```php $shown = 0; -foreach ($client->store()->iterate(new StoreListOptions(limit: 10)) as $item) { +// The second argument is the per-page (chunk) size; StoreListOptions::limit would cap the total. +foreach ($client->store()->iterate(new StoreListOptions(), 10) as $item) { echo $item->getName() . PHP_EOL; if (++$shown >= 5) { break; diff --git a/docs/misc.md b/docs/misc.md index e3b45d6..1e86136 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -4,14 +4,15 @@ Snippets assume `$client = new ApifyClient('my-api-token');` and imported types. ## Apify Store — `$client->store()` -- `list(?StoreListOptions $options): PaginationList` — one page of Store Actors. -- `iterate(?StoreListOptions $options): iterable` — lazily iterate all matching Actors, paging on demand. +- `list(?StoreListOptions $options = null): PaginationList` — one page of Store Actors. +- `iterate(?StoreListOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all matching Actors, paging on demand. The options' `limit` caps the total number yielded across all pages (unset = all); `$chunkSize` is the per-page size. ```php $page = $client->store()->list(new StoreListOptions(search: 'scraper', limit: 10)); $shown = 0; -foreach ($client->store()->iterate(new StoreListOptions(limit: 50)) as $item) { +// $chunkSize (50) is the per-page size; limit (unset) would cap the total across all pages. +foreach ($client->store()->iterate(new StoreListOptions(search: 'scraper'), 50) as $item) { echo $item->getName() . PHP_EOL; if (++$shown >= 5) { break; @@ -22,7 +23,7 @@ foreach ($client->store()->iterate(new StoreListOptions(limit: 50)) as $item) { ## Users — `$client->me()` / `$client->user($id)` - `get(): ?User` — for `me()`, private account details are available via `toArray()`. -- `monthlyUsage(?string $date = null): array` — current-account monthly usage (only for `me()`). +- `monthlyUsage(?string $date = null): array` — current-account monthly usage (only for `me()`). `$date` is an ISO date in `YYYY-MM-DD` format; the report covers the monthly usage cycle containing that date. Omit it (or pass `null`) to report the current month. - `limits(): array`, `updateLimits(mixed $newLimits): void` — account limits (only for `me()`). ```php diff --git a/docs/models.md b/docs/models.md index e2f2c38..a8317e9 100644 --- a/docs/models.md +++ b/docs/models.md @@ -138,7 +138,7 @@ Returned when listing/iterating the Apify Store. | Getter | Description | |---|---| | `getKey(): string` | The record key. | -| `getValue(): mixed` | The record value (decoded for JSON, raw string otherwise). | +| `getValue(): string` | The raw record value, as a string (decode it yourself when it is JSON). | | `getContentType(): ?string` | The record's content type. | ### `KeyValueStoreKey` @@ -168,7 +168,7 @@ One page returned by `listKeys()`. | `getTotalRequestCount(): ?int` | Total number of requests ever added. | ### `RequestQueueHead` -Returned by `listHead()` / `listAndLockHead()`. +Returned by `listHead()`. (`listAndLockHead()` returns a raw `array`, not this model.) | Getter | Description | |---|---| | `getItems(): array` | The `RequestQueueRequest` items at the head of the queue. | diff --git a/docs/options.md b/docs/options.md index fbb9b9a..774b0b2 100644 --- a/docs/options.md +++ b/docs/options.md @@ -10,6 +10,29 @@ $options = new ActorListOptions(my: true, limit: 10); ## Listing and pagination +### Manual offset paging with `withPagination()` +The offset-based options classes — `ListOptions`, `ActorListOptions`, `StorageListOptions`, +`StoreListOptions` and `DatasetListItemsOptions` — each expose a helper +`withPagination(?int $offset, ?int $limit): self`. + +It returns a copy of the options with the given `offset` and `limit`, preserving every other field. +The `iterate()` helpers use it internally to request successive pages, but you can also call it to +page manually through `list()` results — most useful for the Apify Store, whose collection is +otherwise only pageable via `iterate()`: + +```php +$options = new StoreListOptions(search: 'scraper'); +for ($offset = 0; ; $offset += 100) { + $page = $client->store()->list($options->withPagination($offset, 100)); + foreach ($page->getItems() as $item) { + // process each Actor + } + if ($page->getCount() < 100) { + break; // last page reached + } +} +``` + ### `ListOptions` Shared pagination/ordering controls used by most `list()` methods (builds, runs, tasks, schedules, webhooks, dispatches, Actor versions). @@ -19,6 +42,8 @@ webhooks, dispatches, Actor versions). | `limit` | `?int` | Maximum number of items to return. | | `desc` | `?bool` | Return items newest-first. | +Supports [`withPagination($offset, $limit)`](#manual-offset-paging-with-withpagination). + ### `ActorListOptions` | Field | Type | Description | |---|---|---| @@ -28,6 +53,8 @@ webhooks, dispatches, Actor versions). | `my` | `?bool` | Return only Actors owned by the current user. | | `sortBy` | `?string` | The sort field (e.g. `createdAt`, `stats.lastRunStartedAt`). | +Supports [`withPagination($offset, $limit)`](#manual-offset-paging-with-withpagination). + ### `StorageListOptions` Used when listing datasets, key-value stores and request queues. | Field | Type | Description | @@ -38,6 +65,8 @@ Used when listing datasets, key-value stores and request queues. | `unnamed` | `?bool` | Include unnamed storages in the result. | | `ownership` | `?string` | Filter by ownership (e.g. `OWNED`, `ACCESSIBLE`). | +Supports [`withPagination($offset, $limit)`](#manual-offset-paging-with-withpagination). + ### `RunListOptions` Extra filters for `runs()->list()`, combined with a `ListOptions`. | Field | Type | Description | @@ -51,7 +80,7 @@ For `store()->list()` / `store()->iterate()`. | Field | Type | Description | |---|---|---| | `offset` | `?int` | Number of Actors to skip. | -| `limit` | `?int` | Maximum number of Actors to return (also the per-page size when iterating). | +| `limit` | `?int` | Maximum number of Actors to return. When iterating, caps the total across all pages (the per-page size is `iterate()`'s separate `$chunkSize` argument). | | `search` | `?string` | Full-text search query. | | `sortBy` | `?string` | The sort field (e.g. `popularity`, `newest`). | | `category` | `?string` | Filter Actors by category. | @@ -61,6 +90,8 @@ For `store()->list()` / `store()->iterate()`. | `allowsAgenticUsers` | `?bool` | Filter to Actors that allow agentic users. | | `responseFormat` | `?string` | The response format (`full`, `agent`). | +Supports [`withPagination($offset, $limit)`](#manual-offset-paging-with-withpagination). + ### `LastRunOptions` For `actor()->lastRun()` / `task()->lastRun()`. | Field | Type | Description | @@ -137,7 +168,7 @@ For `actor()->build()`. ## Datasets ### `DatasetListItemsOptions` -For `dataset()->listItems()` and `createItemsPublicUrl()`. +For `dataset()->listItems()`, `iterateItems()`, and `createItemsPublicUrl()`. | Field | Type | Description | |---|---|---| | `offset` | `?int` | Number of items to skip. | @@ -156,6 +187,8 @@ For `dataset()->listItems()` and `createItemsPublicUrl()`. | `skipFailedPages` | `?bool` | Skip items that come from failed pages. | | `signature` | `?string` | A pre-shared URL signature granting access without an API token. | +Supports [`withPagination($offset, $limit)`](#manual-offset-paging-with-withpagination). + ### `DatasetDownloadOptions` For `dataset()->downloadItems()` (export formatting on top of the filtering above). | Field | Type | Description | diff --git a/docs/runs.md b/docs/runs.md index 56013b6..cb65bac 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -5,9 +5,14 @@ Snippets assume `$client = new ApifyClient('my-api-token');` and imported types. ## Run collection — `$client->runs()` - `list(?ListOptions $options = null, ?RunListOptions $filter = null): PaginationList` — list runs. +- `iterate(?ListOptions $options = null, ?RunListOptions $filter = null, ?int $chunkSize = null): iterable` — lazily iterate all runs, applying the filters to every page. The options' `limit` caps the total number yielded across all pages (unset = all); `$chunkSize` is the per-page size. ```php $page = $client->runs()->list(new ListOptions(limit: 10), new RunListOptions(status: ['SUCCEEDED'])); + +foreach ($client->runs()->iterate(new ListOptions(limit: 100), new RunListOptions(status: ['SUCCEEDED']), 50) as $run) { + echo $run->getId() . PHP_EOL; +} ``` An Actor's or task's runs are available at `$client->actor($id)->runs()` / `$client->task($id)->runs()`. @@ -17,7 +22,7 @@ An Actor's or task's runs are available at `$client->actor($id)->runs()` / `$cli - `get(?int $waitForFinishSecs = null): ?ActorRun` — fetch, optionally waiting server-side (max 60s). - `update(mixed $newFields): ActorRun` - `delete(): void` -- `abort(?bool $gracefully = null): ActorRun` +- `abort(?bool $gracefully = null): ActorRun` — aborts the run; with `$gracefully` `true` the run is signalled so it can finish its current request before terminating, `false` aborts immediately, and `null` (the default) lets the server apply its default (immediate abort). - `metamorph(string $targetActorId, mixed $input = null, ?MetamorphOptions $options = null): ActorRun` - `reboot(): ActorRun` - `resurrect(?RunResurrectOptions $options = null): ActorRun` diff --git a/docs/schedules.md b/docs/schedules.md index 0b87435..fb878ff 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -5,7 +5,8 @@ Schedules automatically start Actor or task runs at specified times. Snippets as ## Schedule collection — `$client->schedules()` -- `list(?ListOptions $options): PaginationList` +- `list(?ListOptions $options = null): PaginationList` +- `iterate(?ListOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all schedules, paging on demand. The options' `limit` caps the total number yielded across all pages (unset = all); `$chunkSize` is the per-page size. - `create(mixed $schedule): Schedule` ```php @@ -15,6 +16,10 @@ $schedule = $client->schedules()->create([ 'isEnabled' => true, 'actions' => [], ]); + +foreach ($client->schedules()->iterate(new ListOptions(), 50) as $s) { + echo $s->getId() . PHP_EOL; +} ``` ## A single schedule — `$client->schedule($id)` diff --git a/docs/storages.md b/docs/storages.md index d464312..72c5e3b 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -9,52 +9,76 @@ key-value-store collections additionally accept an optional `?array $schema` on ## Datasets -Collection — `$client->datasets()`: `list(?StorageListOptions $options = null): PaginationList`, -`getOrCreate(?string $name = null, ?array $schema = null): Dataset`. +Collection — `$client->datasets()`: + +- `list(?StorageListOptions $options = null): PaginationList` +- `iterate(?StorageListOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all datasets, paging on demand. The options' `limit` caps the total number yielded across all pages (unset = all); `$chunkSize` is the per-page size. +- `getOrCreate(?string $name = null, ?array $schema = null): Dataset` Single — `$client->dataset($id)`: - `get(): ?Dataset`, `update(mixed $newFields): Dataset`, `delete(): void` -- `listItems(?DatasetListItemsOptions $options = null): PaginationList` — items decoded to arrays. +- `listItems(?DatasetListItemsOptions $options = null): PaginationList` — one page of items decoded to PHP values. +- `iterateItems(?DatasetListItemsOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all items, paging on demand. The options' `limit` caps the total number of items yielded across all pages (unset = all); `$chunkSize` is the per-page size. Note: item-dropping filters (`skipEmpty`, and `clean` which implies it) are applied after `offset`/`limit`, so combining them with multi-page iteration can repeat or skip items (the iterator advances the offset by the post-filter count, matching the reference JS client). Iterate without those filters, or page explicitly with `listItems()` and filter client-side. (`skipHidden` only strips hidden fields from each item, not whole items, so it does not affect paging.) - `downloadItems(DownloadItemsFormat $format, ?DatasetDownloadOptions $options = null): string` — raw export bytes. - `pushItems(mixed $items): void` - `getStatistics(): ?array` -- `createItemsPublicUrl(?DatasetListItemsOptions $options = null, ?int $expiresInSecs = null): string` +- `createItemsPublicUrl(?DatasetListItemsOptions $options = null, ?int $expiresInSecs = null): string` — builds a shareable URL for downloading this dataset's items (forwarding the given item filters); for a private dataset it appends an access signature, optionally bounded to `$expiresInSecs`. ```php $dataset = $client->datasets()->getOrCreate('my-dataset'); $client->dataset($dataset->getId())->pushItems([['url' => 'https://a.com'], ['url' => 'https://b.com']]); $items = $client->dataset($dataset->getId())->listItems(new DatasetListItemsOptions(limit: 100)); $csv = $client->dataset($dataset->getId())->downloadItems(DownloadItemsFormat::CSV, new DatasetDownloadOptions(bom: true)); + +// Lazily iterate every item, fetching pages of 1000 on demand. +foreach ($client->dataset($dataset->getId())->iterateItems(new DatasetListItemsOptions(), 1000) as $item) { + echo ($item['url'] ?? '') . PHP_EOL; +} ``` ## Key-value stores -Collection — `$client->keyValueStores()`: `list(?StorageListOptions $options = null): PaginationList`, -`getOrCreate(?string $name = null, ?array $schema = null): KeyValueStore`. +Collection — `$client->keyValueStores()`: + +- `list(?StorageListOptions $options = null): PaginationList` +- `iterate(?StorageListOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all stores, paging on demand. The options' `limit` caps the total number yielded across all pages (unset = all); `$chunkSize` is the per-page size. +- `getOrCreate(?string $name = null, ?array $schema = null): KeyValueStore` Single — `$client->keyValueStore($id)`: - `get(): ?KeyValueStore`, `update(mixed $newFields): KeyValueStore`, `delete(): void` - `listKeys(?ListKeysOptions $options = null): KeyValueStoreKeysPage` -- `recordExists(string $key): bool` +- `iterateKeys(?ListKeysOptions $options = null): iterable` — lazily iterate all keys, following cursor pagination (`exclusiveStartKey`/`nextExclusiveStartKey`). The options' `limit` caps the total number of keys yielded across all pages (unset = all); there is no separate page-size argument (the per-page size follows the remaining cap, like the reference client). +- `recordExists(string $key): bool` — reports whether a record with the given key exists, without downloading its value (a `HEAD` request). - `getRecord(string $key, ?GetRecordOptions $options = null): ?KeyValueStoreRecord` - `setRecord(string $key, string $value, string $contentType, ?SetRecordOptions $options = null): void` -- `setRecordJson(string $key, mixed $value): void` -- `deleteRecord(string $key): void` -- `getRecordPublicUrl(string $key): string`, `createKeysPublicUrl(?ListKeysOptions, ?int $expiresInSecs): string` +- `setRecordJson(string $key, mixed $value): void` — convenience over `setRecord()` that JSON-encodes `$value` and stores it with a JSON content type. +- `deleteRecord(string $key): void` — permanently removes the record with the given key. +- `getRecordPublicUrl(string $key): string` — builds a shareable URL for downloading a single record; for a private store it appends an access signature so the URL works without an API token. +- `createKeysPublicUrl(?ListKeysOptions $options = null, ?int $expiresInSecs = null): string` — builds a shareable URL for listing this store's keys (forwarding the given key filters); for a private store it appends an access signature, optionally bounded to `$expiresInSecs`. ```php $store = $client->keyValueStores()->getOrCreate('my-store'); $client->keyValueStore($store->getId())->setRecordJson('OUTPUT', ['answer' => 42]); $record = $client->keyValueStore($store->getId())->getRecord('OUTPUT'); -echo $record?->getValue() ?? ''; +// getRecord() returns a KeyValueStoreRecord; getValue() gives the raw string - decode it yourself when it is JSON. +$decoded = json_decode($record?->getValue() ?? 'null', true); +echo ($decoded['answer'] ?? '') . PHP_EOL; + +// Lazily iterate every key (cursor-paginated) and read each record. +foreach ($client->keyValueStore($store->getId())->iterateKeys() as $key) { + echo $key->getKey() . PHP_EOL; +} ``` ## Request queues -Collection — `$client->requestQueues()`: `list(?StorageListOptions $options = null): PaginationList`, -`getOrCreate(?string $name = null): RequestQueue`. +Collection — `$client->requestQueues()`: + +- `list(?StorageListOptions $options = null): PaginationList` +- `iterate(?StorageListOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all request queues, paging on demand. The options' `limit` caps the total number yielded across all pages (unset = all); `$chunkSize` is the per-page size. +- `getOrCreate(?string $name = null): RequestQueue` A specific queue client is obtained with `$client->requestQueue($id, ?RequestQueueClientOptions $options = null)`. The optional `RequestQueueClientOptions` sets a stable `clientKey` (required to operate on locks the @@ -64,12 +88,16 @@ Single — `$client->requestQueue($id)`: - `get(): ?RequestQueue`, `update(mixed $newFields): RequestQueue`, `delete(): void` - `listHead(?int $limit = null): RequestQueueHead` -- `addRequest(RequestQueueRequest $request, bool $forefront = false): RequestQueueOperationInfo` -- `getRequest(string $id): ?RequestQueueRequest`, `updateRequest(RequestQueueRequest $request, bool $forefront = false): RequestQueueOperationInfo`, `deleteRequest(string $id): void` -- `batchAddRequests(array $requests, bool $forefront = false, ?BatchAddRequestsOptions $options = null): BatchAddResult` — every request must have a non-empty `uniqueKey`; input is split into batches of at most 25 requests that also respect the ~9 MiB payload limit. -- `batchDeleteRequests(mixed $requests): array` -- `listRequests(?ListRequestsOptions $options = null): array`, `paginateRequests(?PaginateRequestsOptions $options = null): iterable` -- `listAndLockHead(int $lockSecs, ?int $limit = null): array`, `prolongRequestLock(...)`, `deleteRequestLock(...)`, `unlockRequests(): array` +- `addRequest(RequestQueueRequest $request, bool $forefront = false): RequestQueueOperationInfo` — adds a request to the queue; when `$forefront` is `true` it is added to the front (handled before the rest) instead of the back. +- `getRequest(string $id): ?RequestQueueRequest`, `updateRequest(RequestQueueRequest $request, bool $forefront = false): RequestQueueOperationInfo` (with `$forefront` `true` the updated request is moved to the front of the queue), `deleteRequest(string $id): void` +- `batchAddRequests(array $requests, bool $forefront = false, ?BatchAddRequestsOptions $options = null): BatchAddResult` — every request must have a non-empty `uniqueKey`; with `$forefront` `true` the requests are added to the front of the queue; input is split into batches of at most 25 requests that also respect the ~9 MiB payload limit. +- `batchDeleteRequests(mixed $requests): array` — `$requests` is a list of entries that each identify a request to delete (e.g. by `id` or `uniqueKey`); returns the raw batch result as a decoded `array`. +- `listRequests(?ListRequestsOptions $options = null): array` — returns the raw paginated response as a decoded `array`. +- `paginateRequests(?PaginateRequestsOptions $options = null): iterable` — lazily iterate the queue's requests, yielding `RequestQueueRequest` instances and following cursor pagination (see the options note below). +- `listAndLockHead(int $lockSecs, ?int $limit = null): array` — atomically returns and locks up to `$limit` requests for `$lockSecs` seconds; returns the raw locked-head object as a decoded `array`. +- `prolongRequestLock(string $id, int $lockSecs, bool $forefront = false): array` — extends a request's lock by `$lockSecs`; with `$forefront` `true` the request is placed at the front of the queue once its lock expires; returns the raw response as a decoded `array`. +- `deleteRequestLock(string $id, bool $forefront = false): void` — releases the lock on a single request; with `$forefront` `true` the request is returned to the front of the queue. +- `unlockRequests(): array` — releases all locks the client holds on this queue; returns the raw response as a decoded `array`. - `withClientKey(string $clientKey): RequestQueueClient` `paginateRequests()` accepts a `PaginateRequestsOptions` with `limit` (total across all pages), diff --git a/docs/tasks.md b/docs/tasks.md index 75e7b0d..bd186d6 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -5,7 +5,8 @@ Tasks are pre-configured Actor runs with stored input. Snippets assume ## Task collection — `$client->tasks()` -- `list(?ListOptions $options): PaginationList` +- `list(?ListOptions $options = null): PaginationList` +- `iterate(?ListOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all tasks, paging on demand. The options' `limit` caps the total number yielded across all pages (unset = all); `$chunkSize` is the per-page size. - `create(mixed $task): Task` ```php @@ -14,6 +15,10 @@ $task = $client->tasks()->create([ 'name' => 'my-task', 'input' => ['message' => 'hello'], ]); + +foreach ($client->tasks()->iterate(new ListOptions(), 50) as $t) { + echo $t->getId() . PHP_EOL; +} ``` ## A single task — `$client->task($id)` diff --git a/docs/webhooks.md b/docs/webhooks.md index 1faced5..27d4e1d 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -4,12 +4,14 @@ Snippets assume `$client = new ApifyClient('my-api-token');` and imported types. ## Webhook collection — `$client->webhooks()` -- `list(?ListOptions $options): PaginationList` +- `list(?ListOptions $options = null): PaginationList` +- `iterate(?ListOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all webhooks, paging on demand. The options' `limit` caps the total number yielded across all pages (unset = all); `$chunkSize` is the per-page size. - `create(mixed $webhook): Webhook` Webhooks nested under an Actor or task (`$client->actor($id)->webhooks()`, -`$client->task($id)->webhooks()`) are **read-only** — they support `list(...)` only. Create webhooks -through the account-wide collection, targeting an Actor or task via the webhook's `condition`. +`$client->task($id)->webhooks()`) are **read-only** — they support `list(...)` and `iterate(...)` +only. Create webhooks through the account-wide collection, targeting an Actor or task via the +webhook's `condition`. ```php $webhook = $client->webhooks()->create([ @@ -32,10 +34,14 @@ $client->webhook('WEBHOOK_ID')->dispatches()->list(new ListOptions(limit: 10)); ## Webhook dispatches — `$client->webhookDispatches()` / `$client->webhookDispatch($id)` -- Collection: `list(?ListOptions $options): PaginationList`. +- Collection: `list(?ListOptions $options = null): PaginationList`, `iterate(?ListOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all dispatches, paging on demand (options' `limit` caps the total; `$chunkSize` is the per-page size). - Single: `get(): ?WebhookDispatch`. ```php $page = $client->webhookDispatches()->list(new ListOptions(limit: 5)); $dispatch = $client->webhookDispatch('DISPATCH_ID')->get(); + +foreach ($client->webhookDispatches()->iterate(new ListOptions(), 50) as $d) { + echo $d->getId() . PHP_EOL; +} ``` diff --git a/phpunit.xml.dist b/phpunit.xml.dist index a003a70..7a952cd 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -4,6 +4,7 @@ bootstrap="vendor/autoload.php" colors="true" failOnWarning="true" + failOnEmptyTestSuite="true" cacheDirectory=".phpunit.cache"> diff --git a/src/Internal/Compression.php b/src/Internal/Compression.php index e19b728..3bcdc27 100644 --- a/src/Internal/Compression.php +++ b/src/Internal/Compression.php @@ -9,7 +9,10 @@ * * Large request bodies are compressed before being sent, saving bandwidth on uploads (Actor inputs, * key-value-store records, dataset item batches, ...). Brotli ({@code Content-Encoding: br}) is - * preferred when available and gzip ({@code Content-Encoding: gzip}) is used as a fallback. + * preferred when available and gzip ({@code Content-Encoding: gzip}) is used as a fallback — the same + * codec choice, brotli quality (6), and size threshold (1024 bytes) as the reference client's + * {@code maybeCompressValue}. The API accepts br/gzip/deflate as request {@code Content-Encoding} + * (see apify-docs #2750), so preferring brotli is valid. * * In PHP, brotli lives in the optional PECL {@code brotli} extension, which is frequently absent, * while gzip ({@code gzencode}) ships with the standard {@code zlib} extension. We therefore prefer diff --git a/src/Internal/HttpClientCore.php b/src/Internal/HttpClientCore.php index 5d49e51..ee67f3b 100644 --- a/src/Internal/HttpClientCore.php +++ b/src/Internal/HttpClientCore.php @@ -33,6 +33,9 @@ final class HttpClientCore /** Exponential-backoff multiplier applied to the inter-retry delay after each attempt. */ private const BACKOFF_FACTOR = 2; + /** Multiplier applied to the per-attempt timeout on each retry (independent of {@see BACKOFF_FACTOR}). */ + private const TIMEOUT_BACKOFF_FACTOR = 2; + private const NOT_FOUND = 404; public function __construct( @@ -205,15 +208,15 @@ private function doAttempt( } /** - * Returns {@code min(overall, base * 2^(attempt-1))}: the first attempt uses the base timeout; - * each retry doubles it (a slow-but-progressing connection gets more time) while never exceeding - * the overall budget. + * Returns {@code min(overall, base * TIMEOUT_BACKOFF_FACTOR^(attempt-1))}: the first attempt uses + * the base timeout; each retry scales it up by {@see TIMEOUT_BACKOFF_FACTOR} (a slow-but-progressing + * connection gets more time) while never exceeding the overall budget. */ private function attemptTimeout(float $base, int $attempt): float { $scaled = $base; for ($i = 1; $i < $attempt; $i++) { - $scaled *= 2; + $scaled *= self::TIMEOUT_BACKOFF_FACTOR; if ($scaled >= $this->retry->timeoutSecs) { return $this->retry->timeoutSecs; } diff --git a/src/Internal/Json.php b/src/Internal/Json.php index e145277..cc993a8 100644 --- a/src/Internal/Json.php +++ b/src/Internal/Json.php @@ -13,6 +13,9 @@ */ final class Json { + /** Maximum nesting depth passed to {@see json_decode()} (PHP's own default). */ + private const MAX_JSON_DEPTH = 512; + private function __construct() { } @@ -33,7 +36,7 @@ public static function decode(string $body): mixed if ($body === '') { return null; } - return json_decode($body, true, 512, JSON_THROW_ON_ERROR); + return json_decode($body, true, self::MAX_JSON_DEPTH, JSON_THROW_ON_ERROR); } /** diff --git a/src/Internal/ResourceContext.php b/src/Internal/ResourceContext.php index de5b945..2b9a3dd 100644 --- a/src/Internal/ResourceContext.php +++ b/src/Internal/ResourceContext.php @@ -6,6 +6,7 @@ use Apify\Client\Exception\ApifyApiException; use Apify\Client\Model\PaginationList; +use Generator; use Psr\Http\Message\ResponseInterface; use RuntimeException; @@ -188,6 +189,70 @@ public function listResource(string $subPath, QueryParams $params, callable $hyd return PaginationList::fromData($data, $hydrate); } + /** + * Lazily iterates over every item of an offset/limit-paginated listing, fetching pages on demand. + * + * Ports the reference client's paginated iterator ({@code _listPaginatedFromCallback}): + * {@code $limit} caps the TOTAL number of items yielded across all pages ({@code null} = no cap, + * i.e. all items), while {@code $chunkSize} caps how many items are requested per page + * ({@code null} = the server default). The two are independent — {@code $limit} is never reused + * as the page size. {@code $startOffset} is the offset of the first page. + * + * @template T + * @param callable(int,?int):PaginationList $fetchPage receives (offset, pageLimit) and returns that page + * @return Generator + */ + public static function paginateOffset(int $startOffset, ?int $limit, ?int $chunkSize, callable $fetchPage): Generator + { + // First page: request min(limit, chunkSize) items. A null/0 on either side means "unbounded", + // so the other bound wins (mirrors the reference client's minForLimitParam). + $page = $fetchPage($startOffset, self::minLimit($limit, $chunkSize)); + $items = $page->getItems(); + foreach ($items as $item) { + yield $item; + } + + $total = $page->getTotal(); + // Effective total cap: the smaller of the requested limit (0/null => all) and what exists. + $cap = min(($limit !== null && $limit > 0) ? $limit : $total, $total); + $currentOffset = $startOffset + count($items); + // Items still to yield, bounded both by what remains after the start offset and by the cap. + $remaining = min($total - $startOffset, $cap) - count($items); + + // Guard on the previous page being non-empty so an over-reported total (a page shorter than + // its claimed total) terminates instead of looping forever. + while (count($items) > 0 && $remaining > 0) { + $page = $fetchPage($currentOffset, self::minLimit($remaining, $chunkSize)); + $items = $page->getItems(); + foreach ($items as $item) { + yield $item; + } + $currentOffset += count($items); + $remaining -= count($items); + } + } + + /** + * Returns the smaller of two optional positive bounds, treating {@code null} or {@code 0} as + * "unbounded" (the API treats {@code limit=0} as unset). Mirrors the reference minForLimitParam. + */ + private static function minLimit(?int $a, ?int $b): ?int + { + if ($a === 0) { + $a = null; + } + if ($b === 0) { + $b = null; + } + if ($a === null) { + return $b; + } + if ($b === null) { + return $a; + } + return min($a, $b); + } + /** * POST to create a resource with a JSON-serializable body, returning the decoded {@code data}. * diff --git a/src/Options/ActorListOptions.php b/src/Options/ActorListOptions.php index b05acb9..c41dafa 100644 --- a/src/Options/ActorListOptions.php +++ b/src/Options/ActorListOptions.php @@ -23,6 +23,15 @@ public function __construct( ) { } + /** + * Returns a copy of these options with a new {@code offset}/{@code limit}, preserving the other + * filters. Used by lazy iteration to request successive pages. + */ + public function withPagination(?int $offset, ?int $limit): self + { + return new self($offset, $limit, $this->desc, $this->my, $this->sortBy); + } + /** @internal */ public function appendTo(QueryParams $q): void { diff --git a/src/Options/DatasetListItemsOptions.php b/src/Options/DatasetListItemsOptions.php index 7080eb1..d82b849 100644 --- a/src/Options/DatasetListItemsOptions.php +++ b/src/Options/DatasetListItemsOptions.php @@ -48,6 +48,31 @@ public function __construct( ) { } + /** + * Returns a copy of these options with a new {@code offset}/{@code limit}, preserving every other + * field. Used by {@see \Apify\Client\Resource\DatasetClient::iterateItems()} to request pages. + */ + public function withPagination(?int $offset, ?int $limit): self + { + return new self( + $offset, + $limit, + $this->desc, + $this->fields, + $this->outputFields, + $this->omit, + $this->skipEmpty, + $this->skipHidden, + $this->clean, + $this->unwind, + $this->flatten, + $this->view, + $this->simplified, + $this->skipFailedPages, + $this->signature, + ); + } + /** @internal */ public function appendTo(QueryParams $q): void { diff --git a/src/Options/ListOptions.php b/src/Options/ListOptions.php index c16ea64..651f00e 100644 --- a/src/Options/ListOptions.php +++ b/src/Options/ListOptions.php @@ -23,6 +23,15 @@ public function __construct( ) { } + /** + * Returns a copy of these options with a new {@code offset}/{@code limit}, preserving the other + * filters. Used by lazy iteration to request successive pages. + */ + public function withPagination(?int $offset, ?int $limit): self + { + return new self($offset, $limit, $this->desc); + } + /** @internal */ public function appendTo(QueryParams $q): void { diff --git a/src/Options/StorageListOptions.php b/src/Options/StorageListOptions.php index ebb0a29..921b9e8 100644 --- a/src/Options/StorageListOptions.php +++ b/src/Options/StorageListOptions.php @@ -27,6 +27,15 @@ public function __construct( ) { } + /** + * Returns a copy of these options with a new {@code offset}/{@code limit}, preserving the other + * filters. Used by lazy iteration to request successive pages. + */ + public function withPagination(?int $offset, ?int $limit): self + { + return new self($offset, $limit, $this->desc, $this->unnamed, $this->ownership); + } + /** @internal */ public function appendTo(QueryParams $q): void { diff --git a/src/Options/StoreListOptions.php b/src/Options/StoreListOptions.php index abf95d6..10ca75c 100644 --- a/src/Options/StoreListOptions.php +++ b/src/Options/StoreListOptions.php @@ -12,7 +12,10 @@ final class StoreListOptions public function __construct( /** Number of Actors to skip. */ public readonly ?int $offset = null, - /** Maximum number of Actors to return (also the per-page size when iterating). */ + /** + * Maximum number of Actors to return. When iterating, this caps the total number of Actors + * yielded across all pages (the per-page size is the separate {@code chunkSize} argument). + */ public readonly ?int $limit = null, /** Full-text search query. */ public readonly ?string $search = null, @@ -36,12 +39,15 @@ public function __construct( ) { } - /** Returns a copy of these options with a new {@code offset} (used by lazy iteration). */ - public function withOffset(?int $offset): self + /** + * Returns a copy of these options with a new {@code offset}/{@code limit}, preserving the other + * filters. Used by lazy iteration to request successive pages. + */ + public function withPagination(?int $offset, ?int $limit): self { return new self( $offset, - $this->limit, + $limit, $this->search, $this->sortBy, $this->category, diff --git a/src/Resource/AbstractWebhookCollectionClient.php b/src/Resource/AbstractWebhookCollectionClient.php index 32543d4..c865c23 100644 --- a/src/Resource/AbstractWebhookCollectionClient.php +++ b/src/Resource/AbstractWebhookCollectionClient.php @@ -10,6 +10,7 @@ use Apify\Client\Model\PaginationList; use Apify\Client\Model\Webhook; use Apify\Client\Options\ListOptions; +use Generator; /** * Shared read-only behavior for webhook collections. Both the account-wide collection @@ -40,4 +41,22 @@ public function list(?ListOptions $options = null): PaginationList ($options ?? new ListOptions())->appendTo($params); return $this->ctx->listResource('', $params, static fn (array $d) => new Webhook($d)); } + + /** + * Lazily iterates over webhooks, fetching pages on demand. The options' {@code limit} caps the + * total number of webhooks yielded across all pages ({@code null} = all); {@code $chunkSize} is + * the per-page size ({@code null} = the server default). + * + * @return Generator + */ + public function iterate(?ListOptions $options = null, ?int $chunkSize = null): Generator + { + $options ??= new ListOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit)), + ); + } } diff --git a/src/Resource/ActorCollectionClient.php b/src/Resource/ActorCollectionClient.php index f39b9e5..af756df 100644 --- a/src/Resource/ActorCollectionClient.php +++ b/src/Resource/ActorCollectionClient.php @@ -10,6 +10,7 @@ use Apify\Client\Model\Actor; use Apify\Client\Model\PaginationList; use Apify\Client\Options\ActorListOptions; +use Generator; /** A client for the Actor collection ({@code GET/POST /v2/actors}). */ final class ActorCollectionClient @@ -34,6 +35,24 @@ public function list(?ActorListOptions $options = null): PaginationList return $this->ctx->listResource('', $params, static fn (array $d) => new Actor($d)); } + /** + * Lazily iterates over the account's Actors, fetching pages on demand. The options' {@code limit} + * caps the total number of Actors yielded across all pages ({@code null} = all); {@code $chunkSize} + * is the per-page size ({@code null} = the server default). + * + * @return Generator + */ + public function iterate(?ActorListOptions $options = null, ?int $chunkSize = null): Generator + { + $options ??= new ActorListOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit)), + ); + } + /** * Creates a new Actor. * diff --git a/src/Resource/ActorEnvVarCollectionClient.php b/src/Resource/ActorEnvVarCollectionClient.php index 77f80aa..b0d57e5 100644 --- a/src/Resource/ActorEnvVarCollectionClient.php +++ b/src/Resource/ActorEnvVarCollectionClient.php @@ -9,6 +9,7 @@ use Apify\Client\Internal\ResourceContext; use Apify\Client\Model\ActorEnvVar; use Apify\Client\Model\PaginationList; +use Generator; /** * A client for an Actor version's environment variable collection ({@code GET/POST @@ -34,6 +35,27 @@ public function list(): PaginationList return $this->ctx->listResource('', new QueryParams(), static fn (array $d) => ActorEnvVar::fromArray($d)); } + /** + * Lazily iterates over the version's environment variables, fetching pages on demand. + * {@code $chunkSize} caps the per-page size ({@code null} = the server default). This endpoint is + * not filtered, so iteration mirrors the reference client's parameterless {@code list()} iterator. + * + * @return Generator + */ + public function iterate(?int $chunkSize = null): Generator + { + return ResourceContext::paginateOffset( + 0, + null, + $chunkSize, + function (int $offset, ?int $pageLimit) { + $params = new QueryParams(); + $params->addInt('offset', $offset)->addInt('limit', $pageLimit); + return $this->ctx->listResource('', $params, static fn (array $d) => ActorEnvVar::fromArray($d)); + }, + ); + } + /** Creates a new environment variable. */ public function create(ActorEnvVar $envVar): ActorEnvVar { diff --git a/src/Resource/ActorVersionCollectionClient.php b/src/Resource/ActorVersionCollectionClient.php index 2a21b48..0614849 100644 --- a/src/Resource/ActorVersionCollectionClient.php +++ b/src/Resource/ActorVersionCollectionClient.php @@ -10,6 +10,7 @@ use Apify\Client\Model\ActorVersion; use Apify\Client\Model\PaginationList; use Apify\Client\Options\ListOptions; +use Generator; /** A client for an Actor's version collection ({@code GET/POST /v2/actors/{actorId}/versions}). */ final class ActorVersionCollectionClient @@ -34,6 +35,24 @@ public function list(?ListOptions $options = null): PaginationList return $this->ctx->listResource('', $params, static fn (array $d) => new ActorVersion($d)); } + /** + * Lazily iterates over the Actor's versions, fetching pages on demand. The options' {@code limit} + * caps the total number of versions yielded across all pages ({@code null} = all); + * {@code $chunkSize} is the per-page size ({@code null} = the server default). + * + * @return Generator + */ + public function iterate(?ListOptions $options = null, ?int $chunkSize = null): Generator + { + $options ??= new ListOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit)), + ); + } + /** * Creates a new Actor version. * diff --git a/src/Resource/BuildClient.php b/src/Resource/BuildClient.php index bfa1f36..cdbedd5 100644 --- a/src/Resource/BuildClient.php +++ b/src/Resource/BuildClient.php @@ -23,7 +23,10 @@ public function __construct(private HttpClientCore $http, string $baseUrl, strin /** * Fetches the build, optionally asking the API to wait up to {@code $waitForFinishSecs} seconds - * (max 60) for the build to finish before responding. Returns {@code null} if it does not exist. + * for the build to finish before responding. The value is clamped client-side to the per-request + * timeout budget (minus a safety margin) so the server is never asked to hold the connection + * longer than the client will wait; the server additionally caps the wait at 60s. Returns + * {@code null} if it does not exist. */ public function get(?int $waitForFinishSecs = null): ?Build { diff --git a/src/Resource/BuildCollectionClient.php b/src/Resource/BuildCollectionClient.php index 492d380..a62f9b7 100644 --- a/src/Resource/BuildCollectionClient.php +++ b/src/Resource/BuildCollectionClient.php @@ -10,6 +10,7 @@ use Apify\Client\Model\Build; use Apify\Client\Model\PaginationList; use Apify\Client\Options\ListOptions; +use Generator; /** * A client for a build collection: the account-wide collection ({@code GET /v2/actor-builds}) or an @@ -36,4 +37,22 @@ public function list(?ListOptions $options = null): PaginationList ($options ?? new ListOptions())->appendTo($params); return $this->ctx->listResource('', $params, static fn (array $d) => new Build($d)); } + + /** + * Lazily iterates over builds, fetching pages on demand. The options' {@code limit} caps the total + * number of builds yielded across all pages ({@code null} = all); {@code $chunkSize} is the + * per-page size ({@code null} = the server default). + * + * @return Generator + */ + public function iterate(?ListOptions $options = null, ?int $chunkSize = null): Generator + { + $options ??= new ListOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit)), + ); + } } diff --git a/src/Resource/DatasetClient.php b/src/Resource/DatasetClient.php index d80350e..1861926 100644 --- a/src/Resource/DatasetClient.php +++ b/src/Resource/DatasetClient.php @@ -14,6 +14,7 @@ use Apify\Client\Options\DatasetDownloadOptions; use Apify\Client\Options\DatasetListItemsOptions; use Apify\Client\Options\DownloadItemsFormat; +use Generator; use Psr\Http\Message\ResponseInterface; /** A client for a specific dataset (and run-nested variants). */ @@ -103,6 +104,38 @@ public function listItems(?DatasetListItemsOptions $options = null): PaginationL ); } + /** + * Lazily iterates over the dataset's items, fetching pages on demand. Each item is decoded to a + * PHP value (an associative array for objects), like {@see listItems()}. + * + * The options' {@code limit} caps the total number of items yielded across all pages ({@code null} + * = all), {@code offset} is the starting offset, and {@code $chunkSize} is the per-page size + * ({@code null} = the server default). All other {@see DatasetListItemsOptions} fields (field + * selection, filtering, ordering) are applied to every page. + * + * Note: item-dropping filters ({@code skipEmpty}, and {@code clean} which implies it) are applied + * after {@code offset}/{@code limit}, so a page can return fewer items than requested while + * {@code X-Apify-Pagination-Total} still reflects the raw total. Because the iterator advances + * the offset by the post-filter item count (matching the reference JS client), combining those + * filters with multi-page iteration can repeat items across overlapping windows or, if a whole + * offset window is filtered out, end iteration early and skip the remaining items. Iterate + * without those filters, or page explicitly with {@see listItems()} and filter client-side. + * ({@code skipHidden} only strips hidden fields from each item, not whole items, so it does not + * affect paging.) + * + * @return Generator + */ + public function iterateItems(?DatasetListItemsOptions $options = null, ?int $chunkSize = null): Generator + { + $options ??= new DatasetListItemsOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->listItems($options->withPagination($offset, $pageLimit)), + ); + } + /** * Downloads dataset items serialized in the given format, returning the raw bytes as a string. * Unlike {@see listItems()} (parsed items), this returns the items already serialized to JSON, diff --git a/src/Resource/DatasetCollectionClient.php b/src/Resource/DatasetCollectionClient.php index 7fe101b..166dcad 100644 --- a/src/Resource/DatasetCollectionClient.php +++ b/src/Resource/DatasetCollectionClient.php @@ -10,6 +10,7 @@ use Apify\Client\Model\Dataset; use Apify\Client\Model\PaginationList; use Apify\Client\Options\StorageListOptions; +use Generator; /** A client for the dataset collection ({@code GET/POST /v2/datasets}). */ final class DatasetCollectionClient @@ -34,6 +35,24 @@ public function list(?StorageListOptions $options = null): PaginationList return $this->ctx->listResource('', $params, static fn (array $d) => new Dataset($d)); } + /** + * Lazily iterates over datasets, fetching pages on demand. The options' {@code limit} caps the + * total number of datasets yielded across all pages ({@code null} = all); {@code $chunkSize} is + * the per-page size ({@code null} = the server default). + * + * @return Generator + */ + public function iterate(?StorageListOptions $options = null, ?int $chunkSize = null): Generator + { + $options ??= new StorageListOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit)), + ); + } + /** * Gets the dataset with the given name, creating it if it does not exist. An empty/{@code null} * name creates a new unnamed dataset. An optional {@code $schema} (an associative array) is sent diff --git a/src/Resource/KeyValueStoreClient.php b/src/Resource/KeyValueStoreClient.php index 13f2047..b723912 100644 --- a/src/Resource/KeyValueStoreClient.php +++ b/src/Resource/KeyValueStoreClient.php @@ -10,11 +10,13 @@ use Apify\Client\Internal\ResourceContext; use Apify\Client\Internal\Signatures; use Apify\Client\Model\KeyValueStore; +use Apify\Client\Model\KeyValueStoreKey; use Apify\Client\Model\KeyValueStoreKeysPage; use Apify\Client\Model\KeyValueStoreRecord; use Apify\Client\Options\GetRecordOptions; use Apify\Client\Options\ListKeysOptions; use Apify\Client\Options\SetRecordOptions; +use Generator; /** A client for a specific key-value store (and run-nested variants). */ final class KeyValueStoreClient @@ -81,6 +83,58 @@ public function listKeys(?ListKeysOptions $options = null): KeyValueStoreKeysPag return KeyValueStoreKeysPage::fromData($this->ctx->getResourceRequired('keys', $params)); } + /** + * Lazily iterates over the store's keys, transparently following cursor pagination + * ({@code exclusiveStartKey}/{@code nextExclusiveStartKey}), mirroring the reference client's + * async-iterable {@code listKeys()}. + * + * The options' {@code limit} caps the total number of keys yielded across all pages ({@code null} + * or {@code 0} = all); {@code exclusiveStartKey} starts the listing after a given key; {@code prefix} and + * {@code collection} restrict which keys are listed. Unlike the offset/limit collection iterators, + * there is no separate page-size argument: the per-page size follows the remaining total cap (or + * the server default when unbounded), exactly as the reference client does. + * + * @return Generator + */ + public function iterateKeys(?ListKeysOptions $options = null): Generator + { + $options ??= new ListKeysOptions(); + // Total cap across all pages. null or 0 means "iterate the whole store" (the API treats + // limit=0 as unset). Normalizing 0 -> null here matches the offset paginator's minLimit + // convention and the sibling clients, and stops a per-page limit=0 from short-circuiting + // the iteration after a single page. + $limit = ($options->limit !== null && $options->limit > 0) ? $options->limit : null; + $exclusiveStartKey = $options->exclusiveStartKey; + $iterated = 0; + + while (true) { + // Ask for only as many keys as remain under the total cap (null = server default). + $remaining = $limit !== null ? $limit - $iterated : null; + $page = $this->listKeys(new ListKeysOptions( + limit: $remaining, + exclusiveStartKey: $exclusiveStartKey, + prefix: $options->prefix, + collection: $options->collection, + signature: $options->signature, + )); + + $items = $page->getItems(); + if ($items === []) { + return; + } + foreach ($items as $item) { + yield $item; + } + $iterated += count($items); + + $nextKey = $page->getNextExclusiveStartKey(); + if (($limit !== null && $iterated >= $limit) || !$page->isTruncated() || $nextKey === null || $nextKey === '') { + return; + } + $exclusiveStartKey = $nextKey; + } + } + /** Reports whether a record with the given key exists. */ public function recordExists(string $key): bool { diff --git a/src/Resource/KeyValueStoreCollectionClient.php b/src/Resource/KeyValueStoreCollectionClient.php index 144fac1..ed74380 100644 --- a/src/Resource/KeyValueStoreCollectionClient.php +++ b/src/Resource/KeyValueStoreCollectionClient.php @@ -10,6 +10,7 @@ use Apify\Client\Model\KeyValueStore; use Apify\Client\Model\PaginationList; use Apify\Client\Options\StorageListOptions; +use Generator; /** A client for the key-value store collection ({@code GET/POST /v2/key-value-stores}). */ final class KeyValueStoreCollectionClient @@ -34,6 +35,24 @@ public function list(?StorageListOptions $options = null): PaginationList return $this->ctx->listResource('', $params, static fn (array $d) => new KeyValueStore($d)); } + /** + * Lazily iterates over key-value stores, fetching pages on demand. The options' {@code limit} + * caps the total number of stores yielded across all pages ({@code null} = all); {@code $chunkSize} + * is the per-page size ({@code null} = the server default). + * + * @return Generator + */ + public function iterate(?StorageListOptions $options = null, ?int $chunkSize = null): Generator + { + $options ??= new StorageListOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit)), + ); + } + /** * Gets the store with the given name, creating it if it does not exist. An empty/{@code null} * name creates a new unnamed store. An optional {@code $schema} (an associative array) is sent diff --git a/src/Resource/RequestQueueClient.php b/src/Resource/RequestQueueClient.php index 920f1a6..ac3f6d3 100644 --- a/src/Resource/RequestQueueClient.php +++ b/src/Resource/RequestQueueClient.php @@ -213,6 +213,13 @@ public function batchAddRequests( $options ??= new BatchAddRequestsOptions(); $requests = array_values($requests); + $payloadSizeLimitBytes = self::MAX_PAYLOAD_SIZE_BYTES + - (int) ceil(self::MAX_PAYLOAD_SIZE_BYTES * self::PAYLOAD_SAFETY_BUFFER_PERCENT); + + // Validate the whole input up front, before any HTTP call. Both the empty-uniqueKey check and + // the per-request oversized check must run here (not inside the send loop): otherwise an + // oversized request in the middle of a large batch would only be discovered after earlier + // chunks had already been POSTed, leaving the queue partially mutated. foreach ($requests as $i => $request) { $uniqueKey = $request->getUniqueKey(); if ($uniqueKey === null || $uniqueKey === '') { @@ -220,18 +227,23 @@ public function batchAddRequests( sprintf('batchAddRequests: the request at index %d is missing a non-empty uniqueKey', $i) ); } + $itemBytes = strlen(Json::encode($request->toArray())); + if ($itemBytes > $payloadSizeLimitBytes) { + throw new InvalidArgumentException(sprintf( + 'batchAddRequests: the request at index %d exceeds the maximum payload size (%d bytes)', + $i, + $payloadSizeLimitBytes + )); + } } - $payloadSizeLimitBytes = self::MAX_PAYLOAD_SIZE_BYTES - - (int) ceil(self::MAX_PAYLOAD_SIZE_BYTES * self::PAYLOAD_SAFETY_BUFFER_PERCENT); - $merged = new BatchAddResult(); $index = 0; $count = count($requests); while ($index < $count) { // Bound each batch first by the count limit (25), then by payload byte size. $countSlice = array_slice($requests, $index, self::MAX_REQUESTS_PER_BATCH); - $chunk = self::sliceByByteLength($countSlice, $payloadSizeLimitBytes, $index); + $chunk = self::sliceByByteLength($countSlice, $payloadSizeLimitBytes); $merged->merge($this->batchAddChunkWithRetries($chunk, $forefront, $options)); $index += count($chunk); } @@ -243,11 +255,15 @@ public function batchAddRequests( * {@code $maxByteLength}, always keeping at least one request so iteration makes progress. Ports * the reference client's {@code sliceArrayByByteLength}. * + * Callers must have already validated (in {@see batchAddRequests()}) that every individual request + * fits under {@code $maxByteLength}, so the always-keep-one fallback never produces an over-limit + * chunk. That up-front validation is what lets this slicer run inside the send loop without risking + * a partially-mutated queue. + * * @param list $requests * @return list - * @throws InvalidArgumentException if a single request exceeds {@code $maxByteLength} */ - private static function sliceByByteLength(array $requests, int $maxByteLength, int $startIndex): array + private static function sliceByByteLength(array $requests, int $maxByteLength): array { $payloads = array_map(static fn (RequestQueueRequest $r) => $r->toArray(), $requests); if (strlen(Json::encode($payloads)) < $maxByteLength) { @@ -256,15 +272,8 @@ private static function sliceByByteLength(array $requests, int $maxByteLength, i $sliced = []; $byteLength = 2; // the two bytes of an empty array "[]" - foreach ($requests as $i => $request) { + foreach ($requests as $request) { $itemBytes = strlen(Json::encode($request->toArray())); - if ($itemBytes > $maxByteLength) { - throw new InvalidArgumentException(sprintf( - 'batchAddRequests: the request at index %d exceeds the maximum payload size (%d bytes)', - $startIndex + $i, - $maxByteLength - )); - } if ($byteLength + $itemBytes >= $maxByteLength) { break; } @@ -272,7 +281,7 @@ private static function sliceByByteLength(array $requests, int $maxByteLength, i $sliced[] = $request; } - // Guarantee forward progress: keep at least the first request (it fits under the hard max). + // Guarantee forward progress: keep at least the first request (pre-validated to fit under the max). if ($sliced === []) { $sliced[] = $requests[0]; } @@ -476,7 +485,10 @@ public function paginateRequests(?PaginateRequestsOptions $options = null): Gene $options->validate(); $maxPageLimit = $options->maxPageLimit ?? PaginateRequestsOptions::DEFAULT_MAX_PAGE_LIMIT; - $limit = $options->limit; // total across all pages; null = unbounded + // Total cap across all pages. null or 0 means "iterate all" (the API treats limit=0 as + // unset). Normalizing 0 -> null here matches iterateKeys and the offset paginator's minLimit + // convention, and stops a per-page limit=0 from short-circuiting the iteration after one page. + $limit = ($options->limit !== null && $options->limit > 0) ? $options->limit : null; $nextCursor = $options->cursor; $nextExclusiveStartId = $options->exclusiveStartId; // used for the first page only $iterated = 0; diff --git a/src/Resource/RequestQueueCollectionClient.php b/src/Resource/RequestQueueCollectionClient.php index 8d5cebb..2216980 100644 --- a/src/Resource/RequestQueueCollectionClient.php +++ b/src/Resource/RequestQueueCollectionClient.php @@ -10,6 +10,7 @@ use Apify\Client\Model\PaginationList; use Apify\Client\Model\RequestQueue; use Apify\Client\Options\StorageListOptions; +use Generator; /** A client for the request queue collection ({@code GET/POST /v2/request-queues}). */ final class RequestQueueCollectionClient @@ -34,6 +35,24 @@ public function list(?StorageListOptions $options = null): PaginationList return $this->ctx->listResource('', $params, static fn (array $d) => new RequestQueue($d)); } + /** + * Lazily iterates over request queues, fetching pages on demand. The options' {@code limit} caps + * the total number of queues yielded across all pages ({@code null} = all); {@code $chunkSize} is + * the per-page size ({@code null} = the server default). + * + * @return Generator + */ + public function iterate(?StorageListOptions $options = null, ?int $chunkSize = null): Generator + { + $options ??= new StorageListOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit)), + ); + } + /** * Gets the queue with the given name, creating it if it does not exist. An empty/{@code null} * name creates a new unnamed queue. diff --git a/src/Resource/RunClient.php b/src/Resource/RunClient.php index dbe4bc7..3088062 100644 --- a/src/Resource/RunClient.php +++ b/src/Resource/RunClient.php @@ -58,7 +58,10 @@ public function setLastRunParams(LastRunOptions $options): void /** * Fetches the run, optionally asking the API to wait up to {@code $waitForFinishSecs} seconds - * (max 60) for the run to reach a terminal state. Returns {@code null} if it does not exist. + * for the run to reach a terminal state. The value is clamped client-side to the per-request + * timeout budget (minus a safety margin) so the server is never asked to hold the connection + * longer than the client will wait; the server additionally caps the wait at 60s. Returns + * {@code null} if it does not exist. */ public function get(?int $waitForFinishSecs = null): ?ActorRun { @@ -106,7 +109,9 @@ public function metamorph(string $targetActorId, mixed $input = null, ?Metamorph { $options ??= new MetamorphOptions(); $params = new QueryParams(); - $params->addString('targetActorId', $targetActorId); + // Normalize the target Actor id to the URL-safe `username~actor-name` form (first `/`→`~`), + // matching the reference JS client, so a slash-form id is sent as the same wire value. + $params->addString('targetActorId', ResourceContext::toSafeId($targetActorId)); if ($options->build !== null && $options->build !== '') { $params->addString('build', $options->build); } diff --git a/src/Resource/RunCollectionClient.php b/src/Resource/RunCollectionClient.php index e94000b..7ea64a8 100644 --- a/src/Resource/RunCollectionClient.php +++ b/src/Resource/RunCollectionClient.php @@ -11,6 +11,7 @@ use Apify\Client\Model\PaginationList; use Apify\Client\Options\ListOptions; use Apify\Client\Options\RunListOptions; +use Generator; /** * A client for a run collection: the account-wide collection ({@code GET /v2/actor-runs}), an @@ -39,4 +40,22 @@ public function list(?ListOptions $options = null, ?RunListOptions $filter = nul ($filter ?? new RunListOptions())->appendTo($params); return $this->ctx->listResource('', $params, static fn (array $d) => new ActorRun($d)); } + + /** + * Lazily iterates over runs, fetching pages on demand and applying the run-specific filters to + * every page. The options' {@code limit} caps the total number of runs yielded across all pages + * ({@code null} = all); {@code $chunkSize} is the per-page size ({@code null} = the server default). + * + * @return Generator + */ + public function iterate(?ListOptions $options = null, ?RunListOptions $filter = null, ?int $chunkSize = null): Generator + { + $options ??= new ListOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit), $filter), + ); + } } diff --git a/src/Resource/ScheduleCollectionClient.php b/src/Resource/ScheduleCollectionClient.php index b1a161f..637b57e 100644 --- a/src/Resource/ScheduleCollectionClient.php +++ b/src/Resource/ScheduleCollectionClient.php @@ -10,6 +10,7 @@ use Apify\Client\Model\PaginationList; use Apify\Client\Model\Schedule; use Apify\Client\Options\ListOptions; +use Generator; /** A client for the schedule collection ({@code GET/POST /v2/schedules}). */ final class ScheduleCollectionClient @@ -34,6 +35,24 @@ public function list(?ListOptions $options = null): PaginationList return $this->ctx->listResource('', $params, static fn (array $d) => new Schedule($d)); } + /** + * Lazily iterates over the account's schedules, fetching pages on demand. The options' + * {@code limit} caps the total number of schedules yielded across all pages ({@code null} = all); + * {@code $chunkSize} is the per-page size ({@code null} = the server default). + * + * @return Generator + */ + public function iterate(?ListOptions $options = null, ?int $chunkSize = null): Generator + { + $options ??= new ListOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit)), + ); + } + /** * Creates a new schedule. * diff --git a/src/Resource/StoreCollectionClient.php b/src/Resource/StoreCollectionClient.php index 9b12370..4acdd7f 100644 --- a/src/Resource/StoreCollectionClient.php +++ b/src/Resource/StoreCollectionClient.php @@ -36,25 +36,21 @@ public function list(?StoreListOptions $options = null): PaginationList } /** - * Lazily iterates over all Store Actors matching the options, fetching pages on demand. The - * options' {@code limit} (if set) is used as the per-page size. + * Lazily iterates over Store Actors matching the options, fetching pages on demand. + * + * The options' {@code limit} caps the total number of Actors yielded across all pages ({@code + * null} = all); {@code $chunkSize} is the per-page size ({@code null} = the server default). * * @return Generator */ - public function iterate(?StoreListOptions $options = null): Generator + public function iterate(?StoreListOptions $options = null, ?int $chunkSize = null): Generator { $options ??= new StoreListOptions(); - $offset = $options->offset ?? 0; - while (true) { - $page = $this->list($options->withOffset($offset)); - $items = $page->getItems(); - foreach ($items as $item) { - yield $item; - } - $offset += count($items); - if ($items === [] || $offset >= $page->getTotal()) { - return; - } - } + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit)), + ); } } diff --git a/src/Resource/TaskCollectionClient.php b/src/Resource/TaskCollectionClient.php index 4473a42..c0bd8c3 100644 --- a/src/Resource/TaskCollectionClient.php +++ b/src/Resource/TaskCollectionClient.php @@ -10,6 +10,7 @@ use Apify\Client\Model\PaginationList; use Apify\Client\Model\Task; use Apify\Client\Options\ListOptions; +use Generator; /** A client for the Actor task collection ({@code GET/POST /v2/actor-tasks}). */ final class TaskCollectionClient @@ -34,6 +35,24 @@ public function list(?ListOptions $options = null): PaginationList return $this->ctx->listResource('', $params, static fn (array $d) => new Task($d)); } + /** + * Lazily iterates over the account's tasks, fetching pages on demand. The options' {@code limit} + * caps the total number of tasks yielded across all pages ({@code null} = all); {@code $chunkSize} + * is the per-page size ({@code null} = the server default). + * + * @return Generator + */ + public function iterate(?ListOptions $options = null, ?int $chunkSize = null): Generator + { + $options ??= new ListOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit)), + ); + } + /** * Creates a new task. * diff --git a/src/Resource/WebhookDispatchCollectionClient.php b/src/Resource/WebhookDispatchCollectionClient.php index a337db8..e4a6e0e 100644 --- a/src/Resource/WebhookDispatchCollectionClient.php +++ b/src/Resource/WebhookDispatchCollectionClient.php @@ -10,6 +10,7 @@ use Apify\Client\Model\PaginationList; use Apify\Client\Model\WebhookDispatch; use Apify\Client\Options\ListOptions; +use Generator; /** * A client for a webhook dispatch collection: the account-wide collection ({@code GET @@ -36,4 +37,22 @@ public function list(?ListOptions $options = null): PaginationList ($options ?? new ListOptions())->appendTo($params); return $this->ctx->listResource('', $params, static fn (array $d) => new WebhookDispatch($d)); } + + /** + * Lazily iterates over webhook dispatches, fetching pages on demand. The options' {@code limit} + * caps the total number of dispatches yielded across all pages ({@code null} = all); + * {@code $chunkSize} is the per-page size ({@code null} = the server default). + * + * @return Generator + */ + public function iterate(?ListOptions $options = null, ?int $chunkSize = null): Generator + { + $options ??= new ListOptions(); + return ResourceContext::paginateOffset( + $options->offset ?? 0, + $options->limit, + $chunkSize, + fn (int $offset, ?int $pageLimit) => $this->list($options->withPagination($offset, $pageLimit)), + ); + } } diff --git a/src/Version.php b/src/Version.php index 48b2265..02b225d 100644 --- a/src/Version.php +++ b/src/Version.php @@ -17,13 +17,13 @@ final class Version * The semantic version of this client library (see https://semver.org/). * Changes to the public interface other than additive ones are considered breaking changes. */ - public const CLIENT_VERSION = '0.2.0'; + public const CLIENT_VERSION = '0.3.3'; /** * The version of the Apify OpenAPI specification this client was generated and verified * against. Corresponds to the {@code info.version} field of the Apify OpenAPI document. */ - public const API_SPEC_VERSION = 'v2-2026-07-08T143931Z'; + public const API_SPEC_VERSION = 'v2-2026-07-10T105921Z'; private function __construct() { diff --git a/tests/Examples/IterateStore.php b/tests/Examples/IterateStore.php index d860bf4..1cbea2e 100644 --- a/tests/Examples/IterateStore.php +++ b/tests/Examples/IterateStore.php @@ -13,7 +13,9 @@ final class IterateStore public static function run(ApifyClient $client): void { $shown = 0; - foreach ($client->store()->iterate(new StoreListOptions(limit: 10)) as $item) { + // The second argument is the per-page (chunk) size; the iterator fetches pages lazily as we + // consume items. StoreListOptions::limit (unset here) would cap the total across all pages. + foreach ($client->store()->iterate(new StoreListOptions(), 10) as $item) { echo $item->getName() . PHP_EOL; if (++$shown >= 5) { break; diff --git a/tests/Integration/ActorIntegrationTest.php b/tests/Integration/ActorIntegrationTest.php index 65b3b5a..c0e5ecc 100644 --- a/tests/Integration/ActorIntegrationTest.php +++ b/tests/Integration/ActorIntegrationTest.php @@ -74,6 +74,71 @@ public function testActorVersionCrudFlow(): void } } + public function testIterateActors(): void + { + $client = $this->requireClient(); + $ids = []; + for ($i = 0; $i < 3; $i++) { + $ids[] = (string) $client->actors()->create(self::minimalActor(self::uniqueName('iter')))->getId(); + } + try { + $seen = []; + // chunkSize=2 forces multi-page iteration across at least the three created Actors. + foreach ($client->actors()->iterate(new ActorListOptions(my: true), 2) as $actor) { + $seen[(string) $actor->getId()] = true; + } + foreach ($ids as $id) { + self::assertArrayHasKey($id, $seen, "iterate() did not yield created Actor $id"); + } + } finally { + foreach ($ids as $id) { + $client->actor($id)->delete(); + } + } + } + + public function testIterateActorVersions(): void + { + $client = $this->requireClient(); + $created = $client->actors()->create(self::minimalActor(self::uniqueName('iter-ver'))); + try { + $actor = $client->actor((string) $created->getId()); + $actor->versions()->create([ + 'versionNumber' => '0.1', + 'sourceType' => 'SOURCE_FILES', + 'buildTag' => 'latest', + 'sourceFiles' => [], + ]); + $seen = []; + foreach ($actor->versions()->iterate(null, 1) as $version) { + $seen[(string) $version->getVersionNumber()] = true; + } + self::assertArrayHasKey('0.0', $seen); + self::assertArrayHasKey('0.1', $seen); + } finally { + $client->actor((string) $created->getId())->delete(); + } + } + + public function testIterateActorEnvVars(): void + { + $client = $this->requireClient(); + $created = $client->actors()->create(self::minimalActor(self::uniqueName('iter-env'))); + try { + $version = $client->actor((string) $created->getId())->version('0.0'); + $version->envVars()->create(new ActorEnvVar('ITER_VAR_1', 'v1')); + $version->envVars()->create(new ActorEnvVar('ITER_VAR_2', 'v2')); + $seen = []; + foreach ($version->envVars()->iterate(1) as $envVar) { + $seen[(string) $envVar->getName()] = true; + } + self::assertArrayHasKey('ITER_VAR_1', $seen); + self::assertArrayHasKey('ITER_VAR_2', $seen); + } finally { + $client->actor((string) $created->getId())->delete(); + } + } + public function testValidateInput(): void { $client = $this->requireClient(); diff --git a/tests/Integration/ActorRunIntegrationTest.php b/tests/Integration/ActorRunIntegrationTest.php index 3202cbd..5ff8182 100644 --- a/tests/Integration/ActorRunIntegrationTest.php +++ b/tests/Integration/ActorRunIntegrationTest.php @@ -20,6 +20,23 @@ public function testListRuns(): void self::assertGreaterThanOrEqual(count($page->getItems()), $page->getTotal()); } + public function testIterateRuns(): void + { + $client = $this->requireClient(); + // Ensure at least one run exists for this account, then iterate with a small total cap and a + // page size that forces multi-page paging. Runs are shared account state, so the test asserts + // the cap and shape rather than an exact set, keeping it parallel-safe. + $client->actor('apify/hello-world')->call(null, null, 120); + $count = 0; + foreach ($client->runs()->iterate(new ListOptions(limit: 3), new RunListOptions(), 2) as $run) { + self::assertNotNull($run->getId()); + self::assertNotSame('', $run->getId()); + $count++; + } + self::assertGreaterThanOrEqual(1, $count); + self::assertLessThanOrEqual(3, $count, 'the total-item cap (limit) must bound iteration'); + } + public function testRunActorAndReadOutputs(): void { $client = $this->requireClient(); diff --git a/tests/Integration/BuildIntegrationTest.php b/tests/Integration/BuildIntegrationTest.php index c80f2f1..9f6b292 100644 --- a/tests/Integration/BuildIntegrationTest.php +++ b/tests/Integration/BuildIntegrationTest.php @@ -18,6 +18,25 @@ public function testListBuilds(): void self::assertGreaterThanOrEqual(count($page->getItems()), $page->getTotal()); } + public function testIterateBuilds(): void + { + $client = $this->requireClient(); + $created = $client->actors()->create(self::minimalActor(self::uniqueName('iter-build'))); + try { + $actor = $client->actor((string) $created->getId()); + $build = $actor->build('0.0', new ActorBuildOptions()); + $client->build((string) $build->getId())->waitForFinish(300); + // Iterate the Actor's builds (scoped, so the created build is the only expected entry). + $seen = []; + foreach ($actor->builds()->iterate(new ListOptions(), 1) as $b) { + $seen[(string) $b->getId()] = true; + } + self::assertArrayHasKey((string) $build->getId(), $seen, 'iterate() did not yield the created build'); + } finally { + $client->actor((string) $created->getId())->delete(); + } + } + public function testBuildActorFlow(): void { $client = $this->requireClient(); diff --git a/tests/Integration/DatasetIntegrationTest.php b/tests/Integration/DatasetIntegrationTest.php index 095809d..638bbde 100644 --- a/tests/Integration/DatasetIntegrationTest.php +++ b/tests/Integration/DatasetIntegrationTest.php @@ -33,6 +33,59 @@ public function testGetDataset(): void } } + public function testIterateDatasets(): void + { + $client = $this->requireClient(); + $ids = []; + for ($i = 0; $i < 3; $i++) { + $ids[] = (string) $client->datasets()->getOrCreate(self::uniqueName('iter-ds'))->getId(); + } + try { + $seen = []; + foreach ($client->datasets()->iterate(new StorageListOptions(desc: true), 2) as $dataset) { + $seen[(string) $dataset->getId()] = true; + } + foreach ($ids as $id) { + self::assertArrayHasKey($id, $seen, "iterate() did not yield created dataset $id"); + } + } finally { + foreach ($ids as $id) { + $client->dataset($id)->delete(); + } + } + } + + public function testIterateDatasetItems(): void + { + $client = $this->requireClient(); + $ds = $client->datasets()->getOrCreate(self::uniqueName('iter-items')); + try { + $dataset = $client->dataset((string) $ds->getId()); + $dataset->pushItems([['n' => 0], ['n' => 1], ['n' => 2], ['n' => 3], ['n' => 4]]); + + // The dataset's item total is computed asynchronously and can briefly lag a write. + // iterateItems() pages by the reported total (matching the reference client), so wait for + // the count to settle before iterating; otherwise a stale total would stop it early. + $deadline = microtime(true) + 30.0; + while ( + $dataset->listItems(new DatasetListItemsOptions())->getTotal() < 5 + && microtime(true) < $deadline + ) { + usleep(500_000); + } + + $values = []; + // chunkSize=2 across 5 items => three pages (2, 2, 1). + foreach ($dataset->iterateItems(new DatasetListItemsOptions(), 2) as $item) { + $values[] = $item['n']; + } + sort($values); + self::assertSame([0, 1, 2, 3, 4], $values); + } finally { + $client->dataset((string) $ds->getId())->delete(); + } + } + public function testDatasetCrudFlow(): void { $client = $this->requireClient(); diff --git a/tests/Integration/KeyValueStoreIntegrationTest.php b/tests/Integration/KeyValueStoreIntegrationTest.php index e360728..dadca74 100644 --- a/tests/Integration/KeyValueStoreIntegrationTest.php +++ b/tests/Integration/KeyValueStoreIntegrationTest.php @@ -33,6 +33,52 @@ public function testGetKeyValueStore(): void } } + public function testIterateKeyValueStores(): void + { + $client = $this->requireClient(); + $ids = []; + for ($i = 0; $i < 3; $i++) { + $ids[] = (string) $client->keyValueStores()->getOrCreate(self::uniqueName('iter-kvs'))->getId(); + } + try { + $seen = []; + foreach ($client->keyValueStores()->iterate(new StorageListOptions(desc: true), 2) as $store) { + $seen[(string) $store->getId()] = true; + } + foreach ($ids as $id) { + self::assertArrayHasKey($id, $seen, "iterate() did not yield created store $id"); + } + } finally { + foreach ($ids as $id) { + $client->keyValueStore($id)->delete(); + } + } + } + + public function testIterateKeys(): void + { + $client = $this->requireClient(); + $store = $client->keyValueStores()->getOrCreate(self::uniqueName('iter-keys')); + try { + $kvs = $client->keyValueStore((string) $store->getId()); + $expected = []; + for ($i = 0; $i < 5; $i++) { + $key = sprintf('key-%02d', $i); + $kvs->setRecordJson($key, ['n' => $i]); + $expected[] = $key; + } + $seen = []; + // limit as a total cap of 5; the store's cursor pagination threads exclusiveStartKey. + foreach ($kvs->iterateKeys(new ListKeysOptions(limit: 5)) as $key) { + $seen[] = (string) $key->getKey(); + } + sort($seen); + self::assertSame($expected, $seen); + } finally { + $client->keyValueStore((string) $store->getId())->delete(); + } + } + public function testRecordKeyWithSpecialChars(): void { $client = $this->requireClient(); diff --git a/tests/Integration/RequestQueueIntegrationTest.php b/tests/Integration/RequestQueueIntegrationTest.php index 6c037e2..16968d0 100644 --- a/tests/Integration/RequestQueueIntegrationTest.php +++ b/tests/Integration/RequestQueueIntegrationTest.php @@ -33,6 +33,28 @@ public function testGetRequestQueue(): void } } + public function testIterateRequestQueues(): void + { + $client = $this->requireClient(); + $ids = []; + for ($i = 0; $i < 3; $i++) { + $ids[] = (string) $client->requestQueues()->getOrCreate(self::uniqueName('iter-rq'))->getId(); + } + try { + $seen = []; + foreach ($client->requestQueues()->iterate(new StorageListOptions(desc: true), 2) as $queue) { + $seen[(string) $queue->getId()] = true; + } + foreach ($ids as $id) { + self::assertArrayHasKey($id, $seen, "iterate() did not yield created queue $id"); + } + } finally { + foreach ($ids as $id) { + $client->requestQueue($id)->delete(); + } + } + } + public function testRequestQueueCrudFlow(): void { $client = $this->requireClient(); diff --git a/tests/Integration/ScheduleIntegrationTest.php b/tests/Integration/ScheduleIntegrationTest.php index 3693abd..456c6cf 100644 --- a/tests/Integration/ScheduleIntegrationTest.php +++ b/tests/Integration/ScheduleIntegrationTest.php @@ -44,6 +44,28 @@ public function testGetSchedule(): void } } + public function testIterateSchedules(): void + { + $client = $this->requireClient(); + $ids = []; + for ($i = 0; $i < 3; $i++) { + $ids[] = (string) $client->schedules()->create(self::scheduleDef(self::uniqueName('iter-sch')))->getId(); + } + try { + $seen = []; + foreach ($client->schedules()->iterate(new ListOptions(desc: true), 2) as $schedule) { + $seen[(string) $schedule->getId()] = true; + } + foreach ($ids as $id) { + self::assertArrayHasKey($id, $seen, "iterate() did not yield created schedule $id"); + } + } finally { + foreach ($ids as $id) { + $client->schedule($id)->delete(); + } + } + } + public function testScheduleCrudFlow(): void { $client = $this->requireClient(); diff --git a/tests/Integration/StoreIntegrationTest.php b/tests/Integration/StoreIntegrationTest.php index 3d633ba..bb42d2d 100644 --- a/tests/Integration/StoreIntegrationTest.php +++ b/tests/Integration/StoreIntegrationTest.php @@ -19,7 +19,9 @@ public function testIterateStore(): void { $client = $this->requireClient(); $count = 0; - foreach ($client->store()->iterate(new StoreListOptions(limit: 5)) as $item) { + // chunkSize=5 is the per-page size; with no limit the iterator keeps fetching pages until we + // break, proving pagination is followed across more than two pages. + foreach ($client->store()->iterate(new StoreListOptions(), 5) as $item) { self::assertNotNull($item->getId()); self::assertNotSame('', $item->getId()); if (++$count >= 12) { @@ -28,4 +30,16 @@ public function testIterateStore(): void } self::assertGreaterThanOrEqual(12, $count, 'expected to iterate at least 12 store actors'); } + + public function testIterateStoreRespectsTotalLimit(): void + { + $client = $this->requireClient(); + $count = 0; + // limit is a total-item cap across all pages: iteration must stop at 3 even with tiny pages. + foreach ($client->store()->iterate(new StoreListOptions(limit: 3), 1) as $item) { + self::assertNotNull($item->getId()); + $count++; + } + self::assertSame(3, $count, 'limit must cap the total number of iterated items'); + } } diff --git a/tests/Integration/TaskIntegrationTest.php b/tests/Integration/TaskIntegrationTest.php index eb3a8b2..58601de 100644 --- a/tests/Integration/TaskIntegrationTest.php +++ b/tests/Integration/TaskIntegrationTest.php @@ -44,6 +44,28 @@ public function testGetTask(): void } } + public function testIterateTasks(): void + { + $client = $this->requireClient(); + $ids = []; + for ($i = 0; $i < 3; $i++) { + $ids[] = (string) $client->tasks()->create(self::taskDef(self::uniqueName('iter-task')))->getId(); + } + try { + $seen = []; + foreach ($client->tasks()->iterate(new ListOptions(desc: true), 2) as $task) { + $seen[(string) $task->getId()] = true; + } + foreach ($ids as $id) { + self::assertArrayHasKey($id, $seen, "iterate() did not yield created task $id"); + } + } finally { + foreach ($ids as $id) { + $client->task($id)->delete(); + } + } + } + public function testTaskCrudFlow(): void { $client = $this->requireClient(); diff --git a/tests/Integration/WebhookIntegrationTest.php b/tests/Integration/WebhookIntegrationTest.php index 40b95e2..3b82406 100644 --- a/tests/Integration/WebhookIntegrationTest.php +++ b/tests/Integration/WebhookIntegrationTest.php @@ -39,6 +39,45 @@ public function testListWebhookDispatches(): void self::assertGreaterThanOrEqual(count($page->getItems()), $page->getTotal()); } + public function testIterateWebhooks(): void + { + $client = $this->requireClient(); + $ids = []; + for ($i = 0; $i < 3; $i++) { + $ids[] = (string) $client->webhooks()->create(self::webhookDef('https://example.com/iter-' . $i))->getId(); + } + try { + $seen = []; + foreach ($client->webhooks()->iterate(new ListOptions(desc: true), 2) as $webhook) { + $seen[(string) $webhook->getId()] = true; + } + foreach ($ids as $id) { + self::assertArrayHasKey($id, $seen, "iterate() did not yield created webhook $id"); + } + } finally { + foreach ($ids as $id) { + $client->webhook($id)->delete(); + } + } + } + + public function testIterateWebhookDispatches(): void + { + $client = $this->requireClient(); + $wh = $client->webhooks()->create(self::webhookDef('https://example.com/dispatch-iter')); + try { + // test() synchronously creates an ad-hoc dispatch listed under the webhook. + $dispatch = $client->webhook((string) $wh->getId())->test(); + $seen = []; + foreach ($client->webhook((string) $wh->getId())->dispatches()->iterate(new ListOptions(), 2) as $d) { + $seen[(string) $d->getId()] = true; + } + self::assertArrayHasKey((string) $dispatch->getId(), $seen, 'iterate() did not yield the test dispatch'); + } finally { + $client->webhook((string) $wh->getId())->delete(); + } + } + public function testGetWebhook(): void { $client = $this->requireClient(); diff --git a/tests/Unit/BatchAddRequestsTest.php b/tests/Unit/BatchAddRequestsTest.php index a12d342..fdba507 100644 --- a/tests/Unit/BatchAddRequestsTest.php +++ b/tests/Unit/BatchAddRequestsTest.php @@ -185,4 +185,31 @@ public function testOversizedSingleRequestThrows(): void $this->expectException(InvalidArgumentException::class); $this->client(new MockTransport())->requestQueue('q1')->batchAddRequests($requests); } + + public function testOversizedRequestInMiddleOfLargeBatchThrowsBeforeAnyCall(): void + { + // 30 small requests (would be two chunks of 25 + 5) with an oversized request at index 27, + // i.e. only reached by the SECOND chunk. Validation must run entirely up front, so the whole + // call throws before the first (valid) chunk is ever POSTed — leaving the queue unmutated. + $huge = str_repeat('x', 10 * 1024 * 1024); // > 9 MiB on its own + $requests = []; + for ($i = 0; $i < 30; $i++) { + $request = new RequestQueueRequest('https://x/' . $i, 'u' . $i); + if ($i === 27) { + $request->setUserData(['blob' => $huge]); + } + $requests[] = $request; + } + + $transport = new MockTransport(); + try { + $this->client($transport)->requestQueue('q1')->batchAddRequests($requests); + self::fail('expected InvalidArgumentException'); + } catch (InvalidArgumentException $e) { + self::assertStringContainsString('index 27', $e->getMessage()); + self::assertStringContainsString('maximum payload size', $e->getMessage()); + } + // The crucial assertion: no chunk was POSTed before the oversized request was rejected. + self::assertSame(0, $transport->callCount()); + } } diff --git a/tests/Unit/IterationTest.php b/tests/Unit/IterationTest.php new file mode 100644 index 0000000..ada2485 --- /dev/null +++ b/tests/Unit/IterationTest.php @@ -0,0 +1,253 @@ +> $items + */ + private static function page(array $items, int $total, int $offset): string + { + return Json::encode(['data' => [ + 'items' => $items, + 'total' => $total, + 'offset' => $offset, + 'limit' => count($items), + 'count' => count($items), + 'desc' => false, + ]]); + } + + /** + * @param int ...$ids + * @return list> + */ + private static function actors(int ...$ids): array + { + return array_map(static fn (int $id): array => ['id' => "a$id", 'name' => "actor$id"], $ids); + } + + public function testSinglePageStopsAfterOneRequest(): void + { + // total equals the number of returned items => no second request. + $transport = (new MockTransport())->queueResponse(200, self::page(self::actors(1, 2, 3), 3, 0)); + $ids = []; + foreach ($this->client($transport)->actors()->iterate() as $actor) { + $ids[] = $actor->getId(); + } + self::assertSame(['a1', 'a2', 'a3'], $ids); + self::assertSame(1, $transport->callCount()); + } + + public function testOverReportedTotalTerminates(): void + { + // The API claims 10 items but only 3 exist; the iterator must stop, not loop forever. + $transport = (new MockTransport()) + ->queueResponse(200, self::page(self::actors(1, 2, 3), 10, 0)) + ->queueResponse(200, self::page([], 10, 3)); // the follow-up page comes back empty + $ids = []; + foreach ($this->client($transport)->actors()->iterate() as $actor) { + $ids[] = $actor->getId(); + } + self::assertSame(['a1', 'a2', 'a3'], $ids); + // One extra fetch is made (remaining > 0) before the empty page stops iteration. + self::assertSame(2, $transport->callCount()); + } + + public function testLimitIsTotalCapAndChunkSizeIsPageSize(): void + { + // limit=5 total across all pages; chunkSize=2 per page => pages of 2, 2, 1. + $transport = (new MockTransport()) + ->queueResponse(200, self::page(self::actors(1, 2), 100, 0)) + ->queueResponse(200, self::page(self::actors(3, 4), 100, 2)) + ->queueResponse(200, self::page(self::actors(5), 100, 4)); + $ids = []; + foreach ($this->client($transport)->actors()->iterate(new ActorListOptions(limit: 5), 2) as $actor) { + $ids[] = $actor->getId(); + } + self::assertSame(['a1', 'a2', 'a3', 'a4', 'a5'], $ids); + self::assertSame(3, $transport->callCount()); + + // First page requests min(limit=5, chunkSize=2)=2; later pages carry the running offset. + $uris = array_map(static fn ($r) => (string) $r->getUri(), $transport->received); + self::assertStringContainsString('offset=0', $uris[0]); + self::assertStringContainsString('limit=2', $uris[0]); + self::assertStringContainsString('offset=2', $uris[1]); + self::assertStringContainsString('offset=4', $uris[2]); + // The last page is capped by the remaining total (1), not the chunk size (2). + self::assertStringContainsString('limit=1', $uris[2]); + } + + public function testLimitCapStopsBeforeExhaustingPages(): void + { + // limit=2 with a big first page: only 2 items are yielded and no second request is made. + $transport = (new MockTransport()) + ->queueResponse(200, self::page(self::actors(1, 2), 100, 0)); + $ids = []; + foreach ($this->client($transport)->store()->iterate(new StoreListOptions(limit: 2)) as $item) { + $ids[] = $item->getId(); + } + self::assertSame(['a1', 'a2'], $ids); + self::assertSame(1, $transport->callCount()); + self::assertStringContainsString('limit=2', (string) $transport->received[0]->getUri()); + } + + public function testDatasetIterateItemsPagesViaHeaders(): void + { + // The dataset-items endpoint returns a bare array and reports pagination via headers. + $transport = (new MockTransport()) + ->queueResponse(200, Json::encode([['n' => 1], ['n' => 2]]), [ + 'X-Apify-Pagination-Total' => '3', + 'X-Apify-Pagination-Offset' => '0', + 'X-Apify-Pagination-Limit' => '2', + ]) + ->queueResponse(200, Json::encode([['n' => 3]]), [ + 'X-Apify-Pagination-Total' => '3', + 'X-Apify-Pagination-Offset' => '2', + 'X-Apify-Pagination-Limit' => '2', + ]); + $values = []; + foreach ($this->client($transport)->dataset('ds')->iterateItems(null, 2) as $item) { + $values[] = $item['n']; + } + self::assertSame([1, 2, 3], $values); + self::assertSame(2, $transport->callCount()); + self::assertStringContainsString('offset=2', (string) $transport->received[1]->getUri()); + } + + public function testDatasetIterateItemsPreservesFilters(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, Json::encode([['n' => 1]]), [ + 'X-Apify-Pagination-Total' => '1', + 'X-Apify-Pagination-Offset' => '0', + 'X-Apify-Pagination-Limit' => '1', + ]); + $it = $this->client($transport)->dataset('ds')->iterateItems(new DatasetListItemsOptions(fields: ['n'], clean: true)); + iterator_to_array($it); + $uri = (string) $transport->received[0]->getUri(); + self::assertStringContainsString('fields=n', $uri); + self::assertStringContainsString('clean=1', $uri); + } + + /** Builds a cursor-paged request-queue list envelope ({@code {"data": {items, nextCursor}}}). */ + private static function requestsPage(?string $nextCursor, string ...$ids): string + { + return Json::encode(['data' => [ + 'items' => array_map(static fn (string $id): array => ['id' => $id, 'url' => "https://e/$id"], $ids), + 'count' => count($ids), + 'limit' => 1000, + 'nextCursor' => $nextCursor, + ]]); + } + + public function testPaginateRequestsLimitZeroIteratesAll(): void + { + // limit=0 is a total cap of "unbounded": iterate every page, and never forward limit=0 as a + // per-page cap (which would short-circuit the iteration after a single page). + $transport = (new MockTransport()) + ->queueResponse(200, self::requestsPage('cursor2', 'r1', 'r2')) + ->queueResponse(200, self::requestsPage(null, 'r3')); + $ids = []; + foreach ($this->client($transport)->requestQueue('rq')->paginateRequests(new PaginateRequestsOptions(limit: 0)) as $request) { + $ids[] = $request->getId(); + } + self::assertSame(['r1', 'r2', 'r3'], $ids); + self::assertSame(2, $transport->callCount()); + self::assertStringNotContainsString('limit=0', (string) $transport->received[0]->getUri()); + } + + private static function keysPage(bool $isTruncated, ?string $nextKey, string ...$keys): string + { + return Json::encode(['data' => [ + 'items' => array_map(static fn (string $k): array => ['key' => $k, 'size' => 1], $keys), + 'count' => count($keys), + 'limit' => 1000, + 'isTruncated' => $isTruncated, + 'exclusiveStartKey' => null, + 'nextExclusiveStartKey' => $nextKey, + ]]); + } + + public function testIterateKeysThreadsCursor(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::keysPage(true, 'k2', 'k1', 'k2')) + ->queueResponse(200, self::keysPage(false, null, 'k3')); + $keys = []; + foreach ($this->client($transport)->keyValueStore('kvs')->iterateKeys() as $key) { + $keys[] = $key->getKey(); + } + self::assertSame(['k1', 'k2', 'k3'], $keys); + self::assertSame(2, $transport->callCount()); + // The second request must carry the first page's nextExclusiveStartKey. + self::assertStringContainsString('exclusiveStartKey=k2', (string) $transport->received[1]->getUri()); + } + + public function testIterateKeysStopsWhenNotTruncated(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::keysPage(false, null, 'k1', 'k2')); + $keys = []; + foreach ($this->client($transport)->keyValueStore('kvs')->iterateKeys() as $key) { + $keys[] = $key->getKey(); + } + self::assertSame(['k1', 'k2'], $keys); + self::assertSame(1, $transport->callCount()); + } + + public function testIterateKeysRespectsTotalCap(): void + { + // limit=2 total: stop after two keys even though the first page is truncated. + $transport = (new MockTransport()) + ->queueResponse(200, self::keysPage(true, 'k2', 'k1', 'k2')); + $keys = []; + foreach ($this->client($transport)->keyValueStore('kvs')->iterateKeys(new ListKeysOptions(limit: 2)) as $key) { + $keys[] = $key->getKey(); + } + self::assertSame(['k1', 'k2'], $keys); + self::assertSame(1, $transport->callCount()); + self::assertStringContainsString('limit=2', (string) $transport->received[0]->getUri()); + } + + public function testIterateKeysLimitZeroIteratesAll(): void + { + // limit=0 is a total cap of "unbounded": iterate every page, and never forward limit=0 as a + // per-page cap (which would short-circuit the iteration after a single page). + $transport = (new MockTransport()) + ->queueResponse(200, self::keysPage(true, 'k2', 'k1', 'k2')) + ->queueResponse(200, self::keysPage(false, null, 'k3')); + $keys = []; + foreach ($this->client($transport)->keyValueStore('kvs')->iterateKeys(new ListKeysOptions(limit: 0)) as $key) { + $keys[] = $key->getKey(); + } + self::assertSame(['k1', 'k2', 'k3'], $keys); + self::assertSame(2, $transport->callCount()); + self::assertStringNotContainsString('limit=0', (string) $transport->received[0]->getUri()); + self::assertStringNotContainsString('limit=', (string) $transport->received[0]->getUri()); + } +} diff --git a/tests/Unit/RequestShapeTest.php b/tests/Unit/RequestShapeTest.php index 7cf1eb4..b451fd8 100644 --- a/tests/Unit/RequestShapeTest.php +++ b/tests/Unit/RequestShapeTest.php @@ -52,11 +52,22 @@ public function testMetamorphSendsTargetActorIdAndInput(): void self::assertSame('POST', $request->getMethod()); $uri = (string) $request->getUri(); self::assertStringContainsString('/actor-runs/run1/metamorph', $uri); - self::assertStringContainsString('targetActorId=apify%2Fother', $uri); + self::assertStringContainsString('targetActorId=apify~other', $uri); self::assertStringContainsString('build=latest', $uri); self::assertSame(['x' => 1], Json::decode((string) $request->getBody())); } + public function testMetamorphNormalizesSlashFormTargetActorId(): void + { + $transport = (new MockTransport())->queueResponse(200, Json::encode(['data' => ['id' => 'r']])); + $this->client($transport)->run('run1')->metamorph('username/actor-name'); + + $uri = (string) $transport->lastRequest()->getUri(); + // The first `/` must be normalized to `~` (matching the JS reference), not percent-encoded. + self::assertStringContainsString('targetActorId=username~actor-name', $uri); + self::assertStringNotContainsString('username%2Factor-name', $uri); + } + public function testResurrectSendsOptions(): void { $transport = (new MockTransport())->queueResponse(200, Json::encode(['data' => ['id' => 'r']]));