Feat/sdk targeted list limits (list-limits part of issue #1107)

README.md

@@ -60,5 +60,6 @@ Check out the [examples](./examples) directory for complete working examples:
 | [scheduling.ts](./examples/scheduling.ts) | Set up robot schedules to execute runs |
 | [webhooks.ts](./examples/webhooks.ts) | Configure webhook notifications |
 | [robot-management.ts](./examples/robot-management.ts) | CRUD operations for robots |
+| [list-limit.ts](./examples/list-limit.ts) | Change a robot's limit without resending its workflow |
 | [complete-workflow.ts](./examples/complete-workflow.ts) | Create a robot combining multiple features |
 

examples/list-limit.ts

@@ -0,0 +1,113 @@
+/**
+ * List Limit Example
+ *
+ * This example demonstrates:
+ * - Changing a robot's limit without resending its workflow
+ * - Doing so for extract, crawl, and search robots
+ * - Targeting a specific list on a robot that has several
+ *
+ * Only the limit is sent to the backend. Selectors, pagination, crawl depth,
+ * search filters, and everything else are left exactly as they were.
+ */
+
+import 'dotenv/config';
+import { Extract, Crawl, Search, Client, Robot } from 'maxun-sdk';
+
+const config = {
+  apiKey: process.env.MAXUN_API_KEY!,
+  baseUrl: process.env.MAXUN_BASE_URL,
+};
+
+/** The stored config object holding the limit, for a given action. */
+function configOf(robot: Robot, action: string) {
+  return (robot.getData().recording?.workflow || [])
+    .flatMap((pair: any) => pair.what || [])
+    .filter((a: any) => a.action === action)
+    .flatMap((a: any) => a.args || [])
+    .find((arg: any) => arg && typeof arg === 'object' && 'limit' in arg);
+}
+
+/** Prints the limit alongside the settings that sit next to it. */
+function describe(label: string, robot: Robot, action: string, neighbours: string[]) {
+  const cfg = configOf(robot, action) || {};
+  const rest = neighbours.map((k) => `${k}=${JSON.stringify(cfg[k])}`).join(', ');
+  console.log(`  ${label.padEnd(7)} limit=${String(cfg.limit).padEnd(4)} ${rest}`);
+}
+
+async function main() {
+  const extractor = new Extract(config);
+
+  try {
+    // --- extract robot: how many items the list collects -----------------
+    console.log('\nExtract robot (scrapeList)');
+
+    const robot = await extractor
+      .create(`Books Scraper ${Date.now()}`)
+      .navigate('https://books.toscrape.com/')
+      .captureList({ selector: 'article.product_pod', maxItems: 10 });
+
+    describe('before', robot, 'scrapeList', ['listSelector']);
+    await robot.setListLimit(25);
+    describe('after', robot, 'scrapeList', ['listSelector']);
+
+    // --- crawl robot: how many pages it visits ---------------------------
+    console.log('\nCrawl robot (crawl)');
+
+    const crawler = await new Crawl(config).create(
+      `Site Crawler ${Date.now()}`,
+      'https://books.toscrape.com/',
+      { mode: 'domain', limit: 15, maxDepth: 2 }
+    );
+
+    describe('before', crawler, 'crawl', ['mode', 'maxDepth']);
+    await crawler.setListLimit(50);
+    describe('after', crawler, 'crawl', ['mode', 'maxDepth']);
+
+    // --- search robot: how many results it returns -----------------------
+    console.log('\nSearch robot (search)');
+
+    const searcher = await new Search(config).create(`Web Search ${Date.now()}`, {
+      query: 'web scraping',
+      mode: 'discover',
+      limit: 8,
+    });
+
+    describe('before', searcher, 'search', ['query', 'provider']);
+    await searcher.setListLimit(20);
+    describe('after', searcher, 'search', ['query', 'provider']);
+
+    /**
+     * setListLimit updates the first action it finds that carries a limit.
+     * For a robot with more than one list, use the client directly and name
+     * the position. Positions are assigned server-side, so read them from the
+     * robot rather than assuming them.
+     */
+    console.log('\nUpdating by explicit position');
+
+    const client = new Client(config);
+    const workflow = robot.getData().recording?.workflow || [];
+
+    workflow.forEach((pair: any, pairIndex: number) => {
+      (pair.what || []).forEach((action: any, actionIndex: number) => {
+        (action.args || []).forEach((arg: any, argIndex: number) => {
+          if (arg && typeof arg === 'object' && 'limit' in arg) {
+            console.log(
+              `  found ${action.action} limit=${arg.limit} at pair ${pairIndex}, action ${actionIndex}, arg ${argIndex}`
+            );
+          }
+        });
+      });
+    });
+
+    await client.updateListLimits(robot.id, [
+      { pairIndex: 0, actionIndex: 0, argIndex: 0, limit: 50 },
+    ]);
+
+    const updated = await extractor.getRobot(robot.id);
+    describe('after', updated, 'scrapeList', ['listSelector']);
+  } catch (error: any) {
+    console.error('Failed:', error.message);
+  }
+}
+
+main();

src/client/maxun-client.ts

@@ -22,6 +22,8 @@ import {
   CrawlOptions,
   SearchOptions,
   LlmOptions,
+  ListLimitUpdate, //added
+
 } from '../types';
 
 /**
@@ -165,6 +167,22 @@ export class Client {
     return response.data.data;
   }
 
+
+   /**
+   * Update one or more list limits without resending the whole workflow.
+   */
+  async updateListLimits(robotId: string, limits: ListLimitUpdate[]): Promise<RobotData> {
+    const response = await this.axios.put<ApiResponse<RobotData>>(
+      `/robots/${robotId}`,
+      { limits }
+    );
+    if (!response.data.data) {
+      throw new MaxunError(`Failed to update list limits for robot ${robotId}`);
+    }
+    return response.data.data;
+  }
+
+
   /**
    * Delete a robot
    */

src/robot/robot.ts

@@ -2,7 +2,7 @@
  * Robot class - represents a saved workflow that can be executed
  */
 
-import { RunResult, RobotData, ScheduleConfig, WebhookConfig, ExecutionOptions, Run } from '../types';
+import { RunResult, RobotData, ScheduleConfig, WebhookConfig, ExecutionOptions, Run, MaxunError } from '../types';
 import { Client } from '../client/maxun-client';
 
 export class Robot {
@@ -110,6 +110,42 @@ export class Robot {
     this.robotData = updated;
   }
 
+  /**
+   * Set the maximum number of items this robot collects.
+   *
+   * Applies to the three actions that carry a limit: `scrapeList` on extract
+   * robots, `crawl` on crawl robots, and `search` on search robots. The action
+   * is located automatically, so callers do not need to know its position in
+   * the workflow. Only the limit is sent; the rest of the workflow is untouched.
+   *
+   * @throws MaxunError if the robot has no action with a limit.
+   */
+  async setListLimit(limit: number): Promise<void> {
+    const LIMIT_ACTIONS = ['scrapeList', 'crawl', 'search'];
+    const workflow = this.robotData.recording?.workflow || [];
+
+    for (let p = 0; p < workflow.length; p++) {
+      const what = workflow[p].what || [];
+      for (let a = 0; a < what.length; a++) {
+        if (!LIMIT_ACTIONS.includes(what[a].action)) continue;
+        const args = what[a].args || [];
+        for (let g = 0; g < args.length; g++) {
+          const arg = args[g];
+          if (arg && typeof arg === 'object' && 'limit' in arg) {
+            this.robotData = await this.client.updateListLimits(this.id, [
+              { pairIndex: p, actionIndex: a, argIndex: g, limit },
+            ]);
+            return;
+          }
+        }
+      }
+    }
+
+    throw new MaxunError('This robot has no scrapeList, crawl, or search action with a limit to update.');
+  }
+
+
+
   /**
    * Get all webhooks for this robot
    */

src/types/index.ts

@@ -78,6 +78,20 @@ export interface RobotData {
   updatedAt?: string;
 }
 
+
+/**
+ * Coordinates of a single list limit within a robot's workflow,
+ * plus the new value to set.
+ */
+export interface ListLimitUpdate {
+  pairIndex: number;
+  actionIndex: number;
+  argIndex: number;
+  limit: number;
+}
+
+
+
 export interface Run {
   id: string;
   status: RunStatus;