fix(providers): send a numeric Firecrawl wait as the v2 waitFor delay

On the Firecrawl cloud endpoint, a wait_for_selector that is a plain
non-negative integer is now sent as the v2 waitFor delay in milliseconds,
capped at the v2 limit of 60000. Any other value still goes through the
wait action as a CSS selector. Self-hosted base URLs keep the v1 body.

The mapping moves into applyWaitForSelector() so firecrawlFetch does not
grow. Adds the changelog fragment for the v2 scrape migration (#19636).

Grafted from #19799 (numeric waitFor handling and changelog entry), with
the integer-only rule and the 60000 ms clamp from the review.

Co-authored-by: Rafa Martins <rafacpti23@users.noreply.github.com>
This commit is contained in:
diegosouzapw
2026-11-20 11:34:24 -03:00
parent 8d4ae3e0ec
commit 10cff9e8ca
3 changed files with 80 additions and 9 deletions

View File

@@ -0,0 +1 @@
- Move the Firecrawl cloud scrape call and the dashboard key probe from `/v1/scrape` to `/v2/scrape` before the 2026-12-10 v1 shutdown (#19636): a CSS `wait_for_selector` becomes a v2 `wait` action, a plain integer becomes the `waitFor` delay in milliseconds (capped at 60000), and `maxDepth` is no longer sent to the cloud endpoint. Self-hosted base URLs stay on `/v1/scrape` with the previous body.

View File

@@ -24,7 +24,8 @@ function getFirecrawlBaseUrl(credentials?: WebFetchCredentials): string {
const envBase = process.env.FIRECRAWL_BASE_URL?.trim();
if (envBase) return envBase.replace(/\/+$/, "");
const providerData = credentials?.providerSpecificData;
const credBase = typeof credentials?.baseUrl === "string" ? credentials.baseUrl : providerData?.baseUrl;
const credBase =
typeof credentials?.baseUrl === "string" ? credentials.baseUrl : providerData?.baseUrl;
if (typeof credBase === "string" && credBase.trim()) {
return credBase.trim().replace(/\/+$/, "");
}
@@ -58,6 +59,34 @@ function mapFormat(format: WebFetchFormat): string {
}
}
/** Firecrawl v2 scrape rejects a `waitFor` delay above this many milliseconds. */
const FIRECRAWL_V2_MAX_WAIT_MS = 60_000;
/**
* Map `wait_for_selector` onto the scrape body.
*
* v2 `waitFor` is an integer millisecond delay, not a selector: a plain
* non-negative integer becomes that delay (clamped to the v2 limit), anything
* else goes through the actions list, which waits until the element is visible.
* A self-hosted base URL still speaks v1, which takes the string in `waitFor`.
*/
function applyWaitForSelector(
requestBody: Record<string, unknown>,
waitForSelector: string,
isDefaultBaseUrl: boolean
): void {
if (!isDefaultBaseUrl) {
requestBody.waitFor = waitForSelector;
return;
}
const trimmed = waitForSelector.trim();
if (/^\d+$/.test(trimmed)) {
requestBody.waitFor = Math.min(Number(trimmed), FIRECRAWL_V2_MAX_WAIT_MS);
return;
}
requestBody.actions = [{ type: "wait", selector: waitForSelector }];
}
interface FirecrawlScrapeOptions {
url: string;
format: WebFetchFormat;
@@ -104,15 +133,8 @@ export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise<WebF
requestBody.maxDepth = depth;
}
// v2 waitFor is a millisecond delay, not a selector. A selector goes through
// the actions list, which waits until the element is visible. v1 takes the
// selector string directly in waitFor.
if (waitForSelector) {
if (isDefaultBaseUrl) {
requestBody.actions = [{ type: "wait", selector: waitForSelector }];
} else {
requestBody.waitFor = waitForSelector;
}
applyWaitForSelector(requestBody, waitForSelector, isDefaultBaseUrl);
}
const controller = new AbortController();

View File

@@ -184,6 +184,54 @@ test("firecrawlFetch keeps depth and selector off the v2 cloud scrape body", asy
}
});
async function captureCloudScrapeBody(waitForSelector: string) {
const originalFetch = globalThis.fetch;
let capturedBody: Record<string, unknown> = {};
globalThis.fetch = async (_url, init = {}) => {
capturedBody = JSON.parse(String((init as RequestInit).body ?? "{}"));
return new Response(JSON.stringify({ data: { markdown: "" } }), {
status: 200,
headers: { "content-type": "application/json" },
});
};
try {
await firecrawlFetch({
url: "https://example.com",
format: "markdown",
depth: 0,
waitForSelector,
includeMetadata: false,
credentials: { apiKey: "fc-key" },
});
} finally {
globalThis.fetch = originalFetch;
}
return capturedBody;
}
test("firecrawlFetch sends a numeric wait_for_selector as v2 waitFor milliseconds", async () => {
const body = await captureCloudScrapeBody("1500");
assert.equal(body.waitFor, 1500, "should set numeric waitFor ms");
assert.equal(body.actions, undefined, "should not set actions for a numeric delay");
});
test("firecrawlFetch clamps a numeric wait_for_selector to the v2 limit of 60000 ms", async () => {
const body = await captureCloudScrapeBody("90000");
assert.equal(body.waitFor, 60000, "v2 waitFor accepts at most 60000 ms");
assert.equal(body.actions, undefined);
});
test("firecrawlFetch keeps a non-integer wait_for_selector off the v2 waitFor field", async () => {
const body = await captureCloudScrapeBody("1.5");
assert.equal(body.waitFor, undefined, "v2 waitFor must be an integer");
assert.deepEqual(body.actions, [{ type: "wait", selector: "1.5" }]);
});
// ── #4692 regression: includeMetadata must NOT send invalid includeTags ────────
// Firecrawl returns metadata automatically in response.data.metadata. Sending
// includeTags with non-CSS-selector values ("og:title", "description") crashed