mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-21 15:05:56 +00:00
fix(providers): send a numeric Firecrawl wait as the v2 waitFor delay
On the Firecrawl cloud endpoint, a wait_for_selector that is a plain non-negative integer is now sent as the v2 waitFor delay in milliseconds, capped at the v2 limit of 60000. Any other value still goes through the wait action as a CSS selector. Self-hosted base URLs keep the v1 body. The mapping moves into applyWaitForSelector() so firecrawlFetch does not grow. Adds the changelog fragment for the v2 scrape migration (#19636). Grafted from #19799 (numeric waitFor handling and changelog entry), with the integer-only rule and the 60000 ms clamp from the review. Co-authored-by: Rafa Martins <rafacpti23@users.noreply.github.com>
This commit is contained in:
1
changelog.d/fixes/19665-firecrawl-v2-scrape-migration.md
Normal file
1
changelog.d/fixes/19665-firecrawl-v2-scrape-migration.md
Normal file
@@ -0,0 +1 @@
|
||||
- Move the Firecrawl cloud scrape call and the dashboard key probe from `/v1/scrape` to `/v2/scrape` before the 2026-12-10 v1 shutdown (#19636): a CSS `wait_for_selector` becomes a v2 `wait` action, a plain integer becomes the `waitFor` delay in milliseconds (capped at 60000), and `maxDepth` is no longer sent to the cloud endpoint. Self-hosted base URLs stay on `/v1/scrape` with the previous body.
|
||||
@@ -24,7 +24,8 @@ function getFirecrawlBaseUrl(credentials?: WebFetchCredentials): string {
|
||||
const envBase = process.env.FIRECRAWL_BASE_URL?.trim();
|
||||
if (envBase) return envBase.replace(/\/+$/, "");
|
||||
const providerData = credentials?.providerSpecificData;
|
||||
const credBase = typeof credentials?.baseUrl === "string" ? credentials.baseUrl : providerData?.baseUrl;
|
||||
const credBase =
|
||||
typeof credentials?.baseUrl === "string" ? credentials.baseUrl : providerData?.baseUrl;
|
||||
if (typeof credBase === "string" && credBase.trim()) {
|
||||
return credBase.trim().replace(/\/+$/, "");
|
||||
}
|
||||
@@ -58,6 +59,34 @@ function mapFormat(format: WebFetchFormat): string {
|
||||
}
|
||||
}
|
||||
|
||||
/** Firecrawl v2 scrape rejects a `waitFor` delay above this many milliseconds. */
|
||||
const FIRECRAWL_V2_MAX_WAIT_MS = 60_000;
|
||||
|
||||
/**
|
||||
* Map `wait_for_selector` onto the scrape body.
|
||||
*
|
||||
* v2 `waitFor` is an integer millisecond delay, not a selector: a plain
|
||||
* non-negative integer becomes that delay (clamped to the v2 limit), anything
|
||||
* else goes through the actions list, which waits until the element is visible.
|
||||
* A self-hosted base URL still speaks v1, which takes the string in `waitFor`.
|
||||
*/
|
||||
function applyWaitForSelector(
|
||||
requestBody: Record<string, unknown>,
|
||||
waitForSelector: string,
|
||||
isDefaultBaseUrl: boolean
|
||||
): void {
|
||||
if (!isDefaultBaseUrl) {
|
||||
requestBody.waitFor = waitForSelector;
|
||||
return;
|
||||
}
|
||||
const trimmed = waitForSelector.trim();
|
||||
if (/^\d+$/.test(trimmed)) {
|
||||
requestBody.waitFor = Math.min(Number(trimmed), FIRECRAWL_V2_MAX_WAIT_MS);
|
||||
return;
|
||||
}
|
||||
requestBody.actions = [{ type: "wait", selector: waitForSelector }];
|
||||
}
|
||||
|
||||
interface FirecrawlScrapeOptions {
|
||||
url: string;
|
||||
format: WebFetchFormat;
|
||||
@@ -104,15 +133,8 @@ export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise<WebF
|
||||
requestBody.maxDepth = depth;
|
||||
}
|
||||
|
||||
// v2 waitFor is a millisecond delay, not a selector. A selector goes through
|
||||
// the actions list, which waits until the element is visible. v1 takes the
|
||||
// selector string directly in waitFor.
|
||||
if (waitForSelector) {
|
||||
if (isDefaultBaseUrl) {
|
||||
requestBody.actions = [{ type: "wait", selector: waitForSelector }];
|
||||
} else {
|
||||
requestBody.waitFor = waitForSelector;
|
||||
}
|
||||
applyWaitForSelector(requestBody, waitForSelector, isDefaultBaseUrl);
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
|
||||
@@ -184,6 +184,54 @@ test("firecrawlFetch keeps depth and selector off the v2 cloud scrape body", asy
|
||||
}
|
||||
});
|
||||
|
||||
async function captureCloudScrapeBody(waitForSelector: string) {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let capturedBody: Record<string, unknown> = {};
|
||||
|
||||
globalThis.fetch = async (_url, init = {}) => {
|
||||
capturedBody = JSON.parse(String((init as RequestInit).body ?? "{}"));
|
||||
return new Response(JSON.stringify({ data: { markdown: "" } }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
await firecrawlFetch({
|
||||
url: "https://example.com",
|
||||
format: "markdown",
|
||||
depth: 0,
|
||||
waitForSelector,
|
||||
includeMetadata: false,
|
||||
credentials: { apiKey: "fc-key" },
|
||||
});
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
return capturedBody;
|
||||
}
|
||||
|
||||
test("firecrawlFetch sends a numeric wait_for_selector as v2 waitFor milliseconds", async () => {
|
||||
const body = await captureCloudScrapeBody("1500");
|
||||
|
||||
assert.equal(body.waitFor, 1500, "should set numeric waitFor ms");
|
||||
assert.equal(body.actions, undefined, "should not set actions for a numeric delay");
|
||||
});
|
||||
|
||||
test("firecrawlFetch clamps a numeric wait_for_selector to the v2 limit of 60000 ms", async () => {
|
||||
const body = await captureCloudScrapeBody("90000");
|
||||
|
||||
assert.equal(body.waitFor, 60000, "v2 waitFor accepts at most 60000 ms");
|
||||
assert.equal(body.actions, undefined);
|
||||
});
|
||||
|
||||
test("firecrawlFetch keeps a non-integer wait_for_selector off the v2 waitFor field", async () => {
|
||||
const body = await captureCloudScrapeBody("1.5");
|
||||
|
||||
assert.equal(body.waitFor, undefined, "v2 waitFor must be an integer");
|
||||
assert.deepEqual(body.actions, [{ type: "wait", selector: "1.5" }]);
|
||||
});
|
||||
|
||||
// ── #4692 regression: includeMetadata must NOT send invalid includeTags ────────
|
||||
// Firecrawl returns metadata automatically in response.data.metadata. Sending
|
||||
// includeTags with non-CSS-selector values ("og:title", "description") crashed
|
||||
|
||||
Reference in New Issue
Block a user