Fix graph URL projection by extractor
This commit is contained in:
@@ -459,12 +459,18 @@ chrome.runtime.onMessage.addListener((raw: unknown) => {
|
||||
|
||||
if (message.type === 'crawl-collect') {
|
||||
const collectorsToRun = Array.isArray(message.collectors) ? message.collectors.filter(isCollectorName) : [];
|
||||
const urls = collectorsToRun.flatMap((collectorName) => collectors[collectorName]({}));
|
||||
const linkUrls = collectorsToRun.includes('links') ? collectors.links({}) : [];
|
||||
const mediaUrls = collectorsToRun
|
||||
.filter((collectorName) => collectorName === 'videos' || collectorName === 'images')
|
||||
.flatMap((collectorName) => collectors[collectorName]({}));
|
||||
const urls = [...linkUrls, ...mediaUrls];
|
||||
const elementsLoaded = document.querySelectorAll('a[href], img, video, source').length;
|
||||
const pageText = `${document.title}\n${document.body?.innerText?.slice(0, 8_000) || ''}`;
|
||||
const rateLimitMatch = pageText.match(/(?:\b429\b|too many requests|rate[\s-]?limit(?:ed|ing)?|slow down|try again later|temporarily blocked|access denied|captcha|verify you are human)/i);
|
||||
return Promise.resolve({
|
||||
urls: [...new Set(urls)].slice(0, contentLimits.maxCollectorUrls),
|
||||
linkUrls: [...new Set(linkUrls)].slice(0, contentLimits.maxCollectorUrls),
|
||||
mediaUrls: [...new Set(mediaUrls)].slice(0, contentLimits.maxCollectorUrls),
|
||||
elementsLoaded,
|
||||
pageSignals: {
|
||||
rateLimited: Boolean(rateLimitMatch),
|
||||
|
||||
@@ -8,7 +8,7 @@ type CollectorMessage = {
|
||||
type TabDownloadStartedMessage = { type: 'tab-download-started'; tabId: number; };
|
||||
type SiteLogoMessage = { type: 'get-site-logo'; origin: string; };
|
||||
type BrowserDeliveryTrackMessage = { type: 'track-browser-job'; jobId: string; apiUrl: string; };
|
||||
import { DEFAULT_TAB_PIPELINE_CONTROLS, isExtractionTransportId, normalizeExtractionSettingsSnapshot, normalizeTabPipelineControls, extractorPlanForUrl, type ExtractorId, type ExtractionSettingsSnapshot, type ExtractionTransportId, type TabPipelineControls } from './extractor-pipeline';
|
||||
import { DEFAULT_TAB_PIPELINE_CONTROLS, isExtractionTransportId, normalizeExtractionSettingsSnapshot, normalizeTabPipelineControls, extractorPlanForUrl, urlsForExtractor, type ExtractorId, type ExtractionSettingsSnapshot, type ExtractionTransportId, type TabPipelineControls } from './extractor-pipeline';
|
||||
import { DEFAULT_MAX_INDEXED_DB_BYTES, HARD_MAX_INDEXED_DB_BYTES, MIN_MAX_INDEXED_DB_BYTES, ensureLargeStorageReady, largeStorage } from './indexeddb-storage';
|
||||
import { pollBrowserDeliveryJobs as processBrowserDeliveryJobs, type TrackedBrowserJob } from './browser-delivery-client';
|
||||
|
||||
@@ -545,11 +545,13 @@ async function queryCrawlTab<T>(tabId: number, message: unknown, timeoutMs = 15_
|
||||
|
||||
type CrawlCollectionResponse = {
|
||||
urls?: unknown;
|
||||
linkUrls?: unknown;
|
||||
mediaUrls?: unknown;
|
||||
elementsLoaded?: unknown;
|
||||
pageSignals?: { rateLimited?: unknown; rateLimitReason?: unknown; challengeDetected?: unknown };
|
||||
};
|
||||
|
||||
async function collectCrawlUrls(tabId: number, collectors: CrawlCollector[]): Promise<{ urls: string[]; elementsLoaded: number; rateLimited: boolean; rateLimitReason?: string; responded: boolean }> {
|
||||
async function collectCrawlUrls(tabId: number, collectors: CrawlCollector[]): Promise<{ urls: string[]; linkUrls: string[]; mediaUrls: string[]; elementsLoaded: number; rateLimited: boolean; rateLimitReason?: string; responded: boolean }> {
|
||||
const response = await queryCrawlTab<CrawlCollectionResponse>(tabId, { type: 'crawl-collect', collectors });
|
||||
const maxCollectorUrls = await getExtensionLimit('maxCollectorUrls');
|
||||
return {
|
||||
@@ -557,6 +559,12 @@ async function collectCrawlUrls(tabId: number, collectors: CrawlCollector[]): Pr
|
||||
? response.urls.filter((url): url is string => typeof url === 'string' && url.length <= 20_000)
|
||||
.slice(0, maxCollectorUrls)
|
||||
: [],
|
||||
linkUrls: Array.isArray(response?.linkUrls)
|
||||
? response.linkUrls.filter((url): url is string => typeof url === 'string' && url.length <= 20_000).slice(0, maxCollectorUrls)
|
||||
: [],
|
||||
mediaUrls: Array.isArray(response?.mediaUrls)
|
||||
? response.mediaUrls.filter((url): url is string => typeof url === 'string' && url.length <= 20_000).slice(0, maxCollectorUrls)
|
||||
: [],
|
||||
elementsLoaded: typeof response?.elementsLoaded === 'number' ? response.elementsLoaded : 0,
|
||||
rateLimited: response?.pageSignals?.rateLimited === true || response?.pageSignals?.challengeDetected === true,
|
||||
rateLimitReason: typeof response?.pageSignals?.rateLimitReason === 'string' ? response.pageSignals.rateLimitReason : undefined,
|
||||
@@ -835,7 +843,7 @@ type CollectorProgressEvent = {
|
||||
};
|
||||
|
||||
type PipelineGraphRecord = { url: string; sourceUrl: string; parentUrl: string | null; extractor: ExtractorId; depth: number; status: 'visited' | 'skipped' | 'failed'; detail?: string; path?: string[] };
|
||||
type GraphPageCollection = { urls: string[]; rateLimited: boolean; rateLimitReason?: string; source?: 'cache' | 'existing-tab' | 'temporary-tab' };
|
||||
type GraphPageCollection = { urls: string[]; linkUrls?: string[]; mediaUrls?: string[]; rateLimited: boolean; rateLimitReason?: string; source?: 'cache' | 'existing-tab' | 'temporary-tab' };
|
||||
|
||||
const activeTabPipelineControls = new Map<string, CrawlControl>();
|
||||
const startingTabPipelines = new Set<string>();
|
||||
@@ -979,17 +987,8 @@ async function findExistingTabForUrl(url: string): Promise<number | undefined> {
|
||||
}
|
||||
|
||||
function graphUrlsForExtractor(collection: GraphPageCollection, sourceUrl: string, extractor: ExtractorId): GraphPageCollection {
|
||||
const sourceHost = new URL(sourceUrl).hostname;
|
||||
const media = collection.urls.filter((candidate) => /\.(?:jpe?g|png|gif|webp|avif|mp4|webm|mov)(?:[?#].*)?$/i.test(candidate));
|
||||
const links = collection.urls.filter((candidate) => {
|
||||
try { return new URL(candidate).hostname !== sourceHost; } catch { return false; }
|
||||
});
|
||||
const shared = { rateLimited: collection.rateLimited, rateLimitReason: collection.rateLimitReason, source: collection.source };
|
||||
if (extractor === 'reddit-post' || extractor === 'external-source') return { urls: links, ...shared };
|
||||
if (extractor === 'reddit-gallery' || extractor === 'redgifs-post' || extractor === 'youtube-video') return { urls: media, ...shared };
|
||||
if (extractor === 'direct-media') return { urls: [], ...shared };
|
||||
if (extractor === 'redgifs-uploader') return { urls: collection.urls.filter((candidate) => /redgifs\.com\/(?:watch|users?|profile)\//i.test(candidate)), ...shared };
|
||||
return { urls: collection.urls, ...shared };
|
||||
return { urls: urlsForExtractor(collection, sourceUrl, extractor), ...shared };
|
||||
}
|
||||
|
||||
async function waitForRedgifsGraphTab(tabId: number, windowId: number | undefined, loadTimeoutMs: number, waitAfterLoadMs = crawlPlugins['redgifs-suggested'].waitAfterLoad): Promise<void> {
|
||||
@@ -1069,7 +1068,7 @@ async function collectGraphUrls(url: string, extractor: ExtractorId, cache: Map<
|
||||
collection = await collectCrawlUrls(tabId, ['links', 'videos', 'images']);
|
||||
}
|
||||
const current = await chrome.tabs.get(tabId);
|
||||
const snapshot = { urls: collection.urls, rateLimited: collection.rateLimited || /429|rate.?limit|too many requests|slow down|temporarily blocked|captcha/i.test(String(current.title || '')), rateLimitReason: collection.rateLimitReason, source: ownsTab ? 'temporary-tab' as const : 'existing-tab' as const };
|
||||
const snapshot = { urls: collection.urls, linkUrls: collection.linkUrls, mediaUrls: collection.mediaUrls, rateLimited: collection.rateLimited || /429|rate.?limit|too many requests|slow down|temporarily blocked|captcha/i.test(String(current.title || '')), rateLimitReason: collection.rateLimitReason, source: ownsTab ? 'temporary-tab' as const : 'existing-tab' as const };
|
||||
if (!cache.has(cacheKey) && cache.size >= maxCacheEntries) {
|
||||
const oldest = cache.keys().next().value;
|
||||
if (typeof oldest === 'string') cache.delete(oldest);
|
||||
|
||||
@@ -329,6 +329,44 @@ export function enabledExtractorPlan(value: string, settings: ExtractorSettings)
|
||||
return extractorPlanForUrl(value).filter((extractor) => settings[extractor]);
|
||||
}
|
||||
|
||||
export type ExtractedPageUrls = {
|
||||
urls: string[];
|
||||
linkUrls?: string[];
|
||||
mediaUrls?: string[];
|
||||
};
|
||||
|
||||
function isMediaUrl(value: string): boolean {
|
||||
return /\.(?:jpe?g|png|gif|webp|avif|mp4|webm|mov)(?:[?#].*)?$/i.test(value);
|
||||
}
|
||||
|
||||
function isRedditPostUrl(value: string): boolean {
|
||||
try {
|
||||
const parsed = new URL(value);
|
||||
return hostMatches(parsed.hostname.toLowerCase(), 'reddit.com') && /\/(?:r\/[^/]+\/)?comments\//i.test(parsed.pathname);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Project a page collection into the URLs owned by one extractor. */
|
||||
export function urlsForExtractor(collection: ExtractedPageUrls, sourceUrl: string, extractor: ExtractorId): string[] {
|
||||
const sourceHost = (() => {
|
||||
try { return new URL(sourceUrl).hostname.toLowerCase(); } catch { return ''; }
|
||||
})();
|
||||
const media = (collection.mediaUrls || collection.urls).filter(isMediaUrl);
|
||||
const links = collection.linkUrls || collection.urls.filter((candidate) => {
|
||||
try { return new URL(candidate).hostname.toLowerCase() !== sourceHost; } catch { return false; }
|
||||
});
|
||||
if (extractor === 'reddit-post') return links.filter(isRedditPostUrl);
|
||||
if (extractor === 'external-source') return links.filter((candidate) => {
|
||||
try { return new URL(candidate).hostname.toLowerCase() !== sourceHost; } catch { return false; }
|
||||
});
|
||||
if (extractor === 'reddit-gallery' || extractor === 'redgifs-post' || extractor === 'youtube-video') return media;
|
||||
if (extractor === 'direct-media') return [];
|
||||
if (extractor === 'redgifs-uploader') return links.filter((candidate) => /redgifs\.com\/(?:watch|users?|profile)\//i.test(candidate));
|
||||
return collection.urls;
|
||||
}
|
||||
|
||||
export function normalizeExtractorSettings(value: Partial<ExtractorSettings> | undefined): ExtractorSettings {
|
||||
return {
|
||||
...DEFAULT_EXTRACTOR_SETTINGS,
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
isExtractionTransportId,
|
||||
resolveExtractionSettingsSnapshot,
|
||||
tabExtractorsFromLegacy,
|
||||
urlsForExtractor,
|
||||
} from '../src/extractor-pipeline.ts';
|
||||
|
||||
const base = resolveExtractionSettingsSnapshot(
|
||||
@@ -51,4 +52,23 @@ assert.equal(extractorPlanForUrl('https://www.youtube.com/watch?v=abc')[0], 'you
|
||||
assert.equal(extractorPlanForUrl('https://cdn.example.test/file.mp4')[0], 'direct-media');
|
||||
assert.equal(tabExtractorsFromLegacy({ redgifsSuggested: false })['redgifs-suggested'], false);
|
||||
|
||||
const projected = {
|
||||
urls: [
|
||||
'https://www.reddit.com/r/test/comments/child/title',
|
||||
'https://external.example.test/source',
|
||||
'https://thumbs.redgifs.com/example.mp4',
|
||||
],
|
||||
linkUrls: [
|
||||
'https://www.reddit.com/r/test/comments/child/title',
|
||||
'https://external.example.test/source',
|
||||
'https://www.redgifs.com/users/example',
|
||||
'https://www.redgifs.com/watch/post',
|
||||
],
|
||||
mediaUrls: ['https://thumbs.redgifs.com/example.mp4'],
|
||||
};
|
||||
assert.deepEqual(urlsForExtractor(projected, 'https://www.reddit.com/r/test/comments/root/title', 'reddit-post'), ['https://www.reddit.com/r/test/comments/child/title']);
|
||||
assert.deepEqual(urlsForExtractor(projected, 'https://www.reddit.com/r/test/comments/root/title', 'external-source'), ['https://external.example.test/source', 'https://www.redgifs.com/users/example', 'https://www.redgifs.com/watch/post']);
|
||||
assert.deepEqual(urlsForExtractor(projected, 'https://www.redgifs.com/watch/post', 'redgifs-post'), ['https://thumbs.redgifs.com/example.mp4']);
|
||||
assert.deepEqual(urlsForExtractor(projected, 'https://www.redgifs.com/users/example', 'redgifs-uploader'), ['https://www.redgifs.com/users/example', 'https://www.redgifs.com/watch/post']);
|
||||
|
||||
console.log('extractor pipeline contract checks passed');
|
||||
|
||||
Reference in New Issue
Block a user