scrapex 0.5.3 → 1.0.0-alpha.1

package/dist/index.cjs

@@ -0,0 +1,852 @@
+//#region rolldown:runtime
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+	if (from && typeof from === "object" || typeof from === "function") {
+		for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
+			key = keys[i];
+			if (!__hasOwnProp.call(to, key) && key !== except) {
+				__defProp(to, key, {
+					get: ((k) => from[k]).bind(null, key),
+					enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+				});
+			}
+		}
+	}
+	return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
+	value: mod,
+	enumerable: true
+}) : target, mod));
+
+//#endregion
+const require_enhancer = require('./enhancer-oM4BhYYS.cjs');
+let cheerio = require("cheerio");
+cheerio = __toESM(cheerio);
+let __mozilla_readability = require("@mozilla/readability");
+let turndown = require("turndown");
+turndown = __toESM(turndown);
+
+//#region src/core/context.ts
+let jsdomModule = null;
+/**
+* Preload JSDOM module (called once during scrape initialization)
+*/
+async function preloadJsdom() {
+	if (!jsdomModule) jsdomModule = await import("jsdom");
+}
+/**
+* Create an extraction context with lazy JSDOM loading.
+*
+* Cheerio is always available for fast DOM queries.
+* JSDOM is only loaded when getDocument() is called (for Readability).
+*/
+function createExtractionContext(url, finalUrl, html, options) {
+	let document = null;
+	return {
+		url,
+		finalUrl,
+		html,
+		$: cheerio.load(html),
+		options,
+		results: {},
+		getDocument() {
+			if (!document) {
+				if (!jsdomModule) throw new Error("JSDOM not preloaded. Call preloadJsdom() before using getDocument().");
+				document = new jsdomModule.JSDOM(html, { url: finalUrl }).window.document;
+			}
+			return document;
+		}
+	};
+}
+/**
+* Merge partial results into the context
+*/
+function mergeResults(context, extracted) {
+	return {
+		...context,
+		results: {
+			...context.results,
+			...extracted,
+			custom: extracted.custom || context.results.custom ? {
+				...context.results.custom,
+				...extracted.custom
+			} : void 0
+		}
+	};
+}
+
+//#endregion
+//#region src/extractors/content.ts
+const turndown$1 = new turndown.default({
+	headingStyle: "atx",
+	codeBlockStyle: "fenced",
+	bulletListMarker: "-",
+	emDelimiter: "_",
+	strongDelimiter: "**",
+	linkStyle: "inlined"
+});
+turndown$1.remove([
+	"script",
+	"style",
+	"noscript",
+	"iframe",
+	"nav",
+	"footer"
+]);
+/**
+* Extracts main content using Mozilla Readability.
+* Converts HTML to Markdown for LLM consumption.
+*/
+var ContentExtractor = class {
+	name = "content";
+	priority = 50;
+	async extract(context) {
+		const { options } = context;
+		if (options.extractContent === false) return {};
+		const article = new __mozilla_readability.Readability(context.getDocument().cloneNode(true)).parse();
+		if (!article || !article.content) return this.extractFallback(context);
+		let content = turndown$1.turndown(article.content);
+		const maxLength = options.maxContentLength ?? 5e4;
+		if (content.length > maxLength) content = `${content.slice(0, maxLength)}\n\n[Content truncated...]`;
+		const textContent = (article.textContent ?? "").trim();
+		const excerpt = this.createExcerpt(textContent);
+		const wordCount = textContent.split(/\s+/).filter(Boolean).length;
+		const contentType = this.detectContentType(context);
+		return {
+			content,
+			textContent,
+			excerpt: article.excerpt || excerpt,
+			wordCount,
+			contentType,
+			title: article.title || void 0,
+			author: article.byline || void 0,
+			siteName: article.siteName || void 0
+		};
+	}
+	extractFallback(context) {
+		const { $ } = context;
+		const bodyHtml = $("body").html() || "";
+		const content = turndown$1.turndown(bodyHtml);
+		const textContent = $("body").text().replace(/\s+/g, " ").trim();
+		return {
+			content: content.slice(0, context.options.maxContentLength ?? 5e4),
+			textContent,
+			excerpt: this.createExcerpt(textContent),
+			wordCount: textContent.split(/\s+/).filter(Boolean).length,
+			contentType: "unknown"
+		};
+	}
+	createExcerpt(text, maxLength = 300) {
+		if (text.length <= maxLength) return text;
+		const truncated = text.slice(0, maxLength);
+		const lastSpace = truncated.lastIndexOf(" ");
+		return `${lastSpace > 0 ? truncated.slice(0, lastSpace) : truncated}...`;
+	}
+	detectContentType(context) {
+		const { $, finalUrl } = context;
+		const url = finalUrl.toLowerCase();
+		if (url.includes("github.com") && !url.includes("/blob/") && !url.includes("/issues/")) {
+			if ($("meta[property=\"og:type\"]").attr("content") === "object" || url.match(/github\.com\/[^/]+\/[^/]+\/?$/)) return "repo";
+		}
+		if (url.includes("npmjs.com/package/")) return "package";
+		if (url.includes("pypi.org/project/")) return "package";
+		if (url.includes("/docs/") || url.includes(".readthedocs.") || url.includes("/documentation/")) return "docs";
+		if (url.includes("youtube.com") || url.includes("vimeo.com") || url.includes("youtu.be")) return "video";
+		const hasPrice = $("[class*=\"price\"], [data-price], [itemprop=\"price\"]").length > 0;
+		const hasAddToCart = $("[class*=\"cart\"], [class*=\"buy\"], button:contains(\"Add\")").length > 0;
+		if (hasPrice || hasAddToCart) return "product";
+		const ogType = $("meta[property=\"og:type\"]").attr("content")?.toLowerCase();
+		if (ogType === "article" || ogType === "blog" || ogType === "news") return "article";
+		const hasArticleTag = $("article").length > 0;
+		const hasDateline = $("time[datetime], [class*=\"date\"], [class*=\"byline\"]").length > 0;
+		if (hasArticleTag && hasDateline) return "article";
+		return "unknown";
+	}
+};
+
+//#endregion
+//#region src/utils/url.ts
+/**
+* Common tracking parameters to remove from URLs
+*/
+const TRACKING_PARAMS = [
+	"utm_source",
+	"utm_medium",
+	"utm_campaign",
+	"utm_term",
+	"utm_content",
+	"utm_id",
+	"ref",
+	"fbclid",
+	"gclid",
+	"gclsrc",
+	"dclid",
+	"msclkid",
+	"mc_cid",
+	"mc_eid",
+	"_ga",
+	"_gl",
+	"source",
+	"referrer"
+];
+/**
+* Validate if a string is a valid URL
+*/
+function isValidUrl(url) {
+	try {
+		const parsed = new URL(url);
+		return ["http:", "https:"].includes(parsed.protocol);
+	} catch {
+		return false;
+	}
+}
+/**
+* Normalize URL by removing tracking params and trailing slashes
+*/
+function normalizeUrl(url) {
+	try {
+		const parsed = new URL(url);
+		for (const param of TRACKING_PARAMS) parsed.searchParams.delete(param);
+		let normalized = parsed.toString();
+		if (normalized.endsWith("/") && parsed.pathname !== "/") normalized = normalized.slice(0, -1);
+		return normalized;
+	} catch {
+		return url;
+	}
+}
+/**
+* Extract domain from URL (without www prefix)
+*/
+function extractDomain(url) {
+	try {
+		return new URL(url).hostname.replace(/^www\./, "");
+	} catch {
+		return "";
+	}
+}
+/**
+* Resolve a potentially relative URL against a base URL
+*/
+function resolveUrl(url, baseUrl) {
+	if (!url) return void 0;
+	try {
+		return new URL(url, baseUrl).href;
+	} catch {
+		return url;
+	}
+}
+/**
+* Check if a URL is external relative to a domain
+*/
+function isExternalUrl(url, baseDomain) {
+	try {
+		return new URL(url).hostname.replace(/^www\./, "") !== baseDomain;
+	} catch {
+		return false;
+	}
+}
+/**
+* Extract protocol from URL
+*/
+function getProtocol(url) {
+	try {
+		return new URL(url).protocol;
+	} catch {
+		return "";
+	}
+}
+/**
+* Get the path portion of a URL
+*/
+function getPath(url) {
+	try {
+		return new URL(url).pathname;
+	} catch {
+		return "";
+	}
+}
+/**
+* Check if URL matches a pattern (supports * wildcard)
+*/
+function matchesUrlPattern(url, pattern) {
+	if (!pattern.includes("*")) return url === pattern || url.startsWith(pattern);
+	const regexPattern = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+	return (/* @__PURE__ */ new RegExp(`^${regexPattern}`)).test(url);
+}
+
+//#endregion
+//#region src/extractors/favicon.ts
+/**
+* Extracts favicon URL from the page.
+* Checks multiple sources in order of preference.
+*/
+var FaviconExtractor = class {
+	name = "favicon";
+	priority = 70;
+	async extract(context) {
+		const { $, finalUrl } = context;
+		for (const selector of [
+			"link[rel=\"icon\"][type=\"image/svg+xml\"]",
+			"link[rel=\"icon\"][sizes=\"192x192\"]",
+			"link[rel=\"icon\"][sizes=\"180x180\"]",
+			"link[rel=\"icon\"][sizes=\"128x128\"]",
+			"link[rel=\"icon\"][sizes=\"96x96\"]",
+			"link[rel=\"apple-touch-icon\"][sizes=\"180x180\"]",
+			"link[rel=\"apple-touch-icon\"]",
+			"link[rel=\"icon\"][sizes=\"32x32\"]",
+			"link[rel=\"icon\"]",
+			"link[rel=\"shortcut icon\"]"
+		]) {
+			const href = $(selector).first().attr("href");
+			if (href) return { favicon: resolveUrl(finalUrl, href) };
+		}
+		try {
+			const url = new URL(finalUrl);
+			return { favicon: `${url.protocol}//${url.host}/favicon.ico` };
+		} catch {
+			return {};
+		}
+	}
+};
+
+//#endregion
+//#region src/extractors/jsonld.ts
+/**
+* Extracts JSON-LD structured data from the page.
+* Also extracts additional metadata from structured data.
+*/
+var JsonLdExtractor = class {
+	name = "jsonld";
+	priority = 80;
+	async extract(context) {
+		const { $ } = context;
+		const jsonLd = [];
+		$("script[type=\"application/ld+json\"]").each((_, el) => {
+			const content = $(el).html();
+			if (!content) return;
+			try {
+				const parsed = JSON.parse(content);
+				if (Array.isArray(parsed)) jsonLd.push(...parsed);
+				else if (typeof parsed === "object" && parsed !== null) jsonLd.push(parsed);
+			} catch {}
+		});
+		if (jsonLd.length === 0) return {};
+		return {
+			jsonLd,
+			...this.extractMetadata(jsonLd)
+		};
+	}
+	extractMetadata(jsonLd) {
+		const result = {};
+		for (const item of jsonLd) {
+			const type = this.getType(item);
+			if (type?.match(/Article|BlogPosting|NewsArticle|WebPage/i)) {
+				result.title = result.title || this.getString(item, "headline", "name");
+				result.description = result.description || this.getString(item, "description");
+				result.author = result.author || this.getAuthor(item);
+				result.publishedAt = result.publishedAt || this.getString(item, "datePublished");
+				result.modifiedAt = result.modifiedAt || this.getString(item, "dateModified");
+				result.image = result.image || this.getImage(item);
+			}
+			if (type === "Organization") result.siteName = result.siteName || this.getString(item, "name");
+			if (type === "Product") {
+				result.title = result.title || this.getString(item, "name");
+				result.description = result.description || this.getString(item, "description");
+				result.image = result.image || this.getImage(item);
+			}
+			if (type === "SoftwareApplication") {
+				result.title = result.title || this.getString(item, "name");
+				result.description = result.description || this.getString(item, "description");
+			}
+			const keywords = this.getKeywords(item);
+			if (keywords.length > 0) result.keywords = [...result.keywords || [], ...keywords];
+		}
+		if (result.keywords) result.keywords = [...new Set(result.keywords)];
+		return result;
+	}
+	getType(item) {
+		const type = item["@type"];
+		if (typeof type === "string") return type;
+		if (Array.isArray(type)) return type[0];
+	}
+	getString(item, ...keys) {
+		for (const key of keys) {
+			const value = item[key];
+			if (typeof value === "string") return value;
+			if (typeof value === "object" && value !== null && "@value" in value) return String(value["@value"]);
+		}
+	}
+	getAuthor(item) {
+		const author = item.author;
+		if (typeof author === "string") return author;
+		if (Array.isArray(author)) return author.map((a) => typeof a === "string" ? a : this.getString(a, "name")).filter(Boolean).join(", ") || void 0;
+		if (typeof author === "object" && author !== null) {
+			const authorObj = author;
+			return this.getString(authorObj, "name") || void 0;
+		}
+	}
+	getImage(item) {
+		const image = item.image;
+		if (typeof image === "string") return image;
+		if (Array.isArray(image) && image.length > 0) return this.getImage({ image: image[0] });
+		if (typeof image === "object" && image !== null) {
+			const imageObj = image;
+			return this.getString(imageObj, "url", "contentUrl") || void 0;
+		}
+	}
+	getKeywords(item) {
+		const keywords = item.keywords;
+		if (typeof keywords === "string") return keywords.split(",").map((k) => k.trim()).filter(Boolean);
+		if (Array.isArray(keywords)) return keywords.filter((k) => typeof k === "string");
+		return [];
+	}
+};
+
+//#endregion
+//#region src/extractors/links.ts
+/**
+* Extracts links from the page content.
+* Filters out navigation/footer links and focuses on content links.
+*/
+var LinksExtractor = class {
+	name = "links";
+	priority = 30;
+	async extract(context) {
+		const { $, finalUrl } = context;
+		const links = [];
+		const seen = /* @__PURE__ */ new Set();
+		const contentArea = $("article, main, [role=\"main\"]").first();
+		const container = contentArea.length > 0 ? contentArea : $("body");
+		const skipSelectors = "nav, header, footer, aside, [role=\"navigation\"], [class*=\"nav\"], [class*=\"footer\"], [class*=\"header\"], [class*=\"sidebar\"], [class*=\"menu\"]";
+		container.find("a[href]").each((_, el) => {
+			const $el = $(el);
+			if ($el.closest(skipSelectors).length > 0) return;
+			const href = $el.attr("href");
+			if (!href) return;
+			if (href.startsWith("#") || href.startsWith("javascript:") || href.startsWith("mailto:") || href.startsWith("tel:")) return;
+			const resolvedUrl = resolveUrl(href, finalUrl);
+			if (!resolvedUrl || !isValidUrl(resolvedUrl)) return;
+			if (seen.has(resolvedUrl)) return;
+			seen.add(resolvedUrl);
+			const text = $el.text().trim() || $el.attr("title") || $el.attr("aria-label") || "";
+			if (text.length < 2) return;
+			const baseDomain = extractDomain(finalUrl);
+			links.push({
+				url: resolvedUrl,
+				text: text.slice(0, 200),
+				isExternal: isExternalUrl(resolvedUrl, baseDomain)
+			});
+		});
+		return { links: links.slice(0, 100) };
+	}
+};
+
+//#endregion
+//#region src/extractors/meta.ts
+/**
+* Extracts metadata from HTML meta tags, Open Graph, and Twitter cards.
+* Runs first to provide basic metadata for other extractors.
+*/
+var MetaExtractor = class {
+	name = "meta";
+	priority = 100;
+	async extract(context) {
+		const { $ } = context;
+		const getMeta = (nameOrProperty) => {
+			return ($(`meta[name="${nameOrProperty}"]`).attr("content") || $(`meta[property="${nameOrProperty}"]`).attr("content") || $(`meta[itemprop="${nameOrProperty}"]`).attr("content"))?.trim() || void 0;
+		};
+		const title = getMeta("og:title") || getMeta("twitter:title") || $("title").first().text().trim() || "";
+		const description = getMeta("og:description") || getMeta("twitter:description") || getMeta("description") || "";
+		const image = getMeta("og:image") || getMeta("twitter:image") || getMeta("twitter:image:src") || void 0;
+		const canonicalUrl = $("link[rel=\"canonical\"]").attr("href") || getMeta("og:url") || context.finalUrl;
+		const author = getMeta("author") || getMeta("article:author") || getMeta("twitter:creator") || $("[rel=\"author\"]").first().text().trim() || void 0;
+		const siteName = getMeta("og:site_name") || getMeta("application-name") || void 0;
+		const publishedAt = getMeta("article:published_time") || getMeta("datePublished") || getMeta("date") || $("time[datetime]").first().attr("datetime") || void 0;
+		const modifiedAt = getMeta("article:modified_time") || getMeta("dateModified") || void 0;
+		const language = $("html").attr("lang") || getMeta("og:locale") || getMeta("language") || void 0;
+		const keywordsRaw = getMeta("keywords") || getMeta("article:tag") || "";
+		return {
+			title,
+			description,
+			image,
+			canonicalUrl,
+			author,
+			siteName,
+			publishedAt,
+			modifiedAt,
+			language,
+			keywords: keywordsRaw ? keywordsRaw.split(",").map((k) => k.trim()).filter(Boolean) : []
+		};
+	}
+};
+
+//#endregion
+//#region src/extractors/index.ts
+/**
+* Default extractors in priority order.
+* Higher priority runs first.
+*/
+function createDefaultExtractors() {
+	return [
+		new MetaExtractor(),
+		new JsonLdExtractor(),
+		new FaviconExtractor(),
+		new ContentExtractor(),
+		new LinksExtractor()
+	];
+}
+/**
+* Sort extractors by priority (higher first).
+*/
+function sortExtractors(extractors) {
+	return [...extractors].sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
+}
+
+//#endregion
+//#region src/fetchers/types.ts
+/**
+* Default user agent string
+*/
+const DEFAULT_USER_AGENT = "Scrapex-Bot/2.0 (+https://github.com/developer-rakeshpaul/scrapex)";
+/**
+* Default timeout in milliseconds
+*/
+const DEFAULT_TIMEOUT = 1e4;
+
+//#endregion
+//#region src/fetchers/fetch.ts
+/**
+* Default fetcher using native fetch API.
+* Works in Node.js 18+ without polyfills.
+*/
+var NativeFetcher = class {
+	name = "native-fetch";
+	async fetch(url, options = {}) {
+		const { timeout = DEFAULT_TIMEOUT, userAgent = DEFAULT_USER_AGENT, headers = {} } = options;
+		let parsedUrl;
+		try {
+			parsedUrl = new URL(url);
+		} catch {
+			throw new require_enhancer.ScrapeError(`Invalid URL: ${url}`, "INVALID_URL");
+		}
+		if (!["http:", "https:"].includes(parsedUrl.protocol)) throw new require_enhancer.ScrapeError(`Invalid protocol: ${parsedUrl.protocol}`, "INVALID_URL");
+		const controller = new AbortController();
+		const timeoutId = setTimeout(() => controller.abort(), timeout);
+		try {
+			const response = await fetch(url, {
+				signal: controller.signal,
+				headers: {
+					"User-Agent": userAgent,
+					Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+					"Accept-Language": "en-US,en;q=0.5",
+					...headers
+				},
+				redirect: "follow"
+			});
+			clearTimeout(timeoutId);
+			if (!response.ok) {
+				if (response.status === 404) throw new require_enhancer.ScrapeError(`Page not found: ${url}`, "NOT_FOUND", 404);
+				if (response.status === 403 || response.status === 401) throw new require_enhancer.ScrapeError(`Access blocked: ${url}`, "BLOCKED", response.status);
+				if (response.status === 429) throw new require_enhancer.ScrapeError(`Rate limited: ${url}`, "BLOCKED", 429);
+				throw new require_enhancer.ScrapeError(`HTTP error ${response.status}: ${url}`, "FETCH_FAILED", response.status);
+			}
+			const contentType = response.headers.get("content-type") || "";
+			if (!contentType.includes("text/html") && !contentType.includes("application/xhtml")) throw new require_enhancer.ScrapeError(`Unexpected content type: ${contentType}`, "PARSE_ERROR");
+			const html = await response.text();
+			const responseHeaders = {};
+			response.headers.forEach((value, key) => {
+				responseHeaders[key] = value;
+			});
+			return {
+				html,
+				finalUrl: response.url,
+				statusCode: response.status,
+				contentType,
+				headers: responseHeaders
+			};
+		} catch (error) {
+			clearTimeout(timeoutId);
+			if (error instanceof require_enhancer.ScrapeError) throw error;
+			if (error instanceof Error && error.name === "AbortError") throw new require_enhancer.ScrapeError(`Request timed out after ${timeout}ms`, "TIMEOUT");
+			if (error instanceof Error) throw new require_enhancer.ScrapeError(`Fetch failed: ${error.message}`, "FETCH_FAILED", void 0, error);
+			throw new require_enhancer.ScrapeError("Unknown fetch error", "FETCH_FAILED");
+		}
+	}
+};
+/**
+* Default fetcher instance
+*/
+const defaultFetcher = new NativeFetcher();
+
+//#endregion
+//#region src/fetchers/robots.ts
+/**
+* Check if URL is allowed by robots.txt
+*
+* @param url - The URL to check
+* @param userAgent - User agent to check rules for
+* @returns Whether the URL is allowed and optional reason
+*/
+async function checkRobotsTxt(url, userAgent = DEFAULT_USER_AGENT) {
+	try {
+		const parsedUrl = new URL(url);
+		const robotsUrl = `${parsedUrl.protocol}//${parsedUrl.host}/robots.txt`;
+		const response = await fetch(robotsUrl, {
+			headers: { "User-Agent": userAgent },
+			signal: AbortSignal.timeout(5e3)
+		});
+		if (!response.ok) return { allowed: true };
+		const allowed = isPathAllowed(parseRobotsTxt(await response.text(), userAgent), parsedUrl.pathname + parsedUrl.search);
+		return {
+			allowed,
+			reason: allowed ? void 0 : "Blocked by robots.txt"
+		};
+	} catch {
+		return { allowed: true };
+	}
+}
+/**
+* Parse robots.txt content for a specific user agent
+*/
+function parseRobotsTxt(content, userAgent) {
+	const rules = {
+		disallow: [],
+		allow: []
+	};
+	const lines = content.split("\n");
+	const botName = userAgent.split(/[\s/]/)[0]?.toLowerCase() || "";
+	let currentAgent = "";
+	let isMatchingAgent = false;
+	let hasFoundSpecificAgent = false;
+	for (const rawLine of lines) {
+		const line = rawLine.trim();
+		if (!line || line.startsWith("#")) continue;
+		const colonIndex = line.indexOf(":");
+		if (colonIndex === -1) continue;
+		const directive = line.slice(0, colonIndex).trim().toLowerCase();
+		const value = line.slice(colonIndex + 1).trim();
+		if (directive === "user-agent") {
+			currentAgent = value.toLowerCase();
+			isMatchingAgent = currentAgent === "*" || currentAgent === botName || botName.includes(currentAgent);
+			if (currentAgent !== "*" && isMatchingAgent) {
+				hasFoundSpecificAgent = true;
+				rules.disallow = [];
+				rules.allow = [];
+			}
+		} else if (isMatchingAgent && (!hasFoundSpecificAgent || currentAgent !== "*")) {
+			if (directive === "disallow" && value) rules.disallow.push(value);
+			else if (directive === "allow" && value) rules.allow.push(value);
+		}
+	}
+	return rules;
+}
+/**
+* Check if a path is allowed based on robots.txt rules
+*/
+function isPathAllowed(rules, path) {
+	if (rules.disallow.length === 0 && rules.allow.length === 0) return true;
+	for (const pattern of rules.allow) if (matchesPattern(path, pattern)) return true;
+	for (const pattern of rules.disallow) if (matchesPattern(path, pattern)) return false;
+	return true;
+}
+/**
+* Check if a path matches a robots.txt pattern
+*/
+function matchesPattern(path, pattern) {
+	if (!pattern) return false;
+	if (pattern.endsWith("*")) return path.startsWith(pattern.slice(0, -1));
+	if (pattern.endsWith("$")) return path === pattern.slice(0, -1);
+	if (pattern.includes("*")) return (/* @__PURE__ */ new RegExp(`^${pattern.replace(/\*/g, ".*").replace(/\?/g, "\\?")}.*`)).test(path);
+	return path.startsWith(pattern);
+}
+
+//#endregion
+//#region src/core/scrape.ts
+/**
+* Scrape a URL and extract metadata and content.
+*
+* @param url - The URL to scrape
+* @param options - Scraping options
+* @returns Scraped data with metadata and content
+*
+* @example
+* ```ts
+* const result = await scrape('https://example.com/article');
+* console.log(result.title, result.content);
+* ```
+*/
+async function scrape(url, options = {}) {
+	const startTime = Date.now();
+	if (!isValidUrl(url)) throw new require_enhancer.ScrapeError("Invalid URL provided", "INVALID_URL");
+	const normalizedUrl = normalizeUrl(url);
+	if (options.respectRobots) {
+		const robotsResult = await checkRobotsTxt(normalizedUrl, options.userAgent);
+		if (!robotsResult.allowed) throw new require_enhancer.ScrapeError(`URL blocked by robots.txt: ${robotsResult.reason || "disallowed"}`, "ROBOTS_BLOCKED");
+	}
+	const fetchResult = await (options.fetcher ?? defaultFetcher).fetch(normalizedUrl, {
+		timeout: options.timeout,
+		userAgent: options.userAgent
+	});
+	await preloadJsdom();
+	let context = createExtractionContext(normalizedUrl, fetchResult.finalUrl, fetchResult.html, options);
+	let extractors;
+	if (options.replaceDefaultExtractors) extractors = options.extractors ?? [];
+	else {
+		const defaults = createDefaultExtractors();
+		extractors = options.extractors ? [...defaults, ...options.extractors] : defaults;
+	}
+	extractors = sortExtractors(extractors);
+	for (const extractor of extractors) try {
+		const extracted = await extractor.extract(context);
+		context = mergeResults(context, extracted);
+	} catch (error) {
+		console.error(`Extractor "${extractor.name}" failed:`, error);
+		context = mergeResults(context, { error: context.results.error ? `${context.results.error}; ${extractor.name}: ${error instanceof Error ? error.message : String(error)}` : `${extractor.name}: ${error instanceof Error ? error.message : String(error)}` });
+	}
+	const intermediateResult = {
+		url: normalizedUrl,
+		canonicalUrl: context.results.canonicalUrl || fetchResult.finalUrl,
+		domain: extractDomain(fetchResult.finalUrl),
+		title: context.results.title || "",
+		description: context.results.description || "",
+		image: context.results.image,
+		favicon: context.results.favicon,
+		content: context.results.content || "",
+		textContent: context.results.textContent || "",
+		excerpt: context.results.excerpt || "",
+		wordCount: context.results.wordCount || 0,
+		author: context.results.author,
+		publishedAt: context.results.publishedAt,
+		modifiedAt: context.results.modifiedAt,
+		siteName: context.results.siteName,
+		language: context.results.language,
+		contentType: context.results.contentType || "unknown",
+		keywords: context.results.keywords || [],
+		jsonLd: context.results.jsonLd,
+		links: context.results.links,
+		custom: context.results.custom,
+		scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		scrapeTimeMs: 0,
+		error: context.results.error
+	};
+	if (options.llm && options.enhance && options.enhance.length > 0) try {
+		const enhanced = await require_enhancer.enhance(intermediateResult, options.llm, options.enhance);
+		Object.assign(intermediateResult, enhanced);
+	} catch (error) {
+		console.error("LLM enhancement failed:", error);
+		intermediateResult.error = intermediateResult.error ? `${intermediateResult.error}; LLM: ${error instanceof Error ? error.message : String(error)}` : `LLM: ${error instanceof Error ? error.message : String(error)}`;
+	}
+	if (options.llm && options.extract) try {
+		intermediateResult.extracted = await require_enhancer.extract(intermediateResult, options.llm, options.extract);
+	} catch (error) {
+		console.error("LLM extraction failed:", error);
+		intermediateResult.error = intermediateResult.error ? `${intermediateResult.error}; LLM extraction: ${error instanceof Error ? error.message : String(error)}` : `LLM extraction: ${error instanceof Error ? error.message : String(error)}`;
+	}
+	const scrapeTimeMs = Date.now() - startTime;
+	return {
+		...intermediateResult,
+		scrapeTimeMs
+	};
+}
+/**
+* Scrape from raw HTML string (no fetch).
+*
+* @param html - The HTML content
+* @param url - The URL (for resolving relative links)
+* @param options - Scraping options
+* @returns Scraped data with metadata and content
+*
+* @example
+* ```ts
+* const html = await fetchSomehow('https://example.com');
+* const result = await scrapeHtml(html, 'https://example.com');
+* ```
+*/
+async function scrapeHtml(html, url, options = {}) {
+	const startTime = Date.now();
+	if (!isValidUrl(url)) throw new require_enhancer.ScrapeError("Invalid URL provided", "INVALID_URL");
+	const normalizedUrl = normalizeUrl(url);
+	await preloadJsdom();
+	let context = createExtractionContext(normalizedUrl, normalizedUrl, html, options);
+	let extractors;
+	if (options.replaceDefaultExtractors) extractors = options.extractors ?? [];
+	else {
+		const defaults = createDefaultExtractors();
+		extractors = options.extractors ? [...defaults, ...options.extractors] : defaults;
+	}
+	extractors = sortExtractors(extractors);
+	for (const extractor of extractors) try {
+		const extracted = await extractor.extract(context);
+		context = mergeResults(context, extracted);
+	} catch (error) {
+		console.error(`Extractor "${extractor.name}" failed:`, error);
+		context = mergeResults(context, { error: context.results.error ? `${context.results.error}; ${extractor.name}: ${error instanceof Error ? error.message : String(error)}` : `${extractor.name}: ${error instanceof Error ? error.message : String(error)}` });
+	}
+	const scrapeTimeMs = Date.now() - startTime;
+	const domain = extractDomain(normalizedUrl);
+	return {
+		url: normalizedUrl,
+		canonicalUrl: context.results.canonicalUrl || normalizedUrl,
+		domain,
+		title: context.results.title || "",
+		description: context.results.description || "",
+		image: context.results.image,
+		favicon: context.results.favicon,
+		content: context.results.content || "",
+		textContent: context.results.textContent || "",
+		excerpt: context.results.excerpt || "",
+		wordCount: context.results.wordCount || 0,
+		author: context.results.author,
+		publishedAt: context.results.publishedAt,
+		modifiedAt: context.results.modifiedAt,
+		siteName: context.results.siteName,
+		language: context.results.language,
+		contentType: context.results.contentType || "unknown",
+		keywords: context.results.keywords || [],
+		jsonLd: context.results.jsonLd,
+		links: context.results.links,
+		summary: context.results.summary,
+		suggestedTags: context.results.suggestedTags,
+		entities: context.results.entities,
+		extracted: context.results.extracted,
+		custom: context.results.custom,
+		scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		scrapeTimeMs,
+		error: context.results.error
+	};
+}
+
+//#endregion
+exports.ContentExtractor = ContentExtractor;
+exports.DEFAULT_TIMEOUT = DEFAULT_TIMEOUT;
+exports.DEFAULT_USER_AGENT = DEFAULT_USER_AGENT;
+exports.FaviconExtractor = FaviconExtractor;
+exports.JsonLdExtractor = JsonLdExtractor;
+exports.LinksExtractor = LinksExtractor;
+exports.MetaExtractor = MetaExtractor;
+exports.NativeFetcher = NativeFetcher;
+exports.ScrapeError = require_enhancer.ScrapeError;
+exports.__toESM = __toESM;
+exports.checkRobotsTxt = checkRobotsTxt;
+exports.createDefaultExtractors = createDefaultExtractors;
+exports.createExtractionContext = createExtractionContext;
+exports.defaultFetcher = defaultFetcher;
+exports.extractDomain = extractDomain;
+exports.getPath = getPath;
+exports.getProtocol = getProtocol;
+exports.isExternalUrl = isExternalUrl;
+exports.isValidUrl = isValidUrl;
+exports.matchesUrlPattern = matchesUrlPattern;
+exports.mergeResults = mergeResults;
+exports.normalizeUrl = normalizeUrl;
+exports.resolveUrl = resolveUrl;
+exports.scrape = scrape;
+exports.scrapeHtml = scrapeHtml;
+exports.sortExtractors = sortExtractors;
+//# sourceMappingURL=index.cjs.map