scrapex 1.0.0-alpha.1 → 1.0.0-beta.2

package/dist/index.cjs

@@ -1,36 +1,12 @@
-//#region rolldown:runtime
-var __create = Object.create;
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __getProtoOf = Object.getPrototypeOf;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __copyProps = (to, from, except, desc) => {
-	if (from && typeof from === "object" || typeof from === "function") {
-		for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
-			key = keys[i];
-			if (!__hasOwnProp.call(to, key) && key !== except) {
-				__defProp(to, key, {
-					get: ((k) => from[k]).bind(null, key),
-					enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
-				});
-			}
-		}
-	}
-	return to;
-};
-var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
-	value: mod,
-	enumerable: true
-}) : target, mod));
-
-//#endregion
-const require_enhancer = require('./enhancer-oM4BhYYS.cjs');
+const require_parsers = require('./parsers-Bneuws8x.cjs');
+const require_http_base = require('./http-base-CHLf-Tco.cjs');
+const require_embeddings = require('./embeddings-BjNTQSG9.cjs');
+const require_enhancer = require('./enhancer-Cs_WyWtJ.cjs');
 let cheerio = require("cheerio");
-cheerio = __toESM(cheerio);
-let __mozilla_readability = require("@mozilla/readability");
+cheerio = require_parsers.__toESM(cheerio);
+let _mozilla_readability = require("@mozilla/readability");
 let turndown = require("turndown");
-turndown = __toESM(turndown);
+turndown = require_parsers.__toESM(turndown);
 
 //#region src/core/context.ts
 let jsdomModule = null;
@@ -109,7 +85,7 @@ var ContentExtractor = class {
 	async extract(context) {
 		const { options } = context;
 		if (options.extractContent === false) return {};
-		const article = new __mozilla_readability.Readability(context.getDocument().cloneNode(true)).parse();
+		const article = new _mozilla_readability.Readability(context.getDocument().cloneNode(true)).parse();
 		if (!article || !article.content) return this.extractFallback(context);
 		let content = turndown$1.turndown(article.content);
 		const maxLength = options.maxContentLength ?? 5e4;
@@ -533,9 +509,9 @@ var NativeFetcher = class {
 		try {
 			parsedUrl = new URL(url);
 		} catch {
-			throw new require_enhancer.ScrapeError(`Invalid URL: ${url}`, "INVALID_URL");
+			throw new require_http_base.ScrapeError(`Invalid URL: ${url}`, "INVALID_URL");
 		}
-		if (!["http:", "https:"].includes(parsedUrl.protocol)) throw new require_enhancer.ScrapeError(`Invalid protocol: ${parsedUrl.protocol}`, "INVALID_URL");
+		if (!["http:", "https:"].includes(parsedUrl.protocol)) throw new require_http_base.ScrapeError(`Invalid protocol: ${parsedUrl.protocol}`, "INVALID_URL");
 		const controller = new AbortController();
 		const timeoutId = setTimeout(() => controller.abort(), timeout);
 		try {
@@ -551,13 +527,15 @@ var NativeFetcher = class {
 			});
 			clearTimeout(timeoutId);
 			if (!response.ok) {
-				if (response.status === 404) throw new require_enhancer.ScrapeError(`Page not found: ${url}`, "NOT_FOUND", 404);
-				if (response.status === 403 || response.status === 401) throw new require_enhancer.ScrapeError(`Access blocked: ${url}`, "BLOCKED", response.status);
-				if (response.status === 429) throw new require_enhancer.ScrapeError(`Rate limited: ${url}`, "BLOCKED", 429);
-				throw new require_enhancer.ScrapeError(`HTTP error ${response.status}: ${url}`, "FETCH_FAILED", response.status);
+				if (response.status === 404) throw new require_http_base.ScrapeError(`Page not found: ${url}`, "NOT_FOUND", 404);
+				if (response.status === 403 || response.status === 401) throw new require_http_base.ScrapeError(`Access blocked: ${url}`, "BLOCKED", response.status);
+				if (response.status === 429) throw new require_http_base.ScrapeError(`Rate limited: ${url}`, "BLOCKED", 429);
+				throw new require_http_base.ScrapeError(`HTTP error ${response.status}: ${url}`, "FETCH_FAILED", response.status);
 			}
 			const contentType = response.headers.get("content-type") || "";
-			if (!contentType.includes("text/html") && !contentType.includes("application/xhtml")) throw new require_enhancer.ScrapeError(`Unexpected content type: ${contentType}`, "PARSE_ERROR");
+			if (options.allowedContentTypes) {
+				if (!options.allowedContentTypes.some((type) => contentType.toLowerCase().includes(type.toLowerCase()))) throw new require_http_base.ScrapeError(`Unexpected content type: ${contentType}`, "PARSE_ERROR");
+			} else if (!contentType.includes("text/html") && !contentType.includes("application/xhtml")) throw new require_http_base.ScrapeError(`Unexpected content type: ${contentType}`, "PARSE_ERROR");
 			const html = await response.text();
 			const responseHeaders = {};
 			response.headers.forEach((value, key) => {
@@ -572,10 +550,10 @@ var NativeFetcher = class {
 			};
 		} catch (error) {
 			clearTimeout(timeoutId);
-			if (error instanceof require_enhancer.ScrapeError) throw error;
-			if (error instanceof Error && error.name === "AbortError") throw new require_enhancer.ScrapeError(`Request timed out after ${timeout}ms`, "TIMEOUT");
-			if (error instanceof Error) throw new require_enhancer.ScrapeError(`Fetch failed: ${error.message}`, "FETCH_FAILED", void 0, error);
-			throw new require_enhancer.ScrapeError("Unknown fetch error", "FETCH_FAILED");
+			if (error instanceof require_http_base.ScrapeError) throw error;
+			if (error instanceof Error && error.name === "AbortError") throw new require_http_base.ScrapeError(`Request timed out after ${timeout}ms`, "TIMEOUT");
+			if (error instanceof Error) throw new require_http_base.ScrapeError(`Fetch failed: ${error.message}`, "FETCH_FAILED", void 0, error);
+			throw new require_http_base.ScrapeError("Unknown fetch error", "FETCH_FAILED");
 		}
 	}
 };
@@ -683,11 +661,11 @@ function matchesPattern(path, pattern) {
 */
 async function scrape(url, options = {}) {
 	const startTime = Date.now();
-	if (!isValidUrl(url)) throw new require_enhancer.ScrapeError("Invalid URL provided", "INVALID_URL");
+	if (!isValidUrl(url)) throw new require_http_base.ScrapeError("Invalid URL provided", "INVALID_URL");
 	const normalizedUrl = normalizeUrl(url);
 	if (options.respectRobots) {
 		const robotsResult = await checkRobotsTxt(normalizedUrl, options.userAgent);
-		if (!robotsResult.allowed) throw new require_enhancer.ScrapeError(`URL blocked by robots.txt: ${robotsResult.reason || "disallowed"}`, "ROBOTS_BLOCKED");
+		if (!robotsResult.allowed) throw new require_http_base.ScrapeError(`URL blocked by robots.txt: ${robotsResult.reason || "disallowed"}`, "ROBOTS_BLOCKED");
 	}
 	const fetchResult = await (options.fetcher ?? defaultFetcher).fetch(normalizedUrl, {
 		timeout: options.timeout,
@@ -748,6 +726,7 @@ async function scrape(url, options = {}) {
 		console.error("LLM extraction failed:", error);
 		intermediateResult.error = intermediateResult.error ? `${intermediateResult.error}; LLM extraction: ${error instanceof Error ? error.message : String(error)}` : `LLM extraction: ${error instanceof Error ? error.message : String(error)}`;
 	}
+	if (options.embeddings) intermediateResult.embeddings = await require_embeddings.generateEmbeddings(intermediateResult, options.embeddings);
 	const scrapeTimeMs = Date.now() - startTime;
 	return {
 		...intermediateResult,
@@ -770,7 +749,7 @@ async function scrape(url, options = {}) {
 */
 async function scrapeHtml(html, url, options = {}) {
 	const startTime = Date.now();
-	if (!isValidUrl(url)) throw new require_enhancer.ScrapeError("Invalid URL provided", "INVALID_URL");
+	if (!isValidUrl(url)) throw new require_http_base.ScrapeError("Invalid URL provided", "INVALID_URL");
 	const normalizedUrl = normalizeUrl(url);
 	await preloadJsdom();
 	let context = createExtractionContext(normalizedUrl, normalizedUrl, html, options);
@@ -788,9 +767,8 @@ async function scrapeHtml(html, url, options = {}) {
 		console.error(`Extractor "${extractor.name}" failed:`, error);
 		context = mergeResults(context, { error: context.results.error ? `${context.results.error}; ${extractor.name}: ${error instanceof Error ? error.message : String(error)}` : `${extractor.name}: ${error instanceof Error ? error.message : String(error)}` });
 	}
-	const scrapeTimeMs = Date.now() - startTime;
 	const domain = extractDomain(normalizedUrl);
-	return {
+	const intermediateResult = {
 		url: normalizedUrl,
 		canonicalUrl: context.results.canonicalUrl || normalizedUrl,
 		domain,
@@ -817,9 +795,127 @@ async function scrapeHtml(html, url, options = {}) {
 		extracted: context.results.extracted,
 		custom: context.results.custom,
 		scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
-		scrapeTimeMs,
+		scrapeTimeMs: 0,
 		error: context.results.error
 	};
+	if (options.embeddings) intermediateResult.embeddings = await require_embeddings.generateEmbeddings(intermediateResult, options.embeddings);
+	const scrapeTimeMs = Date.now() - startTime;
+	return {
+		...intermediateResult,
+		scrapeTimeMs
+	};
+}
+
+//#endregion
+//#region src/utils/feed.ts
+/**
+* Fetch and parse an RSS/Atom feed from a URL.
+* Uses scrapex's fetcher infrastructure for consistent behavior.
+*/
+async function fetchFeed(url, options) {
+	const result = await (options?.fetcher || defaultFetcher).fetch(url, {
+		timeout: options?.timeout,
+		userAgent: options?.userAgent,
+		allowedContentTypes: [
+			"application/rss+xml",
+			"application/atom+xml",
+			"application/rdf+xml",
+			"application/xml",
+			"text/xml",
+			"text/html"
+		]
+	});
+	return new require_parsers.RSSParser(options?.parserOptions).parse(result.html, url);
+}
+/**
+* Detect RSS/Atom feed URLs from HTML.
+* Supports RSS, Atom, and RDF feed types.
+*/
+function discoverFeeds(html, baseUrl) {
+	const $ = cheerio.load(html);
+	const feeds = [];
+	const seen = /* @__PURE__ */ new Set();
+	$([
+		"link[type=\"application/rss+xml\"]",
+		"link[type=\"application/atom+xml\"]",
+		"link[type=\"application/rdf+xml\"]",
+		"link[rel=\"alternate\"][type*=\"xml\"]"
+	].join(", ")).each((_, el) => {
+		const href = $(el).attr("href");
+		if (href) try {
+			const resolved = new URL(href, baseUrl).href;
+			if (!seen.has(resolved)) {
+				seen.add(resolved);
+				feeds.push(resolved);
+			}
+		} catch {}
+	});
+	return feeds;
+}
+/**
+* Filter feed items by date range.
+* Items without publishedAt are included by default.
+*/
+function filterByDate(items, options) {
+	const { after, before, includeUndated = true } = options;
+	return items.filter((item) => {
+		if (!item.publishedAt) return includeUndated;
+		const date = new Date(item.publishedAt);
+		if (after && date < after) return false;
+		if (before && date > before) return false;
+		return true;
+	});
+}
+/**
+* Convert feed items to markdown for LLM consumption.
+* Uses ISO 8601 date format for consistency across environments.
+*/
+function feedToMarkdown(feed, options) {
+	const { includeContent = false, maxItems } = options || {};
+	const lines = [`# ${feed.title}`, ""];
+	if (feed.description) lines.push(feed.description, "");
+	const items = maxItems ? feed.items.slice(0, maxItems) : feed.items;
+	for (const item of items) {
+		lines.push(`## ${item.title}`);
+		if (item.publishedAt) {
+			const date = item.publishedAt.split("T")[0];
+			lines.push(`*${date}*`);
+		}
+		lines.push("");
+		if (includeContent && item.content) lines.push(item.content);
+		else if (item.description) lines.push(item.description);
+		if (item.link) lines.push(`[Read more](${item.link})`, "");
+		else lines.push("");
+	}
+	return lines.join("\n");
+}
+/**
+* Extract plain text from feed items for LLM processing.
+* Concatenates title, description, and content.
+*/
+function feedToText(feed, options) {
+	const { maxItems, separator = "\n\n---\n\n" } = options || {};
+	return (maxItems ? feed.items.slice(0, maxItems) : feed.items).map((item) => {
+		const parts = [item.title];
+		if (item.description) parts.push(item.description);
+		if (item.content) parts.push(item.content);
+		return parts.join("\n\n");
+	}).join(separator);
+}
+/**
+* Paginate through a feed using rel="next" links (RFC 5005).
+* Returns an async generator that yields each page.
+*/
+async function* paginateFeed(url, options) {
+	const { maxPages = 10, ...fetchOptions } = options || {};
+	let currentUrl = url;
+	let pageCount = 0;
+	while (currentUrl && pageCount < maxPages) {
+		const result = await fetchFeed(currentUrl, fetchOptions);
+		yield result.data;
+		currentUrl = result.data.next;
+		pageCount++;
+	}
 }
 
 //#endregion
@@ -827,17 +923,39 @@ exports.ContentExtractor = ContentExtractor;
 exports.DEFAULT_TIMEOUT = DEFAULT_TIMEOUT;
 exports.DEFAULT_USER_AGENT = DEFAULT_USER_AGENT;
 exports.FaviconExtractor = FaviconExtractor;
+exports.InMemoryEmbeddingCache = require_embeddings.InMemoryEmbeddingCache;
 exports.JsonLdExtractor = JsonLdExtractor;
 exports.LinksExtractor = LinksExtractor;
 exports.MetaExtractor = MetaExtractor;
 exports.NativeFetcher = NativeFetcher;
-exports.ScrapeError = require_enhancer.ScrapeError;
-exports.__toESM = __toESM;
+exports.RSSParser = require_parsers.RSSParser;
+exports.ScrapeError = require_http_base.ScrapeError;
+exports.TRANSFORMERS_MODELS = require_embeddings.TRANSFORMERS_MODELS;
+exports.aggregateVectors = require_embeddings.aggregateVectors;
 exports.checkRobotsTxt = checkRobotsTxt;
+exports.chunkText = require_embeddings.chunkText;
+exports.cosineSimilarity = require_embeddings.cosineSimilarity;
+exports.createAzureEmbedding = require_embeddings.createAzureEmbedding;
 exports.createDefaultExtractors = createDefaultExtractors;
+exports.createEmbeddingProvider = require_embeddings.createEmbeddingProvider;
 exports.createExtractionContext = createExtractionContext;
+exports.createHttpEmbedding = require_embeddings.createHttpEmbedding;
+exports.createHuggingFaceEmbedding = require_embeddings.createHuggingFaceEmbedding;
+exports.createOllamaEmbedding = require_embeddings.createOllamaEmbedding;
+exports.createOpenAIEmbedding = require_embeddings.createOpenAIEmbedding;
+exports.createPiiRedactor = require_embeddings.createPiiRedactor;
+exports.createTransformersEmbedding = require_embeddings.createTransformersEmbedding;
 exports.defaultFetcher = defaultFetcher;
+exports.discoverFeeds = discoverFeeds;
+exports.embed = require_embeddings.embed;
+exports.embedScrapedData = require_embeddings.embedScrapedData;
+exports.estimateTokens = require_embeddings.estimateTokens;
 exports.extractDomain = extractDomain;
+exports.feedToMarkdown = feedToMarkdown;
+exports.feedToText = feedToText;
+exports.fetchFeed = fetchFeed;
+exports.filterByDate = filterByDate;
+exports.generateEmbeddings = require_embeddings.generateEmbeddings;
 exports.getPath = getPath;
 exports.getProtocol = getProtocol;
 exports.isExternalUrl = isExternalUrl;
@@ -845,6 +963,8 @@ exports.isValidUrl = isValidUrl;
 exports.matchesUrlPattern = matchesUrlPattern;
 exports.mergeResults = mergeResults;
 exports.normalizeUrl = normalizeUrl;
+exports.paginateFeed = paginateFeed;
+exports.redactPii = require_embeddings.redactPii;
 exports.resolveUrl = resolveUrl;
 exports.scrape = scrape;
 exports.scrapeHtml = scrapeHtml;