scrapex 0.5.2 → 1.0.0-alpha.1

package/dist/index.d.mts

@@ -0,0 +1,264 @@
+import { a as ExtractedLink, c as ExtractionSchemaType, d as FetchResult, f as Fetcher, h as ScrapedData, i as ExtractedEntities, l as Extractor, m as ScrapeOptions, n as ContentType, o as ExtractionContext, p as LLMProvider, r as EnhancementType, s as ExtractionSchema, t as CompletionOptions, u as FetchOptions } from "./types-CNQZVW36.mjs";
+
+//#region src/core/context.d.ts
+
+/**
+ * Create an extraction context with lazy JSDOM loading.
+ *
+ * Cheerio is always available for fast DOM queries.
+ * JSDOM is only loaded when getDocument() is called (for Readability).
+ */
+declare function createExtractionContext(url: string, finalUrl: string, html: string, options: ScrapeOptions): ExtractionContext;
+/**
+ * Merge partial results into the context
+ */
+declare function mergeResults(context: ExtractionContext, extracted: Partial<ScrapedData>): ExtractionContext;
+//#endregion
+//#region src/core/errors.d.ts
+/**
+ * Error codes for scraping failures
+ */
+type ScrapeErrorCode = 'FETCH_FAILED' | 'TIMEOUT' | 'INVALID_URL' | 'BLOCKED' | 'NOT_FOUND' | 'ROBOTS_BLOCKED' | 'PARSE_ERROR' | 'LLM_ERROR' | 'VALIDATION_ERROR';
+/**
+ * Custom error class for scraping failures with structured error codes
+ */
+declare class ScrapeError extends Error {
+  readonly code: ScrapeErrorCode;
+  readonly statusCode?: number;
+  constructor(message: string, code: ScrapeErrorCode, statusCode?: number, cause?: Error);
+  /**
+   * Create a ScrapeError from an unknown error
+   */
+  static from(error: unknown, code?: ScrapeErrorCode): ScrapeError;
+  /**
+   * Check if error is retryable (network issues, timeouts)
+   */
+  isRetryable(): boolean;
+  /**
+   * Convert to a plain object for serialization
+   */
+  toJSON(): Record<string, unknown>;
+}
+//#endregion
+//#region src/core/scrape.d.ts
+/**
+ * Scrape a URL and extract metadata and content.
+ *
+ * @param url - The URL to scrape
+ * @param options - Scraping options
+ * @returns Scraped data with metadata and content
+ *
+ * @example
+ * ```ts
+ * const result = await scrape('https://example.com/article');
+ * console.log(result.title, result.content);
+ * ```
+ */
+declare function scrape(url: string, options?: ScrapeOptions): Promise<ScrapedData>;
+/**
+ * Scrape from raw HTML string (no fetch).
+ *
+ * @param html - The HTML content
+ * @param url - The URL (for resolving relative links)
+ * @param options - Scraping options
+ * @returns Scraped data with metadata and content
+ *
+ * @example
+ * ```ts
+ * const html = await fetchSomehow('https://example.com');
+ * const result = await scrapeHtml(html, 'https://example.com');
+ * ```
+ */
+declare function scrapeHtml(html: string, url: string, options?: ScrapeOptions): Promise<ScrapedData>;
+//#endregion
+//#region src/extractors/content.d.ts
+/**
+ * Extracts main content using Mozilla Readability.
+ * Converts HTML to Markdown for LLM consumption.
+ */
+declare class ContentExtractor implements Extractor {
+  readonly name = "content";
+  readonly priority = 50;
+  extract(context: ExtractionContext): Promise<Partial<ScrapedData>>;
+  private extractFallback;
+  private createExcerpt;
+  private detectContentType;
+}
+//#endregion
+//#region src/extractors/favicon.d.ts
+/**
+ * Extracts favicon URL from the page.
+ * Checks multiple sources in order of preference.
+ */
+declare class FaviconExtractor implements Extractor {
+  readonly name = "favicon";
+  readonly priority = 70;
+  extract(context: ExtractionContext): Promise<Partial<ScrapedData>>;
+}
+//#endregion
+//#region src/extractors/jsonld.d.ts
+/**
+ * Extracts JSON-LD structured data from the page.
+ * Also extracts additional metadata from structured data.
+ */
+declare class JsonLdExtractor implements Extractor {
+  readonly name = "jsonld";
+  readonly priority = 80;
+  extract(context: ExtractionContext): Promise<Partial<ScrapedData>>;
+  private extractMetadata;
+  private getType;
+  private getString;
+  private getAuthor;
+  private getImage;
+  private getKeywords;
+}
+//#endregion
+//#region src/extractors/links.d.ts
+/**
+ * Extracts links from the page content.
+ * Filters out navigation/footer links and focuses on content links.
+ */
+declare class LinksExtractor implements Extractor {
+  readonly name = "links";
+  readonly priority = 30;
+  extract(context: ExtractionContext): Promise<Partial<ScrapedData>>;
+}
+//#endregion
+//#region src/extractors/meta.d.ts
+/**
+ * Extracts metadata from HTML meta tags, Open Graph, and Twitter cards.
+ * Runs first to provide basic metadata for other extractors.
+ */
+declare class MetaExtractor implements Extractor {
+  readonly name = "meta";
+  readonly priority = 100;
+  extract(context: ExtractionContext): Promise<Partial<ScrapedData>>;
+}
+//#endregion
+//#region src/extractors/index.d.ts
+/**
+ * Default extractors in priority order.
+ * Higher priority runs first.
+ */
+declare function createDefaultExtractors(): Extractor[];
+/**
+ * Sort extractors by priority (higher first).
+ */
+declare function sortExtractors(extractors: Extractor[]): Extractor[];
+//#endregion
+//#region src/fetchers/types.d.ts
+/**
+ * Fetcher interface - allows swapping fetch implementation
+ * for Puppeteer, Playwright, or custom solutions
+ */
+interface Fetcher$1 {
+  /**
+   * Fetch HTML from a URL
+   * @returns HTML content and final URL (after redirects)
+   */
+  fetch(url: string, options?: FetchOptions$1): Promise<FetchResult$1>;
+  /** Fetcher name for logging */
+  readonly name: string;
+}
+/**
+ * Options for fetching
+ */
+interface FetchOptions$1 {
+  /** Timeout in milliseconds (default: 10000) */
+  timeout?: number;
+  /** User agent string */
+  userAgent?: string;
+  /** Additional headers to send */
+  headers?: Record<string, string>;
+}
+/**
+ * Result from fetching a URL
+ */
+interface FetchResult$1 {
+  /** Raw HTML content */
+  html: string;
+  /** Final URL after redirects */
+  finalUrl: string;
+  /** HTTP status code */
+  statusCode: number;
+  /** Content-Type header */
+  contentType: string;
+  /** Response headers (optional) */
+  headers?: Record<string, string>;
+}
+/**
+ * Default user agent string
+ */
+declare const DEFAULT_USER_AGENT = "Scrapex-Bot/2.0 (+https://github.com/developer-rakeshpaul/scrapex)";
+/**
+ * Default timeout in milliseconds
+ */
+declare const DEFAULT_TIMEOUT = 10000;
+//#endregion
+//#region src/fetchers/fetch.d.ts
+/**
+ * Default fetcher using native fetch API.
+ * Works in Node.js 18+ without polyfills.
+ */
+declare class NativeFetcher implements Fetcher$1 {
+  readonly name = "native-fetch";
+  fetch(url: string, options?: FetchOptions$1): Promise<FetchResult$1>;
+}
+/**
+ * Default fetcher instance
+ */
+declare const defaultFetcher: NativeFetcher;
+//#endregion
+//#region src/fetchers/robots.d.ts
+/**
+ * Result of robots.txt check
+ */
+interface RobotsCheckResult {
+  allowed: boolean;
+  reason?: string;
+}
+/**
+ * Check if URL is allowed by robots.txt
+ *
+ * @param url - The URL to check
+ * @param userAgent - User agent to check rules for
+ * @returns Whether the URL is allowed and optional reason
+ */
+declare function checkRobotsTxt(url: string, userAgent?: string): Promise<RobotsCheckResult>;
+//#endregion
+//#region src/utils/url.d.ts
+/**
+ * Validate if a string is a valid URL
+ */
+declare function isValidUrl(url: string): boolean;
+/**
+ * Normalize URL by removing tracking params and trailing slashes
+ */
+declare function normalizeUrl(url: string): string;
+/**
+ * Extract domain from URL (without www prefix)
+ */
+declare function extractDomain(url: string): string;
+/**
+ * Resolve a potentially relative URL against a base URL
+ */
+declare function resolveUrl(url: string | undefined | null, baseUrl: string): string | undefined;
+/**
+ * Check if a URL is external relative to a domain
+ */
+declare function isExternalUrl(url: string, baseDomain: string): boolean;
+/**
+ * Extract protocol from URL
+ */
+declare function getProtocol(url: string): string;
+/**
+ * Get the path portion of a URL
+ */
+declare function getPath(url: string): string;
+/**
+ * Check if URL matches a pattern (supports * wildcard)
+ */
+declare function matchesUrlPattern(url: string, pattern: string): boolean;
+//#endregion
+export { type CompletionOptions, ContentExtractor, type ContentType, DEFAULT_TIMEOUT, DEFAULT_USER_AGENT, type EnhancementType, type ExtractedEntities, type ExtractedLink, type ExtractionContext, type ExtractionSchema, type ExtractionSchemaType, type Extractor, FaviconExtractor, type FetchOptions, type FetchResult, type Fetcher, JsonLdExtractor, type LLMProvider, LinksExtractor, MetaExtractor, NativeFetcher, type RobotsCheckResult, ScrapeError, type ScrapeErrorCode, type ScrapeOptions, type ScrapedData, checkRobotsTxt, createDefaultExtractors, createExtractionContext, defaultFetcher, extractDomain, getPath, getProtocol, isExternalUrl, isValidUrl, matchesUrlPattern, mergeResults, normalizeUrl, resolveUrl, scrape, scrapeHtml, sortExtractors };
+//# sourceMappingURL=index.d.mts.map

package/dist/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","names":[],"sources":["../src/core/context.ts","../src/core/errors.ts","../src/core/scrape.ts","../src/extractors/content.ts","../src/extractors/favicon.ts","../src/extractors/jsonld.ts","../src/extractors/links.ts","../src/extractors/meta.ts","../src/extractors/index.ts","../src/fetchers/types.ts","../src/fetchers/fetch.ts","../src/fetchers/robots.ts","../src/utils/url.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;iBAsBgB,uBAAA,uDAIL,gBACR;;ACxBH;AAcA;AACwB,iBDyCR,YAAA,CCzCQ,OAAA,ED0Cb,iBC1Ca,EAAA,SAAA,ED2CX,OC3CW,CD2CH,WC3CG,CAAA,CAAA,ED4CrB,iBC5CqB;;;;;;ADIR,KCnBJ,eAAA,GDmB2B,cAI5B,GAAA,SAAA,GACR,aAAA,GAAiB,SAAA,GAAA,WAAA,GAAA,gBAAA,GAAA,aAAA,GAAA,WAAA,GAAA,kBAAA;AAgCpB;;;AAEa,cC5CA,WAAA,SAAoB,KAAA,CD4CpB;EACV,SAAA,IAAA,EC5CqB,eD4CrB;EAAiB,SAAA,UAAA,CAAA,EAAA,MAAA;qCCzCiB,8CAA8C;;;AAlBnF;EAca,OAAA,IAAA,CAAA,KAAY,EAAA,OAAA,EAAA,IAAA,CAAA,EAmBW,eAnBX,CAAA,EAmB8C,WAnB9C;EACD;;;EAkBY,WAAA,CAAA,CAAA,EAAA,OAAA;EAAmC;;;EAnBjC,MAAA,CAAA,CAAA,EAyC1B,MAzC0B,CAAA,MAAA,EAAA,OAAA,CAAA;;;;;;ADKtC;AAqCA;;;;;;;;;ACxDA;AAca,iBCIS,MAAA,CDJG,GAAA,EAAA,MAAA,EAAA,OAAA,CAAA,ECI0B,aDJ1B,CAAA,ECI+C,ODJ/C,CCIuD,WDJvD,CAAA;;;;;;;;;;;;ACIzB;;;AAAwE,iBAmJlD,UAAA,CAnJkD,IAAA,EAAA,MAAA,EAAA,GAAA,EAAA,MAAA,EAAA,OAAA,CAAA,EAsJ7D,aAtJ6D,CAAA,EAuJrE,OAvJqE,CAuJ7D,WAvJ6D,CAAA;;;;;AFCxE;AAqCA;AACW,cGvCE,gBAAA,YAA4B,SHuC9B,CAAA;EACU,SAAA,IAAA,GAAA,SAAA;EAAR,SAAA,QAAA,GAAA,EAAA;EACV,OAAA,CAAA,OAAA,EGrCsB,iBHqCtB,CAAA,EGrC0C,OHqC1C,CGrCkD,OHqClD,CGrC0D,WHqC1D,CAAA,CAAA;EAAiB,QAAA,eAAA;;;;;;;;AAxCpB;AAqCA;AACW,cIrDE,gBAAA,YAA4B,SJqD9B,CAAA;EACU,SAAA,IAAA,GAAA,SAAA;EAAR,SAAA,QAAA,GAAA,EAAA;EACV,OAAA,CAAA,OAAA,EInDsB,iBJmDtB,CAAA,EInD0C,OJmD1C,CInDkD,OJmDlD,CInD0D,WJmD1D,CAAA,CAAA;;;;;;AAxCH;AAqCA;AACW,cKtDE,eAAA,YAA2B,SLsD7B,CAAA;EACU,SAAA,IAAA,GAAA,QAAA;EAAR,SAAA,QAAA,GAAA,EAAA;EACV,OAAA,CAAA,OAAA,EKpDsB,iBLoDtB,CAAA,EKpD0C,OLoD1C,CKpDkD,OLoDlD,CKpD0D,WLoD1D,CAAA,CAAA;EAAiB,QAAA,eAAA;;;;EC3DR,QAAA,QAAA;EAcC,QAAA,WAAY;;;;;;ADKzB;AAqCA;AACW,cMrDE,cAAA,YAA0B,SNqD5B,CAAA;EACU,SAAA,IAAA,GAAA,OAAA;EAAR,SAAA,QAAA,GAAA,EAAA;EACV,OAAA,CAAA,OAAA,EMnDsB,iBNmDtB,CAAA,EMnD0C,ONmD1C,CMnDkD,ONmDlD,CMnD0D,WNmD1D,CAAA,CAAA;;;;;;AAxCH;AAqCA;AACW,cOtDE,aAAA,YAAyB,SPsD3B,CAAA;EACU,SAAA,IAAA,GAAA,MAAA;EAAR,SAAA,QAAA,GAAA,GAAA;EACV,OAAA,CAAA,OAAA,EOpDsB,iBPoDtB,CAAA,EOpD0C,OPoD1C,COpDkD,OPoDlD,COpD0D,WPoD1D,CAAA,CAAA;;;;;;;;iBQ5Ca,uBAAA,CAAA,GAA2B;;;APf3C;AAca,iBOcG,cAAA,CPdS,UAAA,EOckB,SPdlB,EAAA,CAAA,EOcgC,SPdhC,EAAA;;;;;;ADKzB;AAqCgB,USvDC,SAAA,CTuDW;EACjB;;;;EAES,KAAA,CAAA,GAAA,EAAA,MAAA,EAAA,OAAA,CAAA,ESrDW,cTqDX,CAAA,ESrD0B,OTqD1B,CSrDkC,aTqDlC,CAAA;;;;AC3DpB;AAcA;;AAIqC,UQHpB,cAAA,CRGoB;EAA8C;EAe/C,OAAA,CAAA,EAAA,MAAA;EAAmC;EAsB3D,SAAA,CAAA,EAAA,MAAA;EAzCqB;EAAK,OAAA,CAAA,EQS1B,MRT0B,CAAA,MAAA,EAAA,MAAA,CAAA;;;;ACItC;AAAmD,UOWlC,aAAA,CPXkC;EAA6B;EAAR,IAAA,EAAA,MAAA;EAAO;EAmJzD,QAAA,EAAA,MAAU;EAGrB;EACA,UAAA,EAAA,MAAA;EAAR;EAAO,WAAA,EAAA,MAAA;;YO9HE;;ANzBZ;;;AAIqD,cM2BxC,kBAAA,GN3BwC,oEAAA;;;;cMiCxC,eAAA;;;;;ATpCb;AAqCA;AACW,cU/CE,aAAA,YAAyB,SV+C3B,CAAA;EACU,SAAA,IAAA,GAAA,cAAA;EAAR,KAAA,CAAA,GAAA,EAAA,MAAA,EAAA,OAAA,CAAA,EU7CuB,cV6CvB,CAAA,EU7C2C,OV6C3C,CU7CmD,aV6CnD,CAAA;;;;;cUuDA,gBAAc;;;;;;AV9FX,UWjBC,iBAAA,CXiBsB;EAqCvB,OAAA,EAAA,OAAY;EACjB,MAAA,CAAA,EAAA,MAAA;;;;;;;;ACzDX;AAca,iBUQS,cAAA,CVRG,GAAA,EAAA,MAAA,EAAA,SAAA,CAAA,EAAA,MAAA,CAAA,EUWtB,OVXsB,CUWd,iBVXc,CAAA;;;;;;ADKT,iBYKA,UAAA,CZLuB,GAAA,EAAA,MAI5B,CAAA,EAAA,OAAA;AAiCX;;;AAEa,iBYtBG,YAAA,CZsBH,GAAA,EAAA,MAAA,CAAA,EAAA,MAAA;;;;iBYEG,aAAA;;AX5DhB;AAcA;AACwB,iBWyDR,UAAA,CXzDQ,GAAA,EAAA,MAAA,GAAA,SAAA,GAAA,IAAA,EAAA,OAAA,EAAA,MAAA,CAAA,EAAA,MAAA,GAAA,SAAA;;;;AAkB+C,iBWoDvD,aAAA,CXpDuD,GAAA,EAAA,MAAA,EAAA,UAAA,EAAA,MAAA,CAAA,EAAA,OAAA;;;;iBWiEvD,WAAA;;;AVhFhB;AAAmD,iBU2FnC,OAAA,CV3FmC,GAAA,EAAA,MAAA,CAAA,EAAA,MAAA;;;;AAmJ7B,iBU7CN,iBAAA,CV6CgB,GAAA,EAAA,MAAA,EAAA,OAAA,EAAA,MAAA,CAAA,EAAA,OAAA"}