scrapex 0.5.2 → 1.0.0-alpha.1

package/dist/types-D0HYR95H.d.cts

@@ -0,0 +1,150 @@
+import { CheerioAPI } from "cheerio";
+
+//#region src/core/types.d.ts
+
+/**
+ * Content type classification for scraped URLs
+ */
+type ContentType = 'article' | 'repo' | 'docs' | 'package' | 'video' | 'tool' | 'product' | 'unknown';
+/**
+ * Extracted link from content
+ */
+interface ExtractedLink {
+  url: string;
+  text: string;
+  isExternal: boolean;
+}
+/**
+ * Extracted entities from LLM enhancement
+ */
+interface ExtractedEntities {
+  people: string[];
+  organizations: string[];
+  technologies: string[];
+  locations: string[];
+  concepts: string[];
+}
+/**
+ * Main result of metadata scraping - optimized for LLM consumption
+ */
+interface ScrapedData {
+  url: string;
+  canonicalUrl: string;
+  domain: string;
+  title: string;
+  description: string;
+  image?: string;
+  favicon?: string;
+  content: string;
+  textContent: string;
+  excerpt: string;
+  wordCount: number;
+  author?: string;
+  publishedAt?: string;
+  modifiedAt?: string;
+  siteName?: string;
+  language?: string;
+  contentType: ContentType;
+  keywords: string[];
+  jsonLd?: Record<string, unknown>[];
+  links?: ExtractedLink[];
+  summary?: string;
+  suggestedTags?: string[];
+  entities?: ExtractedEntities;
+  extracted?: Record<string, unknown>;
+  custom?: Record<string, unknown>;
+  scrapedAt: string;
+  scrapeTimeMs: number;
+  error?: string;
+}
+/**
+ * LLM enhancement types
+ */
+type EnhancementType = 'summarize' | 'tags' | 'entities' | 'classify';
+/**
+ * Schema for structured LLM extraction
+ */
+type ExtractionSchemaType = 'string' | 'number' | 'boolean' | 'string[]' | 'number[]' | `${string}?`;
+type ExtractionSchema = Record<string, ExtractionSchemaType>;
+/**
+ * Forward declaration for LLM provider (defined in llm/types.ts)
+ */
+interface LLMProvider {
+  readonly name: string;
+  complete(prompt: string, options?: CompletionOptions): Promise<string>;
+  completeJSON<T>(prompt: string, schema: unknown): Promise<T>;
+}
+interface CompletionOptions {
+  maxTokens?: number;
+  temperature?: number;
+  systemPrompt?: string;
+}
+/**
+ * Forward declaration for Fetcher (defined in fetchers/types.ts)
+ */
+interface Fetcher {
+  readonly name: string;
+  fetch(url: string, options: FetchOptions): Promise<FetchResult>;
+}
+interface FetchOptions {
+  timeout?: number;
+  userAgent?: string;
+  headers?: Record<string, string>;
+}
+interface FetchResult {
+  html: string;
+  finalUrl: string;
+  statusCode: number;
+  contentType: string;
+  headers?: Record<string, string>;
+}
+/**
+ * Forward declaration for Extractor (defined in extractors/types.ts)
+ */
+interface Extractor {
+  readonly name: string;
+  readonly priority?: number;
+  extract(context: ExtractionContext): Promise<Partial<ScrapedData>>;
+}
+/**
+ * Shared context passed to all extractors
+ */
+interface ExtractionContext {
+  url: string;
+  finalUrl: string;
+  html: string;
+  $: CheerioAPI;
+  getDocument(): Document;
+  results: Partial<ScrapedData>;
+  options: ScrapeOptions;
+}
+/**
+ * Options for scraping
+ */
+interface ScrapeOptions {
+  /** Timeout in milliseconds (default: 10000) */
+  timeout?: number;
+  /** User agent string */
+  userAgent?: string;
+  /** Whether to extract full content (default: true) */
+  extractContent?: boolean;
+  /** Maximum content length in characters (default: 50000) */
+  maxContentLength?: number;
+  /** Custom fetcher (for Puppeteer/Playwright) */
+  fetcher?: Fetcher;
+  /** Custom extractors to run */
+  extractors?: Extractor[];
+  /** If true, only run custom extractors (replace defaults) */
+  replaceDefaultExtractors?: boolean;
+  /** Check robots.txt before scraping (default: false) */
+  respectRobots?: boolean;
+  /** LLM provider for enhancements */
+  llm?: LLMProvider;
+  /** LLM enhancement types to run */
+  enhance?: EnhancementType[];
+  /** Schema for structured LLM extraction */
+  extract?: ExtractionSchema;
+}
+//#endregion
+export { ExtractedLink as a, ExtractionSchemaType as c, FetchResult as d, Fetcher as f, ScrapedData as h, ExtractedEntities as i, Extractor as l, ScrapeOptions as m, ContentType as n, ExtractionContext as o, LLMProvider as p, EnhancementType as r, ExtractionSchema as s, CompletionOptions as t, FetchOptions as u };
+//# sourceMappingURL=types-D0HYR95H.d.cts.map

package/dist/types-D0HYR95H.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types-D0HYR95H.d.cts","names":[],"sources":["../src/core/types.ts"],"sourcesContent":[],"mappings":";;;;;;AAKA;AAaiB,KAbL,WAAA,GAakB,SAAA,GAAA,MAAA,GAAA,MAAA,GAAA,SAAA,GAAA,OAAA,GAAA,MAAA,GAAA,SAAA,GAAA,SAAA;AAS9B;AAWA;;AA8BW,UAlDM,aAAA,CAkDN;EAGD,GAAA,EAAA,MAAA;EAKG,IAAA,EAAA,MAAA;EACC,UAAA,EAAA,OAAA;;;AAcd;AAKA;AAQY,UA7EK,iBAAA,CA6E6B;EAK7B,MAAA,EAAA,MAAW,EAAA;EAES,aAAA,EAAA,MAAA,EAAA;EAAoB,YAAA,EAAA,MAAA,EAAA;EACG,SAAA,EAAA,MAAA,EAAA;EAAR,QAAA,EAAA,MAAA,EAAA;;AAGpD;AASA;;AAEqD,UAxFpC,WAAA,CAwFoC;EAAR,GAAA,EAAA,MAAA;EAAO,YAAA,EAAA,MAAA;EAGnC,MAAA,EAAA,MAAA;EAMA,KAAA,EAAA,MAAA;EAWA,WAAA,EAAS,MAAA;EAGP,KAAA,CAAA,EAAA,MAAA;EAAoC,OAAA,CAAA,EAAA,MAAA;EAAR,OAAA,EAAA,MAAA;EAAR,WAAA,EAAA,MAAA;EAAO,OAAA,EAAA,MAAA;EAM7B,SAAA,EAAA,MAAA;EAOZ,MAAA,CAAA,EAAA,MAAA;EAGY,WAAA,CAAA,EAAA,MAAA;EAGE,UAAA,CAAA,EAAA,MAAA;EAAR,QAAA,CAAA,EAAA,MAAA;EAGA,QAAA,CAAA,EAAA,MAAA;EAAa,WAAA,EA3GT,WA2GS;EAMP,QAAA,EAAA,MAAa,EAAA;EAclB,MAAA,CAAA,EA3HD,MA2HC,CAAA,MAAA,EAAA,OAAA,CAAA,EAAA;EAGG,KAAA,CAAA,EA3HL,aA2HK,EAAA;EASP,OAAA,CAAA,EAAA,MAAA;EAGI,aAAA,CAAA,EAAA,MAAA,EAAA;EAGA,QAAA,CAAA,EArIC,iBAqID;EAAgB,SAAA,CAAA,EApId,MAoIc,CAAA,MAAA,EAAA,OAAA,CAAA;WAjIjB;;;;;;;;KAWC,eAAA;;;;KAKA,oBAAA;KAQA,gBAAA,GAAmB,eAAe;;;;UAK7B,WAAA;;qCAEoB,oBAAoB;oDACL,QAAQ;;UAG3C,iBAAA;;;;;;;;UASA,OAAA;;8BAEa,eAAe,QAAQ;;UAGpC,YAAA;;;YAGL;;UAGK,WAAA;;;;;YAKL;;;;;UAMK,SAAA;;;mBAGE,oBAAoB,QAAQ,QAAQ;;;;;UAMtC,iBAAA;;;;KAOZ;iBAGY;WAGN,QAAQ;WAGR;;;;;UAMM,aAAA;;;;;;;;;;YAcL;;eAGG;;;;;;QASP;;YAGI;;YAGA"}

package/package.json

@@ -1,119 +1,99 @@
 {
   "name": "scrapex",
-  "version": "0.5.2",
-  "main": "dist/index.js",
-  "module": "dist/scrapex.esm.js",
-  "typings": "dist/index.d.ts",
+  "version": "1.0.0-alpha.1",
+  "description": "Modern web scraper with LLM-enhanced extraction, extensible pipeline, and pluggable parsers",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.cjs"
+    },
+    "./parsers": {
+      "types": "./dist/parsers/index.d.mts",
+      "import": "./dist/parsers/index.mjs",
+      "require": "./dist/parsers/index.cjs"
+    },
+    "./llm": {
+      "types": "./dist/llm/index.d.mts",
+      "import": "./dist/llm/index.mjs",
+      "require": "./dist/llm/index.cjs"
+    }
+  },
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
   "files": [
     "dist"
   ],
   "scripts": {
-    "start": "tsdx watch",
-    "build": "tsdx build",
-    "test": "tsdx test",
-    "test:watch": "npm run test -- --watch",
-    "prepare": "npm run build",
-    "release": "npx np"
+    "dev": "tsdown --watch",
+    "build": "tsdown",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "type-check": "tsc --noEmit",
+    "lint": "biome lint ./src",
+    "lint:fix": "biome lint --write ./src",
+    "format": "biome format ./src",
+    "format:fix": "biome format --write ./src",
+    "check": "biome check ./src",
+    "check:fix": "biome check --write ./src",
+    "prepublishOnly": "npm run build"
   },
   "repository": {
     "type": "git",
     "url": "https://github.com/developer-rakeshpaul/scrapex"
   },
-  "husky": {
-    "hooks": {
-      "pre-commit": "pretty-quick --staged"
-    }
-  },
-  "np": {
-    "yarn": false,
-    "contents": "dist"
-  },
+  "keywords": [
+    "scraper",
+    "web-scraping",
+    "metadata",
+    "llm",
+    "extraction",
+    "readability",
+    "markdown",
+    "parser"
+  ],
+  "author": "Rakesh Paul <https://binaryroute.com/authors/rk-paul/>",
   "license": "MIT",
-  "prettier": {
-    "printWidth": 80,
-    "semi": true,
-    "singleQuote": true,
-    "trailingComma": "es5"
+  "engines": {
+    "node": ">=20"
   },
   "dependencies": {
-    "@metascraper/helpers": "^5.24.6",
-    "@mozilla/readability": "^0.4.1",
-    "@types/got": "^9.6.12",
-    "agentkeepalive": "^4.1.4",
-    "cheerio": "^1.0.0-rc.10",
-    "domino": "^2.1.6",
-    "got": "^11.8.2",
-    "jsdom": "^16.7.0",
-    "lodash.get": "^4.4.2",
-    "lodash.uniq": "^4.5.0",
-    "metascraper": "^5.24.6",
-    "metascraper-amazon": "^5.24.6",
-    "metascraper-audio": "^5.24.6",
-    "metascraper-author": "^5.24.6",
-    "metascraper-clearbit": "^5.24.6",
-    "metascraper-date": "^5.24.6",
-    "metascraper-description": "^5.24.6",
-    "metascraper-iframe": "^5.24.6",
-    "metascraper-image": "^5.24.6",
-    "metascraper-lang": "^5.24.6",
-    "metascraper-logo": "^5.24.6",
-    "metascraper-logo-favicon": "^5.24.6",
-    "metascraper-media-provider": "^5.24.6",
-    "metascraper-publisher": "^5.24.6",
-    "metascraper-readability": "^5.24.6",
-    "metascraper-soundcloud": "^5.24.6",
-    "metascraper-spotify": "^5.24.6",
-    "metascraper-title": "^5.24.6",
-    "metascraper-url": "^5.24.6",
-    "metascraper-video": "^5.24.6",
-    "metascraper-youtube": "^5.24.6",
-    "node-fetch": "^3.0.0",
-    "page-metadata-parser": "^1.1.4",
-    "robots-parser": "^2.3.0",
-    "sanitize-html": "^2.5.0",
-    "valid-url": "^1.0.9"
+    "@mozilla/readability": "^0.6.0",
+    "cheerio": "^1.1.2",
+    "jsdom": "^27.2.0",
+    "mdast-util-from-markdown": "^2.0.2",
+    "mdast-util-to-string": "^4.0.0",
+    "turndown": "^7.2.2",
+    "unist-util-visit": "^5.0.0",
+    "zod": "^4.1.13"
   },
   "devDependencies": {
-    "@types/cheerio": "^0.22.30",
-    "@types/jest": "^27.0.1",
-    "@types/jsdom": "^16.2.13",
-    "@types/lodash.get": "^4.4.6",
-    "@types/lodash.uniq": "^4.5.6",
-    "@types/metascraper": "^5.14.1",
-    "@types/metascraper-amazon": "^5.14.0",
-    "@types/metascraper-audio": "^5.14.0",
-    "@types/metascraper-author": "^5.14.0",
-    "@types/metascraper-clearbit": "^5.14.1",
-    "@types/metascraper-date": "^5.14.0",
-    "@types/metascraper-description": "^5.14.1",
-    "@types/metascraper-image": "^5.14.0",
-    "@types/metascraper-lang": "^5.14.0",
-    "@types/metascraper-logo": "^5.14.0",
-    "@types/metascraper-logo-favicon": "^5.14.1",
-    "@types/metascraper-media-provider": "^5.14.1",
-    "@types/metascraper-publisher": "^5.14.0",
-    "@types/metascraper-readability": "^5.14.0",
-    "@types/metascraper-soundcloud": "^5.14.0",
-    "@types/metascraper-spotify": "^5.14.0",
-    "@types/metascraper-title": "^5.14.0",
-    "@types/metascraper-url": "^5.14.0",
-    "@types/metascraper-video": "^5.14.0",
-    "@types/metascraper-youtube": "^5.14.0",
-    "@types/mozilla-readability": "^0.2.1",
-    "@types/node": "^16.9.1",
-    "@types/sanitize-html": "^2.3.2",
-    "@types/valid-url": "^1.0.3",
-    "husky": "^7.0.2",
-    "jest-extended": "^0.11.5",
-    "prettier": "^2.4.0",
-    "pretty-quick": "^3.1.1",
-    "tsdx": "^0.14.1",
-    "tslib": "^2.3.1",
-    "typescript": "^4.4.3"
+    "@biomejs/biome": "^2.3.8",
+    "@types/jsdom": "^27.0.0",
+    "@types/mdast": "^4.0.4",
+    "@types/node": "^22.10.0",
+    "@types/turndown": "^5.0.6",
+    "tsdown": "^0.17.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.15"
   },
-  "jest": {
-    "setupFilesAfterEnv": [
-      "jest-extended"
-    ]
+  "peerDependencies": {
+    "@anthropic-ai/sdk": ">=0.30.0",
+    "openai": ">=4.0.0",
+    "puppeteer": ">=23.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@anthropic-ai/sdk": {
+      "optional": true
+    },
+    "openai": {
+      "optional": true
+    },
+    "puppeteer": {
+      "optional": true
+    }
   }
 }

package/dist/index.d.ts

@@ -1,45 +0,0 @@
-/// <reference types="cheerio" />
-import { IOptions as SanitizeHtmlOptions } from 'sanitize-html';
-export interface ILink {
-    text?: string;
-    href?: string;
-}
-export interface IMetadata {
-    url: string;
-    date?: string;
-    image?: string;
-    publisher?: string;
-    title?: string;
-    author?: string;
-    description?: string;
-    audio?: string;
-    logo?: string;
-    lang?: string;
-    text?: string;
-    favicon?: string;
-    tags: Array<string>;
-    keywords: Array<string>;
-    links?: ILink[];
-    content?: string;
-    html?: string;
-    source: string;
-    video?: string;
-    code?: string[];
-    embeds?: Array<Record<string, string | undefined>>;
-    twitter: Record<string, string | undefined>;
-}
-export declare function getEmbedAttrs(el: cheerio.TagElement): {
-    src: string;
-    height: string;
-    width: string;
-    title: string;
-};
-declare type MetaScraperRules = 'audio' | 'amazon' | 'iframe' | 'media-provider' | 'soundcloud' | 'uol' | 'spotify' | 'video' | 'youtube';
-export declare type ScrapeOptions = {
-    timeout?: number;
-    metascraperRules?: Array<MetaScraperRules>;
-    sanitizeOptions?: SanitizeHtmlOptions;
-};
-export declare const scrape: (url: string, options?: ScrapeOptions | undefined) => Promise<IMetadata | null>;
-export declare const scrapeHtml: (url: string, html: string, options?: ScrapeOptions | undefined) => Promise<IMetadata | null>;
-export {};

package/dist/index.js

@@ -1,8 +0,0 @@
-
-'use strict'
-
-if (process.env.NODE_ENV === 'production') {
-  module.exports = require('./scrapex.cjs.production.min.js')
-} else {
-  module.exports = require('./scrapex.cjs.development.js')
-}