scrapex 0.5.3 → 1.0.0-beta.1

package/package.json

@@ -1,119 +1,98 @@
 {
   "name": "scrapex",
-  "version": "0.5.3",
-  "main": "dist/index.js",
-  "module": "dist/scrapex.esm.js",
-  "typings": "dist/index.d.ts",
+  "version": "1.0.0-beta.1",
+  "description": "Modern web scraper with LLM-enhanced extraction, extensible pipeline, and pluggable parsers",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.cjs"
+    },
+    "./parsers": {
+      "types": "./dist/parsers/index.d.mts",
+      "import": "./dist/parsers/index.mjs",
+      "require": "./dist/parsers/index.cjs"
+    },
+    "./llm": {
+      "types": "./dist/llm/index.d.mts",
+      "import": "./dist/llm/index.mjs",
+      "require": "./dist/llm/index.cjs"
+    },
+    "./embeddings": {
+      "types": "./dist/embeddings/index.d.mts",
+      "import": "./dist/embeddings/index.mjs",
+      "require": "./dist/embeddings/index.cjs"
+    }
+  },
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
   "files": [
     "dist"
   ],
   "scripts": {
-    "start": "tsdx watch",
-    "build": "tsdx build",
-    "test": "tsdx test",
-    "test:watch": "npm run test -- --watch",
-    "prepare": "npm run build",
-    "release": "npx np"
+    "dev": "tsdown --watch",
+    "build": "tsdown",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "type-check": "tsc --noEmit",
+    "lint": "biome lint ./src",
+    "lint:fix": "biome lint --write ./src",
+    "format": "biome format ./src",
+    "format:fix": "biome format --write ./src",
+    "check": "biome check ./src",
+    "check:fix": "biome check --write ./src",
+    "prepublishOnly": "npm run build"
   },
   "repository": {
     "type": "git",
     "url": "https://github.com/developer-rakeshpaul/scrapex"
   },
-  "husky": {
-    "hooks": {
-      "pre-commit": "pretty-quick --staged"
-    }
-  },
-  "np": {
-    "yarn": false,
-    "contents": "dist"
-  },
+  "keywords": [
+    "scraper",
+    "web-scraping",
+    "metadata",
+    "llm",
+    "extraction",
+    "readability",
+    "markdown",
+    "parser",
+    "embeddings",
+    "vector-search"
+  ],
+  "author": "Rakesh Paul <https://binaryroute.com/authors/rk-paul/>",
   "license": "MIT",
-  "prettier": {
-    "printWidth": 80,
-    "semi": true,
-    "singleQuote": true,
-    "trailingComma": "es5"
+  "engines": {
+    "node": ">=20"
   },
   "dependencies": {
-    "@metascraper/helpers": "^5.24.6",
-    "@mozilla/readability": "^0.4.1",
-    "@types/got": "^9.6.12",
-    "agentkeepalive": "^4.1.4",
-    "cheerio": "^1.0.0-rc.10",
-    "domino": "^2.1.6",
-    "got": "^11.8.2",
-    "jsdom": "^16.7.0",
-    "lodash.get": "^4.4.2",
-    "lodash.uniq": "^4.5.0",
-    "metascraper": "^5.24.6",
-    "metascraper-amazon": "^5.24.6",
-    "metascraper-audio": "^5.24.6",
-    "metascraper-author": "^5.24.6",
-    "metascraper-clearbit": "^5.24.6",
-    "metascraper-date": "^5.24.6",
-    "metascraper-description": "^5.24.6",
-    "metascraper-iframe": "^5.24.6",
-    "metascraper-image": "^5.24.6",
-    "metascraper-lang": "^5.24.6",
-    "metascraper-logo": "^5.24.6",
-    "metascraper-logo-favicon": "^5.24.6",
-    "metascraper-media-provider": "^5.24.6",
-    "metascraper-publisher": "^5.24.6",
-    "metascraper-readability": "^5.24.6",
-    "metascraper-soundcloud": "^5.24.6",
-    "metascraper-spotify": "^5.24.6",
-    "metascraper-title": "^5.24.6",
-    "metascraper-url": "^5.24.6",
-    "metascraper-video": "^5.24.6",
-    "metascraper-youtube": "^5.24.6",
-    "node-fetch": "^3.0.0",
-    "page-metadata-parser": "^1.1.4",
-    "robots-parser": "^2.3.0",
-    "sanitize-html": "^2.5.0",
-    "valid-url": "^1.0.9"
+    "@mozilla/readability": "^0.6.0",
+    "cheerio": "^1.1.2",
+    "jsdom": "^27.4.0",
+    "mdast-util-from-markdown": "^2.0.2",
+    "mdast-util-to-string": "^4.0.0",
+    "turndown": "^7.2.2",
+    "unist-util-visit": "^5.0.0",
+    "zod": "^4.3.4"
   },
   "devDependencies": {
-    "@types/cheerio": "^0.22.30",
-    "@types/jest": "^27.0.1",
-    "@types/jsdom": "^16.2.13",
-    "@types/lodash.get": "^4.4.6",
-    "@types/lodash.uniq": "^4.5.6",
-    "@types/metascraper": "^5.14.1",
-    "@types/metascraper-amazon": "^5.14.0",
-    "@types/metascraper-audio": "^5.14.0",
-    "@types/metascraper-author": "^5.14.0",
-    "@types/metascraper-clearbit": "^5.14.1",
-    "@types/metascraper-date": "^5.14.0",
-    "@types/metascraper-description": "^5.14.1",
-    "@types/metascraper-image": "^5.14.0",
-    "@types/metascraper-lang": "^5.14.0",
-    "@types/metascraper-logo": "^5.14.0",
-    "@types/metascraper-logo-favicon": "^5.14.1",
-    "@types/metascraper-media-provider": "^5.14.1",
-    "@types/metascraper-publisher": "^5.14.0",
-    "@types/metascraper-readability": "^5.14.0",
-    "@types/metascraper-soundcloud": "^5.14.0",
-    "@types/metascraper-spotify": "^5.14.0",
-    "@types/metascraper-title": "^5.14.0",
-    "@types/metascraper-url": "^5.14.0",
-    "@types/metascraper-video": "^5.14.0",
-    "@types/metascraper-youtube": "^5.14.0",
-    "@types/mozilla-readability": "^0.2.1",
-    "@types/node": "^16.9.1",
-    "@types/sanitize-html": "^2.3.2",
-    "@types/valid-url": "^1.0.3",
-    "husky": "^7.0.2",
-    "jest-extended": "^0.11.5",
-    "prettier": "^2.4.0",
-    "pretty-quick": "^3.1.1",
-    "tsdx": "^0.14.1",
-    "tslib": "^2.3.1",
-    "typescript": "^4.4.3"
+    "@biomejs/biome": "^2.3.10",
+    "@types/jsdom": "^27.0.0",
+    "@types/mdast": "^4.0.4",
+    "@types/node": "^22.10.0",
+    "@types/turndown": "^5.0.6",
+    "tsdown": "^0.18.4",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.16"
   },
-  "jest": {
-    "setupFilesAfterEnv": [
-      "jest-extended"
-    ]
+  "peerDependencies": {
+    "puppeteer": "^24.34.0"
+  },
+  "peerDependenciesMeta": {
+    "puppeteer": {
+      "optional": true
+    }
   }
 }

package/dist/index.d.ts

@@ -1,45 +0,0 @@
-/// <reference types="cheerio" />
-import { IOptions as SanitizeHtmlOptions } from 'sanitize-html';
-export interface ILink {
-    text?: string;
-    href?: string;
-}
-export interface IMetadata {
-    url: string;
-    date?: string;
-    image?: string;
-    publisher?: string;
-    title?: string;
-    author?: string;
-    description?: string;
-    audio?: string;
-    logo?: string;
-    lang?: string;
-    text?: string;
-    favicon?: string;
-    tags: Array<string>;
-    keywords: Array<string>;
-    links?: ILink[];
-    content?: string;
-    html?: string;
-    source: string;
-    video?: string;
-    code?: string[];
-    embeds?: Array<Record<string, string | undefined>>;
-    twitter: Record<string, string | undefined>;
-}
-export declare function getEmbedAttrs(el: cheerio.TagElement): {
-    src: string;
-    height: string;
-    width: string;
-    title: string;
-};
-declare type MetaScraperRules = 'audio' | 'amazon' | 'iframe' | 'media-provider' | 'soundcloud' | 'uol' | 'spotify' | 'video' | 'youtube';
-export declare type ScrapeOptions = {
-    timeout?: number;
-    metascraperRules?: Array<MetaScraperRules>;
-    sanitizeOptions?: SanitizeHtmlOptions;
-};
-export declare const scrape: (url: string, options?: ScrapeOptions | undefined) => Promise<IMetadata | null>;
-export declare const scrapeHtml: (url: string, html: string, options?: ScrapeOptions | undefined) => Promise<IMetadata | null>;
-export {};

package/dist/index.js

@@ -1,8 +0,0 @@
-
-'use strict'
-
-if (process.env.NODE_ENV === 'production') {
-  module.exports = require('./scrapex.cjs.production.min.js')
-} else {
-  module.exports = require('./scrapex.cjs.development.js')
-}