scrapex 1.0.0-alpha.1 → 1.0.0-beta.1

package/dist/parsers/index.d.mts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.mts","names":[],"sources":["../../src/parsers/types.ts","../../src/parsers/github.ts","../../src/parsers/markdown.ts"],"sourcesContent":[],"mappings":";;AAOA;;;;;AAiBA;AAQiB,UAzBA,YAyBY,CAAA,KAAA,EAAA,QAAA,OAAA,CAAA,CAAA;EAUZ,SAAA,IAAA,EAAA,MAAe;EAUf;;;EAKH,QAAA,CAAA,OAAA,EAAA,MAAA,EAAA,GAAA,CAAA,EAAA,MAAA,CAAA,EAAA,OAAA;EACE;;AAMhB;EASiB,KAAA,CAAA,OAAU,EAAA,MAAA,EAAA,GAAA,CAAA,EAAA,MAAA,CAAA,EAvDa,YAuDb,CAvD0B,KAuD1B,EAvDiC,KAuDjC,CAAA;;;;AChE3B;AAOgB,UDQC,YCRa,CAAA,KAAA,EAAA,QAAA,OAAA,CAAA,CAAA;EAYd,IAAA,EDHR,KCGgB;EAUF,IAAA,CAAA,EDZb,KCYa;AAgBtB;;;;AAA2D,UDtB1C,YAAA,CCsB0C;;;;EC7B9C,OAAA,CAAA,EAAA,MAAA;;;;;AAAsC,UFiBlC,eAAA,CEjBkC;EAqJnC,KAAA,EAAA,MAAA;EA+BA,KAAA,EAAA,MAAA;;SF/JP;;;;;UAMQ,cAAA;;;YAGL;SACH;cACK;gBACE;;;;;UAMC,SAAA;;;;;;;;UASA,UAAA;;;;;;;;AAlEjB;;;;;AAiBA;AAQiB,iBCvBD,YAAA,CDuBa,GAAA,EAAA,MAAA,CAAA,EAAA,OAAA;AAU7B;AAUA;;AAIS,iBCxCO,cAAA,CDwCP,GAAA,EAAA,MAAA,CAAA,EAAA;EACK,KAAA,EAAA,MAAA;EACE,IAAA,EAAA,MAAA;CAAM,GAAA,IAAA;AAMtB;AASA;;iBC7CgB,QAAA;;AAnBhB;AAOA;AAYA;AAUsB,iBAAA,aAAA,CAIX,KAAA,EAAR,MAAA,EAAO,IAAA,EAAA,MAAA,EAAA,MAAA,CAAA,EAAA,MAAA,CAAA,EAAP,OAAO,CAAC,UAAD,CAAA;AAYV;;;AAAwD,iBAAxC,eAAA,CAAwC,KAAA,EAAjB,YAAiB,EAAA,CAAA,EAAA,GAAA,CAAA,MAAA,EAAY,YAAZ,EAAA,CAAA;;;AD/CxD;;;;;AAiBA;AAQA;AAUA;AAUA;;;;AAMgB,cEjCH,cAAA,YAA0B,YFiCvB,CEjCoC,cFiCpC,CAAA,CAAA;EAAM,SAAA,IAAA,GAAA,UAAA;EAML,QAAA,CAAA,OAAS,EAAA,MAAA,CAAA,EAAA,OAAA;EAST,KAAA,CAAA,OAAU,EAAA,MAAA,CAAA,EElCD,YFkCC,CElCY,cFkCZ,CAAA;;;;AChE3B;AAOA;AAYA;AAUsB,iBCwIN,gBAAA,CDpIL,QAAR,EAAA,MAAO,CAAA,ECoI0C,YDpI1C,EAAA;AAYV;;;AAAwD,iBCuJxC,eAAA,CDvJwC,QAAA,EAAA,MAAA,EAAA,QAAA,CAAA,EAAA,MAAA,CAAA,ECuJS,eDvJT,EAAA"}

package/dist/parsers/index.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.mjs","names":["sections: MarkdownSection[]","allLinks: MarkdownLink[]","codeBlocks: CodeBlock[]","frontmatter: Record<string, unknown> | undefined","currentSection: MarkdownSection | null","mdastToString","linkData: MarkdownLink","result: Record<string, unknown>","value: string | boolean | number","links: MarkdownLink[]"],"sources":["../../src/parsers/github.ts","../../src/parsers/markdown.ts"],"sourcesContent":["import type { GitHubMeta, MarkdownLink } from './types.js';\n\n/**\n * GitHub-specific utilities for parsing repositories.\n */\n\n/**\n * Check if a URL is a GitHub repository\n */\nexport function isGitHubRepo(url: string): boolean {\n  return /^https?:\\/\\/(www\\.)?github\\.com\\/[^/]+\\/[^/]+\\/?$/.test(url);\n}\n\n/**\n * Extract GitHub repo info from URL\n */\nexport function parseGitHubUrl(url: string): { owner: string; repo: string } | null {\n  const match = url.match(/github\\.com\\/([^/]+)\\/([^/]+)/);\n  if (!match || !match[1] || !match[2]) return null;\n  return {\n    owner: match[1],\n    repo: match[2].replace(/\\.git$/, ''),\n  };\n}\n\n/**\n * Convert a GitHub repo URL to raw content URL\n */\nexport function toRawUrl(url: string, branch = 'main', file = 'README.md'): string {\n  const info = parseGitHubUrl(url);\n  if (!info) return url;\n  return `https://raw.githubusercontent.com/${info.owner}/${info.repo}/${branch}/${file}`;\n}\n\n/**\n * Fetch GitHub API metadata for a repository\n * Note: This is a placeholder - actual implementation would need GitHub API access\n */\nexport async function fetchRepoMeta(\n  owner: string,\n  repo: string,\n  _token?: string\n): Promise<GitHubMeta> {\n  // This would make actual API calls in a full implementation\n  // For now, return basic info\n  return {\n    repoOwner: owner,\n    repoName: repo,\n  };\n}\n\n/**\n * Group links by their category/section\n */\nexport function groupByCategory(links: MarkdownLink[]): Map<string, MarkdownLink[]> {\n  const groups = new Map<string, MarkdownLink[]>();\n\n  for (const link of links) {\n    const category = link.context || 'Uncategorized';\n    const existing = groups.get(category) || [];\n    existing.push(link);\n    groups.set(category, existing);\n  }\n\n  return groups;\n}\n","import type { Code, Heading, Link, ListItem, Root } from 'mdast';\nimport { fromMarkdown } from 'mdast-util-from-markdown';\nimport { toString as mdastToString } from 'mdast-util-to-string';\nimport { visit } from 'unist-util-visit';\nimport type {\n  CodeBlock,\n  MarkdownLink,\n  MarkdownSection,\n  ParsedMarkdown,\n  ParserResult,\n  SourceParser,\n} from './types.js';\n\n/**\n * Generic Markdown parser.\n * Extracts structure, links, and code blocks from markdown content.\n *\n * @example\n * ```ts\n * const parser = new MarkdownParser();\n * const result = parser.parse(markdownContent);\n * console.log(result.data.sections);\n * console.log(result.data.links);\n * ```\n */\nexport class MarkdownParser implements SourceParser<ParsedMarkdown> {\n  readonly name = 'markdown';\n\n  canParse(content: string): boolean {\n    // Check for common markdown patterns\n    return (\n      content.includes('# ') ||\n      content.includes('## ') ||\n      content.includes('- [') ||\n      content.includes('* [') ||\n      content.includes('```')\n    );\n  }\n\n  parse(content: string): ParserResult<ParsedMarkdown> {\n    const tree = fromMarkdown(content);\n    const sections: MarkdownSection[] = [];\n    const allLinks: MarkdownLink[] = [];\n    const codeBlocks: CodeBlock[] = [];\n    let frontmatter: Record<string, unknown> | undefined;\n\n    // Extract frontmatter if present\n    if (content.startsWith('---')) {\n      const endIndex = content.indexOf('---', 3);\n      if (endIndex !== -1) {\n        const frontmatterContent = content.slice(3, endIndex).trim();\n        frontmatter = this.parseFrontmatter(frontmatterContent);\n      }\n    }\n\n    // Track current section\n    let currentSection: MarkdownSection | null = null;\n\n    // Process the AST\n    visit(tree, (node) => {\n      // Handle headings\n      if (node.type === 'heading') {\n        const heading = node as Heading;\n        const title = mdastToString(heading);\n\n        // Finalize previous section\n        if (currentSection) {\n          sections.push(currentSection);\n        }\n\n        currentSection = {\n          level: heading.depth,\n          title,\n          content: '',\n          links: [],\n        };\n      }\n\n      // Handle links\n      if (node.type === 'link') {\n        const link = node as Link;\n        const text = mdastToString(link);\n        const linkData: MarkdownLink = {\n          url: link.url,\n          text,\n          title: link.title ?? undefined,\n          context: currentSection?.title,\n        };\n\n        allLinks.push(linkData);\n        if (currentSection) {\n          currentSection.links.push(linkData);\n        }\n      }\n\n      // Handle code blocks\n      if (node.type === 'code') {\n        const code = node as Code;\n        codeBlocks.push({\n          language: code.lang ?? undefined,\n          code: code.value,\n          meta: code.meta ?? undefined,\n        });\n      }\n\n      // Accumulate content for current section\n      if (currentSection && node.type === 'paragraph') {\n        const text = mdastToString(node);\n        currentSection.content += (currentSection.content ? '\\n\\n' : '') + text;\n      }\n    });\n\n    // Finalize last section\n    if (currentSection) {\n      sections.push(currentSection);\n    }\n\n    // Extract title from first h1 or frontmatter\n    const title = (frontmatter?.title as string) ?? sections.find((s) => s.level === 1)?.title;\n\n    // Extract description from frontmatter or first paragraph before any heading\n    const description = (frontmatter?.description as string) ?? this.extractDescription(tree);\n\n    return {\n      data: {\n        title,\n        description,\n        sections,\n        links: allLinks,\n        codeBlocks,\n        frontmatter,\n      },\n    };\n  }\n\n  private parseFrontmatter(content: string): Record<string, unknown> {\n    const result: Record<string, unknown> = {};\n    const lines = content.split('\\n');\n\n    for (const line of lines) {\n      const colonIndex = line.indexOf(':');\n      if (colonIndex > 0) {\n        const key = line.slice(0, colonIndex).trim();\n        let value: string | boolean | number = line.slice(colonIndex + 1).trim();\n\n        // Parse simple types\n        if (value === 'true') value = true;\n        else if (value === 'false') value = false;\n        else if (/^-?\\d+(\\.\\d+)?$/.test(value)) value = Number(value);\n        else if (value.startsWith('\"') && value.endsWith('\"')) value = value.slice(1, -1);\n        else if (value.startsWith(\"'\") && value.endsWith(\"'\")) value = value.slice(1, -1);\n\n        result[key] = value;\n      }\n    }\n\n    return result;\n  }\n\n  private extractDescription(tree: Root): string | undefined {\n    // Find first paragraph before any heading\n    for (const node of tree.children) {\n      if (node.type === 'heading') break;\n      if (node.type === 'paragraph') {\n        return mdastToString(node);\n      }\n    }\n    return undefined;\n  }\n}\n\n/**\n * Extract links from a list-based markdown structure (like awesome lists)\n */\nexport function extractListLinks(markdown: string): MarkdownLink[] {\n  const tree = fromMarkdown(markdown);\n  const links: MarkdownLink[] = [];\n  let currentHeading = '';\n\n  visit(tree, (node) => {\n    if (node.type === 'heading') {\n      currentHeading = mdastToString(node as Heading);\n    }\n\n    if (node.type === 'listItem') {\n      const listItem = node as ListItem;\n\n      // Find links in this list item\n      visit(listItem, 'link', (linkNode: Link) => {\n        links.push({\n          url: linkNode.url,\n          text: mdastToString(linkNode),\n          title: linkNode.title ?? undefined,\n          context: currentHeading || undefined,\n        });\n      });\n    }\n  });\n\n  return links;\n}\n\n/**\n * Parse markdown into sections by heading level\n */\nexport function parseByHeadings(markdown: string, minLevel = 2): MarkdownSection[] {\n  const parser = new MarkdownParser();\n  const result = parser.parse(markdown);\n  return result.data.sections.filter((s) => s.level >= minLevel);\n}\n"],"mappings":";;;;;;;;;;;AASA,SAAgB,aAAa,KAAsB;AACjD,QAAO,oDAAoD,KAAK,IAAI;;;;;AAMtE,SAAgB,eAAe,KAAqD;CAClF,MAAM,QAAQ,IAAI,MAAM,gCAAgC;AACxD,KAAI,CAAC,SAAS,CAAC,MAAM,MAAM,CAAC,MAAM,GAAI,QAAO;AAC7C,QAAO;EACL,OAAO,MAAM;EACb,MAAM,MAAM,GAAG,QAAQ,UAAU,GAAG;EACrC;;;;;AAMH,SAAgB,SAAS,KAAa,SAAS,QAAQ,OAAO,aAAqB;CACjF,MAAM,OAAO,eAAe,IAAI;AAChC,KAAI,CAAC,KAAM,QAAO;AAClB,QAAO,qCAAqC,KAAK,MAAM,GAAG,KAAK,KAAK,GAAG,OAAO,GAAG;;;;;;AAOnF,eAAsB,cACpB,OACA,MACA,QACqB;AAGrB,QAAO;EACL,WAAW;EACX,UAAU;EACX;;;;;AAMH,SAAgB,gBAAgB,OAAoD;CAClF,MAAM,yBAAS,IAAI,KAA6B;AAEhD,MAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,WAAW,KAAK,WAAW;EACjC,MAAM,WAAW,OAAO,IAAI,SAAS,IAAI,EAAE;AAC3C,WAAS,KAAK,KAAK;AACnB,SAAO,IAAI,UAAU,SAAS;;AAGhC,QAAO;;;;;;;;;;;;;;;;;ACvCT,IAAa,iBAAb,MAAoE;CAClE,AAAS,OAAO;CAEhB,SAAS,SAA0B;AAEjC,SACE,QAAQ,SAAS,KAAK,IACtB,QAAQ,SAAS,MAAM,IACvB,QAAQ,SAAS,MAAM,IACvB,QAAQ,SAAS,MAAM,IACvB,QAAQ,SAAS,MAAM;;CAI3B,MAAM,SAA+C;EACnD,MAAM,OAAO,aAAa,QAAQ;EAClC,MAAMA,WAA8B,EAAE;EACtC,MAAMC,WAA2B,EAAE;EACnC,MAAMC,aAA0B,EAAE;EAClC,IAAIC;AAGJ,MAAI,QAAQ,WAAW,MAAM,EAAE;GAC7B,MAAM,WAAW,QAAQ,QAAQ,OAAO,EAAE;AAC1C,OAAI,aAAa,IAAI;IACnB,MAAM,qBAAqB,QAAQ,MAAM,GAAG,SAAS,CAAC,MAAM;AAC5D,kBAAc,KAAK,iBAAiB,mBAAmB;;;EAK3D,IAAIC,iBAAyC;AAG7C,QAAM,OAAO,SAAS;AAEpB,OAAI,KAAK,SAAS,WAAW;IAC3B,MAAM,UAAU;IAChB,MAAM,QAAQC,SAAc,QAAQ;AAGpC,QAAI,eACF,UAAS,KAAK,eAAe;AAG/B,qBAAiB;KACf,OAAO,QAAQ;KACf;KACA,SAAS;KACT,OAAO,EAAE;KACV;;AAIH,OAAI,KAAK,SAAS,QAAQ;IACxB,MAAM,OAAO;IACb,MAAM,OAAOA,SAAc,KAAK;IAChC,MAAMC,WAAyB;KAC7B,KAAK,KAAK;KACV;KACA,OAAO,KAAK,SAAS;KACrB,SAAS,gBAAgB;KAC1B;AAED,aAAS,KAAK,SAAS;AACvB,QAAI,eACF,gBAAe,MAAM,KAAK,SAAS;;AAKvC,OAAI,KAAK,SAAS,QAAQ;IACxB,MAAM,OAAO;AACb,eAAW,KAAK;KACd,UAAU,KAAK,QAAQ;KACvB,MAAM,KAAK;KACX,MAAM,KAAK,QAAQ;KACpB,CAAC;;AAIJ,OAAI,kBAAkB,KAAK,SAAS,aAAa;IAC/C,MAAM,OAAOD,SAAc,KAAK;AAChC,mBAAe,YAAY,eAAe,UAAU,SAAS,MAAM;;IAErE;AAGF,MAAI,eACF,UAAS,KAAK,eAAe;AAS/B,SAAO,EACL,MAAM;GACJ,OAPW,aAAa,SAAoB,SAAS,MAAM,MAAM,EAAE,UAAU,EAAE,EAAE;GAQjF,aALiB,aAAa,eAA0B,KAAK,mBAAmB,KAAK;GAMrF;GACA,OAAO;GACP;GACA;GACD,EACF;;CAGH,AAAQ,iBAAiB,SAA0C;EACjE,MAAME,SAAkC,EAAE;EAC1C,MAAM,QAAQ,QAAQ,MAAM,KAAK;AAEjC,OAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,aAAa,KAAK,QAAQ,IAAI;AACpC,OAAI,aAAa,GAAG;IAClB,MAAM,MAAM,KAAK,MAAM,GAAG,WAAW,CAAC,MAAM;IAC5C,IAAIC,QAAmC,KAAK,MAAM,aAAa,EAAE,CAAC,MAAM;AAGxE,QAAI,UAAU,OAAQ,SAAQ;aACrB,UAAU,QAAS,SAAQ;aAC3B,kBAAkB,KAAK,MAAM,CAAE,SAAQ,OAAO,MAAM;aACpD,MAAM,WAAW,KAAI,IAAI,MAAM,SAAS,KAAI,CAAE,SAAQ,MAAM,MAAM,GAAG,GAAG;aACxE,MAAM,WAAW,IAAI,IAAI,MAAM,SAAS,IAAI,CAAE,SAAQ,MAAM,MAAM,GAAG,GAAG;AAEjF,WAAO,OAAO;;;AAIlB,SAAO;;CAGT,AAAQ,mBAAmB,MAAgC;AAEzD,OAAK,MAAM,QAAQ,KAAK,UAAU;AAChC,OAAI,KAAK,SAAS,UAAW;AAC7B,OAAI,KAAK,SAAS,YAChB,QAAOH,SAAc,KAAK;;;;;;;AAUlC,SAAgB,iBAAiB,UAAkC;CACjE,MAAM,OAAO,aAAa,SAAS;CACnC,MAAMI,QAAwB,EAAE;CAChC,IAAI,iBAAiB;AAErB,OAAM,OAAO,SAAS;AACpB,MAAI,KAAK,SAAS,UAChB,kBAAiBJ,SAAc,KAAgB;AAGjD,MAAI,KAAK,SAAS,WAIhB,OAHiB,MAGD,SAAS,aAAmB;AAC1C,SAAM,KAAK;IACT,KAAK,SAAS;IACd,MAAMA,SAAc,SAAS;IAC7B,OAAO,SAAS,SAAS;IACzB,SAAS,kBAAkB;IAC5B,CAAC;IACF;GAEJ;AAEF,QAAO;;;;;AAMT,SAAgB,gBAAgB,UAAkB,WAAW,GAAsB;AAGjF,QAFe,IAAI,gBAAgB,CACb,MAAM,SAAS,CACvB,KAAK,SAAS,QAAQ,MAAM,EAAE,SAAS,SAAS"}

package/dist/types-CNQZVW36.d.mts

@@ -1,150 +0,0 @@
-import { CheerioAPI } from "cheerio";
-
-//#region src/core/types.d.ts
-
-/**
- * Content type classification for scraped URLs
- */
-type ContentType = 'article' | 'repo' | 'docs' | 'package' | 'video' | 'tool' | 'product' | 'unknown';
-/**
- * Extracted link from content
- */
-interface ExtractedLink {
-  url: string;
-  text: string;
-  isExternal: boolean;
-}
-/**
- * Extracted entities from LLM enhancement
- */
-interface ExtractedEntities {
-  people: string[];
-  organizations: string[];
-  technologies: string[];
-  locations: string[];
-  concepts: string[];
-}
-/**
- * Main result of metadata scraping - optimized for LLM consumption
- */
-interface ScrapedData {
-  url: string;
-  canonicalUrl: string;
-  domain: string;
-  title: string;
-  description: string;
-  image?: string;
-  favicon?: string;
-  content: string;
-  textContent: string;
-  excerpt: string;
-  wordCount: number;
-  author?: string;
-  publishedAt?: string;
-  modifiedAt?: string;
-  siteName?: string;
-  language?: string;
-  contentType: ContentType;
-  keywords: string[];
-  jsonLd?: Record<string, unknown>[];
-  links?: ExtractedLink[];
-  summary?: string;
-  suggestedTags?: string[];
-  entities?: ExtractedEntities;
-  extracted?: Record<string, unknown>;
-  custom?: Record<string, unknown>;
-  scrapedAt: string;
-  scrapeTimeMs: number;
-  error?: string;
-}
-/**
- * LLM enhancement types
- */
-type EnhancementType = 'summarize' | 'tags' | 'entities' | 'classify';
-/**
- * Schema for structured LLM extraction
- */
-type ExtractionSchemaType = 'string' | 'number' | 'boolean' | 'string[]' | 'number[]' | `${string}?`;
-type ExtractionSchema = Record<string, ExtractionSchemaType>;
-/**
- * Forward declaration for LLM provider (defined in llm/types.ts)
- */
-interface LLMProvider {
-  readonly name: string;
-  complete(prompt: string, options?: CompletionOptions): Promise<string>;
-  completeJSON<T>(prompt: string, schema: unknown): Promise<T>;
-}
-interface CompletionOptions {
-  maxTokens?: number;
-  temperature?: number;
-  systemPrompt?: string;
-}
-/**
- * Forward declaration for Fetcher (defined in fetchers/types.ts)
- */
-interface Fetcher {
-  readonly name: string;
-  fetch(url: string, options: FetchOptions): Promise<FetchResult>;
-}
-interface FetchOptions {
-  timeout?: number;
-  userAgent?: string;
-  headers?: Record<string, string>;
-}
-interface FetchResult {
-  html: string;
-  finalUrl: string;
-  statusCode: number;
-  contentType: string;
-  headers?: Record<string, string>;
-}
-/**
- * Forward declaration for Extractor (defined in extractors/types.ts)
- */
-interface Extractor {
-  readonly name: string;
-  readonly priority?: number;
-  extract(context: ExtractionContext): Promise<Partial<ScrapedData>>;
-}
-/**
- * Shared context passed to all extractors
- */
-interface ExtractionContext {
-  url: string;
-  finalUrl: string;
-  html: string;
-  $: CheerioAPI;
-  getDocument(): Document;
-  results: Partial<ScrapedData>;
-  options: ScrapeOptions;
-}
-/**
- * Options for scraping
- */
-interface ScrapeOptions {
-  /** Timeout in milliseconds (default: 10000) */
-  timeout?: number;
-  /** User agent string */
-  userAgent?: string;
-  /** Whether to extract full content (default: true) */
-  extractContent?: boolean;
-  /** Maximum content length in characters (default: 50000) */
-  maxContentLength?: number;
-  /** Custom fetcher (for Puppeteer/Playwright) */
-  fetcher?: Fetcher;
-  /** Custom extractors to run */
-  extractors?: Extractor[];
-  /** If true, only run custom extractors (replace defaults) */
-  replaceDefaultExtractors?: boolean;
-  /** Check robots.txt before scraping (default: false) */
-  respectRobots?: boolean;
-  /** LLM provider for enhancements */
-  llm?: LLMProvider;
-  /** LLM enhancement types to run */
-  enhance?: EnhancementType[];
-  /** Schema for structured LLM extraction */
-  extract?: ExtractionSchema;
-}
-//#endregion
-export { ExtractedLink as a, ExtractionSchemaType as c, FetchResult as d, Fetcher as f, ScrapedData as h, ExtractedEntities as i, Extractor as l, ScrapeOptions as m, ContentType as n, ExtractionContext as o, LLMProvider as p, EnhancementType as r, ExtractionSchema as s, CompletionOptions as t, FetchOptions as u };
-//# sourceMappingURL=types-CNQZVW36.d.mts.map

package/dist/types-CNQZVW36.d.mts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types-CNQZVW36.d.mts","names":[],"sources":["../src/core/types.ts"],"sourcesContent":[],"mappings":";;;;;;AAKA;AAaiB,KAbL,WAAA,GAakB,SAAA,GAAA,MAAA,GAAA,MAAA,GAAA,SAAA,GAAA,OAAA,GAAA,MAAA,GAAA,SAAA,GAAA,SAAA;AAS9B;AAWA;;AA8BW,UAlDM,aAAA,CAkDN;EAGD,GAAA,EAAA,MAAA;EAKG,IAAA,EAAA,MAAA;EACC,UAAA,EAAA,OAAA;;;AAcd;AAKA;AAQY,UA7EK,iBAAA,CA6E6B;EAK7B,MAAA,EAAA,MAAW,EAAA;EAES,aAAA,EAAA,MAAA,EAAA;EAAoB,YAAA,EAAA,MAAA,EAAA;EACG,SAAA,EAAA,MAAA,EAAA;EAAR,QAAA,EAAA,MAAA,EAAA;;AAGpD;AASA;;AAEqD,UAxFpC,WAAA,CAwFoC;EAAR,GAAA,EAAA,MAAA;EAAO,YAAA,EAAA,MAAA;EAGnC,MAAA,EAAA,MAAA;EAMA,KAAA,EAAA,MAAA;EAWA,WAAA,EAAS,MAAA;EAGP,KAAA,CAAA,EAAA,MAAA;EAAoC,OAAA,CAAA,EAAA,MAAA;EAAR,OAAA,EAAA,MAAA;EAAR,WAAA,EAAA,MAAA;EAAO,OAAA,EAAA,MAAA;EAM7B,SAAA,EAAA,MAAA;EAOZ,MAAA,CAAA,EAAA,MAAA;EAGY,WAAA,CAAA,EAAA,MAAA;EAGE,UAAA,CAAA,EAAA,MAAA;EAAR,QAAA,CAAA,EAAA,MAAA;EAGA,QAAA,CAAA,EAAA,MAAA;EAAa,WAAA,EA3GT,WA2GS;EAMP,QAAA,EAAA,MAAa,EAAA;EAclB,MAAA,CAAA,EA3HD,MA2HC,CAAA,MAAA,EAAA,OAAA,CAAA,EAAA;EAGG,KAAA,CAAA,EA3HL,aA2HK,EAAA;EASP,OAAA,CAAA,EAAA,MAAA;EAGI,aAAA,CAAA,EAAA,MAAA,EAAA;EAGA,QAAA,CAAA,EArIC,iBAqID;EAAgB,SAAA,CAAA,EApId,MAoIc,CAAA,MAAA,EAAA,OAAA,CAAA;WAjIjB;;;;;;;;KAWC,eAAA;;;;KAKA,oBAAA;KAQA,gBAAA,GAAmB,eAAe;;;;UAK7B,WAAA;;qCAEoB,oBAAoB;oDACL,QAAQ;;UAG3C,iBAAA;;;;;;;;UASA,OAAA;;8BAEa,eAAe,QAAQ;;UAGpC,YAAA;;;YAGL;;UAGK,WAAA;;;;;YAKL;;;;;UAMK,SAAA;;;mBAGE,oBAAoB,QAAQ,QAAQ;;;;;UAMtC,iBAAA;;;;KAOZ;iBAGY;WAGN,QAAQ;WAGR;;;;;UAMM,aAAA;;;;;;;;;;YAcL;;eAGG;;;;;;QASP;;YAGI;;YAGA"}

package/dist/types-D0HYR95H.d.cts

@@ -1,150 +0,0 @@
-import { CheerioAPI } from "cheerio";
-
-//#region src/core/types.d.ts
-
-/**
- * Content type classification for scraped URLs
- */
-type ContentType = 'article' | 'repo' | 'docs' | 'package' | 'video' | 'tool' | 'product' | 'unknown';
-/**
- * Extracted link from content
- */
-interface ExtractedLink {
-  url: string;
-  text: string;
-  isExternal: boolean;
-}
-/**
- * Extracted entities from LLM enhancement
- */
-interface ExtractedEntities {
-  people: string[];
-  organizations: string[];
-  technologies: string[];
-  locations: string[];
-  concepts: string[];
-}
-/**
- * Main result of metadata scraping - optimized for LLM consumption
- */
-interface ScrapedData {
-  url: string;
-  canonicalUrl: string;
-  domain: string;
-  title: string;
-  description: string;
-  image?: string;
-  favicon?: string;
-  content: string;
-  textContent: string;
-  excerpt: string;
-  wordCount: number;
-  author?: string;
-  publishedAt?: string;
-  modifiedAt?: string;
-  siteName?: string;
-  language?: string;
-  contentType: ContentType;
-  keywords: string[];
-  jsonLd?: Record<string, unknown>[];
-  links?: ExtractedLink[];
-  summary?: string;
-  suggestedTags?: string[];
-  entities?: ExtractedEntities;
-  extracted?: Record<string, unknown>;
-  custom?: Record<string, unknown>;
-  scrapedAt: string;
-  scrapeTimeMs: number;
-  error?: string;
-}
-/**
- * LLM enhancement types
- */
-type EnhancementType = 'summarize' | 'tags' | 'entities' | 'classify';
-/**
- * Schema for structured LLM extraction
- */
-type ExtractionSchemaType = 'string' | 'number' | 'boolean' | 'string[]' | 'number[]' | `${string}?`;
-type ExtractionSchema = Record<string, ExtractionSchemaType>;
-/**
- * Forward declaration for LLM provider (defined in llm/types.ts)
- */
-interface LLMProvider {
-  readonly name: string;
-  complete(prompt: string, options?: CompletionOptions): Promise<string>;
-  completeJSON<T>(prompt: string, schema: unknown): Promise<T>;
-}
-interface CompletionOptions {
-  maxTokens?: number;
-  temperature?: number;
-  systemPrompt?: string;
-}
-/**
- * Forward declaration for Fetcher (defined in fetchers/types.ts)
- */
-interface Fetcher {
-  readonly name: string;
-  fetch(url: string, options: FetchOptions): Promise<FetchResult>;
-}
-interface FetchOptions {
-  timeout?: number;
-  userAgent?: string;
-  headers?: Record<string, string>;
-}
-interface FetchResult {
-  html: string;
-  finalUrl: string;
-  statusCode: number;
-  contentType: string;
-  headers?: Record<string, string>;
-}
-/**
- * Forward declaration for Extractor (defined in extractors/types.ts)
- */
-interface Extractor {
-  readonly name: string;
-  readonly priority?: number;
-  extract(context: ExtractionContext): Promise<Partial<ScrapedData>>;
-}
-/**
- * Shared context passed to all extractors
- */
-interface ExtractionContext {
-  url: string;
-  finalUrl: string;
-  html: string;
-  $: CheerioAPI;
-  getDocument(): Document;
-  results: Partial<ScrapedData>;
-  options: ScrapeOptions;
-}
-/**
- * Options for scraping
- */
-interface ScrapeOptions {
-  /** Timeout in milliseconds (default: 10000) */
-  timeout?: number;
-  /** User agent string */
-  userAgent?: string;
-  /** Whether to extract full content (default: true) */
-  extractContent?: boolean;
-  /** Maximum content length in characters (default: 50000) */
-  maxContentLength?: number;
-  /** Custom fetcher (for Puppeteer/Playwright) */
-  fetcher?: Fetcher;
-  /** Custom extractors to run */
-  extractors?: Extractor[];
-  /** If true, only run custom extractors (replace defaults) */
-  replaceDefaultExtractors?: boolean;
-  /** Check robots.txt before scraping (default: false) */
-  respectRobots?: boolean;
-  /** LLM provider for enhancements */
-  llm?: LLMProvider;
-  /** LLM enhancement types to run */
-  enhance?: EnhancementType[];
-  /** Schema for structured LLM extraction */
-  extract?: ExtractionSchema;
-}
-//#endregion
-export { ExtractedLink as a, ExtractionSchemaType as c, FetchResult as d, Fetcher as f, ScrapedData as h, ExtractedEntities as i, Extractor as l, ScrapeOptions as m, ContentType as n, ExtractionContext as o, LLMProvider as p, EnhancementType as r, ExtractionSchema as s, CompletionOptions as t, FetchOptions as u };
-//# sourceMappingURL=types-D0HYR95H.d.cts.map

package/dist/types-D0HYR95H.d.cts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types-D0HYR95H.d.cts","names":[],"sources":["../src/core/types.ts"],"sourcesContent":[],"mappings":";;;;;;AAKA;AAaiB,KAbL,WAAA,GAakB,SAAA,GAAA,MAAA,GAAA,MAAA,GAAA,SAAA,GAAA,OAAA,GAAA,MAAA,GAAA,SAAA,GAAA,SAAA;AAS9B;AAWA;;AA8BW,UAlDM,aAAA,CAkDN;EAGD,GAAA,EAAA,MAAA;EAKG,IAAA,EAAA,MAAA;EACC,UAAA,EAAA,OAAA;;;AAcd;AAKA;AAQY,UA7EK,iBAAA,CA6E6B;EAK7B,MAAA,EAAA,MAAW,EAAA;EAES,aAAA,EAAA,MAAA,EAAA;EAAoB,YAAA,EAAA,MAAA,EAAA;EACG,SAAA,EAAA,MAAA,EAAA;EAAR,QAAA,EAAA,MAAA,EAAA;;AAGpD;AASA;;AAEqD,UAxFpC,WAAA,CAwFoC;EAAR,GAAA,EAAA,MAAA;EAAO,YAAA,EAAA,MAAA;EAGnC,MAAA,EAAA,MAAA;EAMA,KAAA,EAAA,MAAA;EAWA,WAAA,EAAS,MAAA;EAGP,KAAA,CAAA,EAAA,MAAA;EAAoC,OAAA,CAAA,EAAA,MAAA;EAAR,OAAA,EAAA,MAAA;EAAR,WAAA,EAAA,MAAA;EAAO,OAAA,EAAA,MAAA;EAM7B,SAAA,EAAA,MAAA;EAOZ,MAAA,CAAA,EAAA,MAAA;EAGY,WAAA,CAAA,EAAA,MAAA;EAGE,UAAA,CAAA,EAAA,MAAA;EAAR,QAAA,CAAA,EAAA,MAAA;EAGA,QAAA,CAAA,EAAA,MAAA;EAAa,WAAA,EA3GT,WA2GS;EAMP,QAAA,EAAA,MAAa,EAAA;EAclB,MAAA,CAAA,EA3HD,MA2HC,CAAA,MAAA,EAAA,OAAA,CAAA,EAAA;EAGG,KAAA,CAAA,EA3HL,aA2HK,EAAA;EASP,OAAA,CAAA,EAAA,MAAA;EAGI,aAAA,CAAA,EAAA,MAAA,EAAA;EAGA,QAAA,CAAA,EArIC,iBAqID;EAAgB,SAAA,CAAA,EApId,MAoIc,CAAA,MAAA,EAAA,OAAA,CAAA;WAjIjB;;;;;;;;KAWC,eAAA;;;;KAKA,oBAAA;KAQA,gBAAA,GAAmB,eAAe;;;;UAK7B,WAAA;;qCAEoB,oBAAoB;oDACL,QAAQ;;UAG3C,iBAAA;;;;;;;;UASA,OAAA;;8BAEa,eAAe,QAAQ;;UAGpC,YAAA;;;YAGL;;UAGK,WAAA;;;;;YAKL;;;;;UAMK,SAAA;;;mBAGE,oBAAoB,QAAQ,QAAQ;;;;;UAMtC,iBAAA;;;;KAOZ;iBAGY;WAGN,QAAQ;WAGR;;;;;UAMM,aAAA;;;;;;;;;;YAcL;;eAGG;;;;;;QASP;;YAGI;;YAGA"}