webcontext-ai 1.0.0

package/dist/core/types.d.ts

@@ -0,0 +1,259 @@
+export interface SitemapEntry {
+    url: string;
+    lastmod?: string;
+    changefreq?: string;
+    priority?: number;
+}
+export interface RetryConfig {
+    maxRetries: number;
+    backoffMs: number;
+    backoffMultiplier: number;
+    retryOn: number[];
+}
+export interface RateLimitConfig {
+    requestsPerSecond: number;
+    burstSize: number;
+}
+export interface CrawlProgress {
+    pagesProcessed: number;
+    totalDiscovered: number;
+    currentUrl: string;
+    status: 'crawling' | 'paused' | 'complete' | 'error';
+}
+export interface PluginHook {
+    name: string;
+    phase: 'pre-fetch' | 'post-fetch' | 'pre-extract' | 'post-extract' | 'pre-transform' | 'post-transform' | 'pre-chunk' | 'post-chunk';
+}
+export interface WebContextPlugin {
+    name: string;
+    hooks: Record<string, (ctx: any) => Promise<any>>;
+}
+export interface CrawlOptions {
+    url: string;
+    depth?: number;
+    maxPages?: number;
+    includePatterns?: string[];
+    excludePatterns?: string[];
+    waitForSelector?: string;
+    timeout?: number;
+    headers?: Record<string, string>;
+    cookies?: Cookie[];
+    auth?: AuthConfig;
+    respectRobotsTxt?: boolean;
+    delay?: number;
+    javascript?: boolean;
+    focusMode?: FocusMode;
+    cache?: boolean;
+    cacheTTL?: number;
+    retry?: RetryConfig;
+    rateLimit?: RateLimitConfig;
+    sitemapUrl?: string;
+    checkpoint?: boolean;
+    checkpointDir?: string;
+    plugins?: WebContextPlugin[];
+    onProgress?: (progress: CrawlProgress) => void;
+}
+export interface Cookie {
+    name: string;
+    value: string;
+    domain: string;
+    path?: string;
+}
+export interface AuthConfig {
+    type: 'basic' | 'bearer' | 'cookie' | 'custom';
+    credentials: Record<string, string>;
+}
+export type FocusMode = 'full' | 'article' | 'code' | 'api' | 'readme' | 'section';
+export interface ExtractedContent {
+    url: string;
+    title: string;
+    description?: string;
+    markdown: string;
+    html?: string;
+    text: string;
+    codeBlocks: CodeBlock[];
+    headings: Heading[];
+    links: LinkInfo[];
+    metadata: PageMetadata;
+    timestamp: string;
+}
+export interface CodeBlock {
+    language: string;
+    code: string;
+    context?: string;
+    lineNumbers?: boolean;
+}
+export interface Heading {
+    level: number;
+    text: string;
+    id?: string;
+}
+export interface LinkInfo {
+    href: string;
+    text: string;
+    isInternal: boolean;
+}
+export interface PageMetadata {
+    author?: string;
+    publishedDate?: string;
+    modifiedDate?: string;
+    language?: string;
+    framework?: string;
+    library?: string;
+    tags?: string[];
+    ogImage?: string;
+    canonical?: string;
+    siteName?: string;
+    type?: ContentType;
+    version?: string;
+}
+export type ContentType = 'documentation' | 'api-reference' | 'blog-post' | 'readme' | 'tutorial' | 'article' | 'changelog' | 'unknown';
+export interface ChunkOptions {
+    maxTokens?: number;
+    overlap?: number;
+    strategy?: ChunkStrategy;
+    preserveCodeBlocks?: boolean;
+    preserveHeadings?: boolean;
+}
+export type ChunkStrategy = 'semantic' | 'fixed' | 'heading' | 'paragraph';
+export interface ContentChunk {
+    id: string;
+    content: string;
+    tokens: number;
+    metadata: ChunkMetadata;
+}
+export interface ChunkMetadata {
+    sourceUrl: string;
+    title: string;
+    headingPath: string[];
+    chunkIndex: number;
+    totalChunks: number;
+    hasCode: boolean;
+    language?: string;
+}
+export interface CrawlCheckpoint {
+    visitedUrls: string[];
+    pendingUrls: string[];
+    pages: ExtractedContent[];
+    errors: CrawlError[];
+    timestamp: string;
+}
+export interface EmbeddingResult {
+    id: string;
+    vector: number[];
+    content: string;
+    metadata: ChunkMetadata;
+}
+export interface SearchResult {
+    chunk: ContentChunk;
+    score: number;
+}
+export interface ContentDiff {
+    url: string;
+    previousHash: string;
+    currentHash: string;
+    changed: boolean;
+    addedSections: string[];
+    removedSections: string[];
+}
+export interface ScheduleConfig {
+    cron: string;
+    urls: string[];
+    options: Partial<CrawlOptions>;
+    onComplete?: (result: CrawlResult) => void;
+}
+export interface MetricsData {
+    crawlsTotal: number;
+    pagesTotal: number;
+    tokensTotal: number;
+    cacheHits: number;
+    cacheMisses: number;
+    avgDuration: number;
+    errors: number;
+}
+export interface ContextPacket {
+    id: string;
+    source: string;
+    chunks: ContentChunk[];
+    summary?: string;
+    totalTokens: number;
+    metadata: PacketMetadata;
+    format: OutputFormat;
+}
+export interface PacketMetadata {
+    crawledAt: string;
+    pageCount: number;
+    contentType: ContentType;
+    framework?: string;
+    version?: string;
+    relationships: PageRelationship[];
+}
+export interface PageRelationship {
+    from: string;
+    to: string;
+    type: 'links-to' | 'parent-of' | 'related-to' | 'next' | 'previous';
+}
+export type OutputFormat = 'markdown' | 'json' | 'chunks' | 'context-packet';
+export interface WebContextConfig {
+    browser?: BrowserConfig;
+    extraction?: ExtractionConfig;
+    chunking?: ChunkOptions;
+    output?: OutputConfig;
+    cache?: CacheConfig;
+    concurrency?: number;
+    retry?: RetryConfig;
+    rateLimit?: RateLimitConfig;
+    plugins?: WebContextPlugin[];
+    metrics?: boolean;
+}
+export interface BrowserConfig {
+    headless?: boolean;
+    proxy?: string;
+    userAgent?: string;
+    viewport?: {
+        width: number;
+        height: number;
+    };
+}
+export interface ExtractionConfig {
+    removeSelectors?: string[];
+    contentSelectors?: string[];
+    preserveImages?: boolean;
+    preserveTables?: boolean;
+    maxContentLength?: number;
+}
+export interface OutputConfig {
+    format: OutputFormat;
+    includeMetadata?: boolean;
+    includeSourceLinks?: boolean;
+    compressWhitespace?: boolean;
+}
+export interface CacheConfig {
+    enabled: boolean;
+    ttl: number;
+    maxSize: number;
+    directory?: string;
+    contentHashing?: boolean;
+}
+export interface CrawlResult {
+    pages: ExtractedContent[];
+    context: ContextPacket;
+    stats: CrawlStats;
+    diffs?: ContentDiff[];
+}
+export interface CrawlStats {
+    pagesProcessed: number;
+    totalTokens: number;
+    duration: number;
+    errors: CrawlError[];
+    cached: number;
+    cacheHits: number;
+    cacheMisses: number;
+    retries: number;
+}
+export interface CrawlError {
+    url: string;
+    error: string;
+    statusCode?: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/core/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,OAAO,CAAC;CACtD;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,WAAW,GAAG,YAAY,GAAG,aAAa,GAAG,cAAc,GAAG,eAAe,GAAG,gBAAgB,GAAG,WAAW,GAAG,YAAY,CAAC;CACtI;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC7B,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;CAChD;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC/C,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACrC;AAED,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEnF,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,QAAQ,EAAE,YAAY,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,WAAW,GACnB,eAAe,GACf,eAAe,GACf,WAAW,GACX,QAAQ,GACR,UAAU,GACV,SAAS,GACT,WAAW,GACX,SAAS,CAAC;AAEd,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,OAAO,GAAG,SAAS,GAAG,WAAW,CAAC;AAE3E,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,OAAO,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAC/B,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,CAAC;CAC5C;AAED,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,gBAAgB,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,YAAY,GAAG,MAAM,GAAG,UAAU,CAAC;CACrE;AAED,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,gBAAgB,CAAC;AAE7E,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9C;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,YAAY,CAAC;IACrB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,OAAO,EAAE,aAAa,CAAC;IACvB,KAAK,EAAE,UAAU,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,UAAU;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}

package/dist/core/types.js

@@ -0,0 +1,4 @@
+"use strict";
+// Core types for the webcontext package
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/core/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":";AAAA,wCAAwC"}

package/dist/export/index.d.ts

@@ -0,0 +1,3 @@
+export { VectorDBExporter, VectorDBExportOptions } from './vectordb';
+export { OutputFormatter, OutputTemplate } from './templates';
+//# sourceMappingURL=index.d.ts.map

package/dist/export/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/export/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC"}

package/dist/export/index.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OutputFormatter = exports.VectorDBExporter = void 0;
+var vectordb_1 = require("./vectordb");
+Object.defineProperty(exports, "VectorDBExporter", { enumerable: true, get: function () { return vectordb_1.VectorDBExporter; } });
+var templates_1 = require("./templates");
+Object.defineProperty(exports, "OutputFormatter", { enumerable: true, get: function () { return templates_1.OutputFormatter; } });
+//# sourceMappingURL=index.js.map

package/dist/export/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/export/index.ts"],"names":[],"mappings":";;;AAAA,uCAAqE;AAA5D,4GAAA,gBAAgB,OAAA;AACzB,yCAA8D;AAArD,4GAAA,eAAe,OAAA"}

package/dist/export/templates.d.ts

@@ -0,0 +1,25 @@
+import { ExtractedContent, ContextPacket } from '../core/types';
+export interface OutputTemplate {
+    name: string;
+    /** Template string with placeholders: {{title}}, {{url}}, {{markdown}}, {{chunks}}, {{summary}}, {{tokens}} */
+    template: string;
+}
+/**
+ * Custom output template engine.
+ * Formats extracted content using configurable templates.
+ */
+export declare class OutputFormatter {
+    private templates;
+    constructor();
+    /** Register a custom template */
+    register(template: OutputTemplate): void;
+    /** List available template names */
+    list(): string[];
+    /** Format a single page using a template */
+    formatPage(page: ExtractedContent, templateName?: string): string;
+    /** Format a full context packet */
+    formatContext(context: ContextPacket, templateName?: string): string;
+    /** Format multiple pages */
+    formatPages(pages: ExtractedContent[], templateName?: string, separator?: string): string;
+}
+//# sourceMappingURL=templates.d.ts.map

package/dist/export/templates.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"templates.d.ts","sourceRoot":"","sources":["../../src/export/templates.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,gBAAgB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE9E,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,+GAA+G;IAC/G,QAAQ,EAAE,MAAM,CAAC;CAClB;AAyBD;;;GAGG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,SAAS,CAA0C;;IAQ3D,iCAAiC;IACjC,QAAQ,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI;IAIxC,oCAAoC;IACpC,IAAI,IAAI,MAAM,EAAE;IAIhB,4CAA4C;IAC5C,UAAU,CAAC,IAAI,EAAE,gBAAgB,EAAE,YAAY,GAAE,MAAkB,GAAG,MAAM;IAY5E,mCAAmC;IACnC,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,YAAY,GAAE,MAAkB,GAAG,MAAM;IAa/E,4BAA4B;IAC5B,WAAW,CAAC,KAAK,EAAE,gBAAgB,EAAE,EAAE,YAAY,GAAE,MAAkB,EAAE,SAAS,GAAE,MAAsB,GAAG,MAAM;CAGpH"}

package/dist/export/templates.js

@@ -0,0 +1,76 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OutputFormatter = void 0;
+const BUILT_IN_TEMPLATES = {
+    default: {
+        name: 'default',
+        template: '# {{title}}\n\nSource: {{url}}\n\n{{markdown}}',
+    },
+    llm: {
+        name: 'llm',
+        template: '<context source="{{url}}" tokens="{{tokens}}">\n{{markdown}}\n</context>',
+    },
+    'xml-tags': {
+        name: 'xml-tags',
+        template: '<document>\n<title>{{title}}</title>\n<source>{{url}}</source>\n<content>\n{{markdown}}\n</content>\n</document>',
+    },
+    summary: {
+        name: 'summary',
+        template: '## {{title}}\n\n> {{summary}}\n\n**Source:** {{url}} | **Tokens:** {{tokens}}\n\n---\n\n{{markdown}}',
+    },
+    minimal: {
+        name: 'minimal',
+        template: '{{markdown}}',
+    },
+};
+/**
+ * Custom output template engine.
+ * Formats extracted content using configurable templates.
+ */
+class OutputFormatter {
+    templates = new Map();
+    constructor() {
+        for (const [key, tmpl] of Object.entries(BUILT_IN_TEMPLATES)) {
+            this.templates.set(key, tmpl);
+        }
+    }
+    /** Register a custom template */
+    register(template) {
+        this.templates.set(template.name, template);
+    }
+    /** List available template names */
+    list() {
+        return [...this.templates.keys()];
+    }
+    /** Format a single page using a template */
+    formatPage(page, templateName = 'default') {
+        const tmpl = this.templates.get(templateName);
+        if (!tmpl)
+            throw new Error(`Unknown template: ${templateName}. Available: ${this.list().join(', ')}`);
+        return tmpl.template
+            .replace(/\{\{title\}\}/g, page.title)
+            .replace(/\{\{url\}\}/g, page.url)
+            .replace(/\{\{markdown\}\}/g, page.markdown)
+            .replace(/\{\{summary\}\}/g, page.description || '')
+            .replace(/\{\{tokens\}\}/g, String(Math.ceil(page.markdown.length / 4)));
+    }
+    /** Format a full context packet */
+    formatContext(context, templateName = 'default') {
+        const tmpl = this.templates.get(templateName);
+        if (!tmpl)
+            throw new Error(`Unknown template: ${templateName}. Available: ${this.list().join(', ')}`);
+        return tmpl.template
+            .replace(/\{\{title\}\}/g, context.source)
+            .replace(/\{\{url\}\}/g, context.source)
+            .replace(/\{\{markdown\}\}/g, context.chunks.map(c => c.content).join('\n\n'))
+            .replace(/\{\{summary\}\}/g, context.summary || '')
+            .replace(/\{\{tokens\}\}/g, String(context.totalTokens))
+            .replace(/\{\{chunks\}\}/g, String(context.chunks.length));
+    }
+    /** Format multiple pages */
+    formatPages(pages, templateName = 'default', separator = '\n\n---\n\n') {
+        return pages.map(p => this.formatPage(p, templateName)).join(separator);
+    }
+}
+exports.OutputFormatter = OutputFormatter;
+//# sourceMappingURL=templates.js.map

package/dist/export/templates.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"templates.js","sourceRoot":"","sources":["../../src/export/templates.ts"],"names":[],"mappings":";;;AAQA,MAAM,kBAAkB,GAAmC;IACzD,OAAO,EAAE;QACP,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,gDAAgD;KAC3D;IACD,GAAG,EAAE;QACH,IAAI,EAAE,KAAK;QACX,QAAQ,EAAE,0EAA0E;KACrF;IACD,UAAU,EAAE;QACV,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE,kHAAkH;KAC7H;IACD,OAAO,EAAE;QACP,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,sGAAsG;KACjH;IACD,OAAO,EAAE;QACP,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,cAAc;KACzB;CACF,CAAC;AAEF;;;GAGG;AACH,MAAa,eAAe;IAClB,SAAS,GAAgC,IAAI,GAAG,EAAE,CAAC;IAE3D;QACE,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAAE,CAAC;YAC7D,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,QAAQ,CAAC,QAAwB;QAC/B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED,oCAAoC;IACpC,IAAI;QACF,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,4CAA4C;IAC5C,UAAU,CAAC,IAAsB,EAAE,eAAuB,SAAS;QACjE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,YAAY,gBAAgB,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEtG,OAAO,IAAI,CAAC,QAAQ;aACjB,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC;aACrC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC;aACjC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC,QAAQ,CAAC;aAC3C,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;aACnD,OAAO,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED,mCAAmC;IACnC,aAAa,CAAC,OAAsB,EAAE,eAAuB,SAAS;QACpE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,YAAY,gBAAgB,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEtG,OAAO,IAAI,CAAC,QAAQ;aACjB,OAAO,CAAC,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC;aACzC,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC;aACvC,OAAO,CAAC,mBAAmB,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAC7E,OAAO,CAAC,kBAAkB,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;aAClD,OAAO,CAAC,iBAAiB,EAAE,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;aACvD,OAAO,CAAC,iBAAiB,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,4BAA4B;IAC5B,WAAW,CAAC,KAAyB,EAAE,eAAuB,SAAS,EAAE,YAAoB,aAAa;QACxG,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC1E,CAAC;CACF;AAlDD,0CAkDC"}

package/dist/export/vectordb.d.ts

@@ -0,0 +1,21 @@
+import { ContentChunk } from '../core/types';
+export interface VectorDBExportOptions {
+    format: 'pinecone' | 'chroma' | 'weaviate' | 'qdrant' | 'json';
+    namespace?: string;
+    collection?: string;
+    includeMetadata?: boolean;
+}
+/**
+ * Export chunks in formats ready for vector database import.
+ * Generates JSON files compatible with each DB's bulk import API.
+ */
+export declare class VectorDBExporter {
+    exportChunks(chunks: ContentChunk[], options: VectorDBExportOptions): string;
+    exportToFile(chunks: ContentChunk[], options: VectorDBExportOptions, outputPath: string): void;
+    private toPinecone;
+    private toChroma;
+    private toWeaviate;
+    private toQdrant;
+    private toJSON;
+}
+//# sourceMappingURL=vectordb.d.ts.map

package/dist/export/vectordb.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectordb.d.ts","sourceRoot":"","sources":["../../src/export/vectordb.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAG7C,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAwBD;;;GAGG;AACH,qBAAa,gBAAgB;IAC3B,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,GAAG,MAAM;IAU5E,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAK9F,OAAO,CAAC,UAAU;IAmBlB,OAAO,CAAC,QAAQ;IAoBhB,OAAO,CAAC,UAAU;IAmBlB,OAAO,CAAC,QAAQ;IAkBhB,OAAO,CAAC,MAAM;CAGf"}

package/dist/export/vectordb.js

@@ -0,0 +1,101 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VectorDBExporter = void 0;
+const fs_1 = require("fs");
+/**
+ * Export chunks in formats ready for vector database import.
+ * Generates JSON files compatible with each DB's bulk import API.
+ */
+class VectorDBExporter {
+    exportChunks(chunks, options) {
+        switch (options.format) {
+            case 'pinecone': return this.toPinecone(chunks, options);
+            case 'chroma': return this.toChroma(chunks, options);
+            case 'weaviate': return this.toWeaviate(chunks, options);
+            case 'qdrant': return this.toQdrant(chunks, options);
+            case 'json':
+            default: return this.toJSON(chunks);
+        }
+    }
+    exportToFile(chunks, options, outputPath) {
+        const data = this.exportChunks(chunks, options);
+        (0, fs_1.writeFileSync)(outputPath, data);
+    }
+    toPinecone(chunks, options) {
+        const records = chunks.map(chunk => ({
+            id: chunk.id,
+            metadata: {
+                content: chunk.content,
+                source: chunk.metadata.sourceUrl,
+                title: chunk.metadata.title,
+                headingPath: chunk.metadata.headingPath.join(' > '),
+                chunkIndex: chunk.metadata.chunkIndex,
+                totalChunks: chunk.metadata.totalChunks,
+                hasCode: chunk.metadata.hasCode,
+                language: chunk.metadata.language || '',
+                tokens: chunk.tokens,
+                ...(options.namespace ? { namespace: options.namespace } : {}),
+            },
+        }));
+        return JSON.stringify({ vectors: records, namespace: options.namespace || '' }, null, 2);
+    }
+    toChroma(chunks, options) {
+        const records = chunks.map(chunk => ({
+            id: chunk.id,
+            document: chunk.content,
+            metadata: {
+                source: chunk.metadata.sourceUrl,
+                title: chunk.metadata.title,
+                headingPath: chunk.metadata.headingPath.join(' > '),
+                chunkIndex: chunk.metadata.chunkIndex,
+                hasCode: chunk.metadata.hasCode,
+                language: chunk.metadata.language || '',
+                tokens: chunk.tokens,
+            },
+        }));
+        return JSON.stringify({
+            collection: options.collection || 'webcontext',
+            documents: records,
+        }, null, 2);
+    }
+    toWeaviate(chunks, options) {
+        const records = chunks.map(chunk => ({
+            class: options.collection || 'WebContent',
+            properties: {
+                content: chunk.content,
+                source: chunk.metadata.sourceUrl,
+                title: chunk.metadata.title,
+                headingPath: chunk.metadata.headingPath.join(' > '),
+                chunkIndex: chunk.metadata.chunkIndex,
+                totalChunks: chunk.metadata.totalChunks,
+                hasCode: chunk.metadata.hasCode,
+                language: chunk.metadata.language || '',
+                tokens: chunk.tokens,
+                chunkId: chunk.id,
+            },
+        }));
+        return JSON.stringify(records, null, 2);
+    }
+    toQdrant(chunks, options) {
+        const records = chunks.map(chunk => ({
+            id: chunk.id,
+            payload: {
+                content: chunk.content,
+                source: chunk.metadata.sourceUrl,
+                title: chunk.metadata.title,
+                headingPath: chunk.metadata.headingPath.join(' > '),
+                chunkIndex: chunk.metadata.chunkIndex,
+                totalChunks: chunk.metadata.totalChunks,
+                hasCode: chunk.metadata.hasCode,
+                language: chunk.metadata.language || '',
+                tokens: chunk.tokens,
+            },
+        }));
+        return JSON.stringify({ collection: options.collection || 'webcontext', points: records }, null, 2);
+    }
+    toJSON(chunks) {
+        return JSON.stringify(chunks, null, 2);
+    }
+}
+exports.VectorDBExporter = VectorDBExporter;
+//# sourceMappingURL=vectordb.js.map

package/dist/export/vectordb.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectordb.js","sourceRoot":"","sources":["../../src/export/vectordb.ts"],"names":[],"mappings":";;;AACA,2BAAmC;AA+BnC;;;GAGG;AACH,MAAa,gBAAgB;IAC3B,YAAY,CAAC,MAAsB,EAAE,OAA8B;QACjE,QAAQ,OAAO,CAAC,MAAM,EAAE,CAAC;YACvB,KAAK,UAAU,CAAC,CAAC,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACzD,KAAK,QAAQ,CAAC,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACrD,KAAK,UAAU,CAAC,CAAC,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACzD,KAAK,QAAQ,CAAC,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACrD,KAAK,MAAM,CAAC;YAAC,OAAO,CAAC,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,YAAY,CAAC,MAAsB,EAAE,OAA8B,EAAE,UAAkB;QACrF,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAChD,IAAA,kBAAa,EAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAClC,CAAC;IAEO,UAAU,CAAC,MAAsB,EAAE,OAA8B;QACvE,MAAM,OAAO,GAAqB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACrD,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,QAAQ,EAAE;gBACR,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS;gBAChC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK;gBAC3B,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;gBACnD,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU;gBACrC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW;gBACvC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;gBAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE;gBACvC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC/D;SACF,CAAC,CAAC,CAAC;QACJ,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3F,CAAC;IAEO,QAAQ,CAAC,MAAsB,EAAE,OAA8B;QACrE,MAAM,OAAO,GAAmB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACnD,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,QAAQ,EAAE,KAAK,CAAC,OAAO;YACvB,QAAQ,EAAE;gBACR,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS;gBAChC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK;gBAC3B,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;gBACnD,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU;gBACrC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;gBAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE;gBACvC,MAAM,EAAE,KAAK,CAAC,MAAM;aACrB;SACF,CAAC,CAAC,CAAC;QACJ,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,YAAY;YAC9C,SAAS,EAAE,OAAO;SACnB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACd,CAAC;IAEO,UAAU,CAAC,MAAsB,EAAE,OAA8B;QACvE,MAAM,OAAO,GAAqB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACrD,KAAK,EAAE,OAAO,CAAC,UAAU,IAAI,YAAY;YACzC,UAAU,EAAE;gBACV,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS;gBAChC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK;gBAC3B,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;gBACnD,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU;gBACrC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW;gBACvC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;gBAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE;gBACvC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,KAAK,CAAC,EAAE;aAClB;SACF,CAAC,CAAC,CAAC;QACJ,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEO,QAAQ,CAAC,MAAsB,EAAE,OAA8B;QACrE,MAAM,OAAO,GAAmB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACnD,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,OAAO,EAAE;gBACP,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS;gBAChC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK;gBAC3B,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;gBACnD,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU;gBACrC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW;gBACvC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;gBAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE;gBACvC,MAAM,EAAE,KAAK,CAAC,MAAM;aACrB;SACF,CAAC,CAAC,CAAC;QACJ,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtG,CAAC;IAEO,MAAM,CAAC,MAAsB;QACnC,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACzC,CAAC;CACF;AA/FD,4CA+FC"}

package/dist/extractors/content.d.ts

@@ -0,0 +1,23 @@
+import * as cheerio from 'cheerio';
+import { ExtractedContent, CodeBlock, FocusMode } from '../core/types';
+/**
+ * Content extractor that cleans HTML and extracts structured content.
+ * Uses Readability algorithm + custom heuristics for developer content.
+ */
+export declare class ContentExtractor {
+    private static NOISE_SELECTORS;
+    private static CONTENT_SELECTORS;
+    extract(html: string, url: string, focusMode?: FocusMode): ExtractedContent;
+    private findContentElement;
+    private extractTitle;
+    private extractCodeBlocks;
+    private extractHeadings;
+    private extractLinks;
+    private extractMetadata;
+    private detectContentType;
+    private detectFramework;
+    private detectVersion;
+    extractOpenAPIEndpoints($: cheerio.CheerioAPI, container: cheerio.Cheerio<any>): CodeBlock[];
+    private detectLanguage;
+}
+//# sourceMappingURL=content.d.ts.map

package/dist/extractors/content.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../src/extractors/content.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAgD,SAAS,EAAE,MAAM,eAAe,CAAC;AAErH;;;GAGG;AACH,qBAAa,gBAAgB;IAE3B,OAAO,CAAC,MAAM,CAAC,eAAe,CAY5B;IAGF,OAAO,CAAC,MAAM,CAAC,iBAAiB,CAM9B;IAEF,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,GAAE,SAAkB,GAAG,gBAAgB;IA6CnF,OAAO,CAAC,kBAAkB;IA8C1B,OAAO,CAAC,YAAY;IAMpB,OAAO,CAAC,iBAAiB;IAoBzB,OAAO,CAAC,eAAe;IAavB,OAAO,CAAC,YAAY;IAsBpB,OAAO,CAAC,eAAe;IAkBvB,OAAO,CAAC,iBAAiB;IAYzB,OAAO,CAAC,eAAe;IAUvB,OAAO,CAAC,aAAa;IAYrB,uBAAuB,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,EAAE,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,EAAE;IAsC5F,OAAO,CAAC,cAAc;CAmCvB"}