webpeel 0.13.4 → 0.14.0

package/dist/core/youtube.js

@@ -0,0 +1,380 @@
+/**
+ * YouTube transcript extraction — no API key required.
+ *
+ * YouTube embeds caption/transcript data directly in the page HTML as JSON
+ * (inside ytInitialPlayerResponse). We parse that JSON, extract caption
+ * track URLs, fetch the timedtext XML, and return structured transcript data.
+ */
+import { simpleFetch } from './fetcher.js';
+// ---------------------------------------------------------------------------
+// URL parsing
+// ---------------------------------------------------------------------------
+/**
+ * Extract the video ID from any common YouTube URL format.
+ * Returns null if the URL is not a recognisable YouTube URL.
+ *
+ * Supported formats:
+ *   https://www.youtube.com/watch?v=VIDEO_ID
+ *   https://youtu.be/VIDEO_ID
+ *   https://www.youtube.com/embed/VIDEO_ID
+ *   https://m.youtube.com/watch?v=VIDEO_ID
+ *   URLs with extra params (&t=120, &list=PLxxx, etc.)
+ */
+export function parseYouTubeUrl(url) {
+    if (!url || typeof url !== 'string')
+        return null;
+    let parsed;
+    try {
+        parsed = new URL(url.trim());
+    }
+    catch {
+        return null;
+    }
+    const host = parsed.hostname.toLowerCase().replace(/^www\./, '').replace(/^m\./, '');
+    if (host === 'youtu.be') {
+        // https://youtu.be/VIDEO_ID
+        const id = parsed.pathname.slice(1).split('/')[0];
+        return isValidVideoId(id) ? id : null;
+    }
+    if (host === 'youtube.com') {
+        // /watch?v=VIDEO_ID
+        if (parsed.pathname === '/watch' || parsed.pathname === '/watch/') {
+            const id = parsed.searchParams.get('v');
+            return id && isValidVideoId(id) ? id : null;
+        }
+        // /embed/VIDEO_ID
+        if (parsed.pathname.startsWith('/embed/')) {
+            const id = parsed.pathname.split('/')[2];
+            return id && isValidVideoId(id) ? id : null;
+        }
+        // /shorts/VIDEO_ID
+        if (parsed.pathname.startsWith('/shorts/')) {
+            const id = parsed.pathname.split('/')[2];
+            return id && isValidVideoId(id) ? id : null;
+        }
+        // /v/VIDEO_ID (old embed format)
+        if (parsed.pathname.startsWith('/v/')) {
+            const id = parsed.pathname.split('/')[2];
+            return id && isValidVideoId(id) ? id : null;
+        }
+    }
+    return null;
+}
+function isValidVideoId(id) {
+    return typeof id === 'string' && /^[A-Za-z0-9_-]{11}$/.test(id);
+}
+// ---------------------------------------------------------------------------
+// Video info extraction
+// ---------------------------------------------------------------------------
+/**
+ * Extract video metadata from YouTube page HTML.
+ * Parses ytInitialPlayerResponse JSON embedded in the page.
+ */
+export function extractVideoInfo(html) {
+    const playerResponse = extractPlayerResponse(html);
+    const videoDetails = playerResponse?.videoDetails ?? {};
+    const microformat = playerResponse?.microformat?.playerMicroformatRenderer ?? {};
+    const videoId = videoDetails.videoId ?? '';
+    const title = videoDetails.title ??
+        microformat.title?.simpleText ??
+        extractMetaTag(html, 'og:title') ??
+        '';
+    const channel = videoDetails.author ?? microformat.ownerChannelName ?? '';
+    const lengthSeconds = parseInt(videoDetails.lengthSeconds ?? microformat.lengthSeconds ?? '0', 10);
+    const viewCount = videoDetails.viewCount ?? microformat.viewCount ?? '';
+    const publishDate = microformat.publishDate ?? microformat.uploadDate ?? '';
+    const description = videoDetails.shortDescription ??
+        microformat.description?.simpleText ??
+        extractMetaTag(html, 'og:description') ??
+        '';
+    const thumbnail = videoDetails.thumbnail?.thumbnails?.slice(-1)[0]?.url ??
+        microformat.thumbnail?.thumbnails?.slice(-1)[0]?.url ??
+        `https://img.youtube.com/vi/${videoId}/maxresdefault.jpg`;
+    // likeCount is often not available without auth
+    const likeCount = videoDetails.likeCount ?? '';
+    return {
+        videoId,
+        title,
+        channel,
+        description,
+        duration: formatDuration(lengthSeconds),
+        publishDate,
+        viewCount,
+        likeCount,
+        thumbnail,
+    };
+}
+// ---------------------------------------------------------------------------
+// Transcript extraction
+// ---------------------------------------------------------------------------
+/**
+ * Fetch and return the transcript for a YouTube video.
+ *
+ * @param url - Any YouTube URL format
+ * @param options.language - Preferred language code (default: "en")
+ */
+export async function getYouTubeTranscript(url, options = {}) {
+    const videoId = parseYouTubeUrl(url);
+    if (!videoId) {
+        throw new Error(`Not a valid YouTube URL: ${url}`);
+    }
+    const preferredLang = options.language ?? 'en';
+    // Fetch the video page
+    const videoUrl = `https://www.youtube.com/watch?v=${videoId}`;
+    const fetchResult = await simpleFetch(videoUrl, undefined, 30000);
+    const html = fetchResult.html;
+    // Extract player response
+    const playerResponse = extractPlayerResponse(html);
+    if (!playerResponse) {
+        throw new Error(`Could not parse YouTube page data for video ${videoId}`);
+    }
+    // Extract video info
+    const videoDetails = playerResponse.videoDetails ?? {};
+    const title = videoDetails.title ?? '';
+    const channel = videoDetails.author ?? '';
+    const lengthSeconds = parseInt(videoDetails.lengthSeconds ?? '0', 10);
+    // Extract caption tracks
+    const captionTracks = extractCaptionTracks(playerResponse);
+    if (captionTracks.length === 0) {
+        throw new Error(`No captions available for video ${videoId}`);
+    }
+    const availableLanguages = captionTracks.map(t => t.languageCode);
+    // Select best track: prefer manual over auto-generated, prefer requested language
+    const selectedTrack = selectBestTrack(captionTracks, preferredLang);
+    // Fetch the caption XML
+    const captionXml = await fetchCaptionXml(selectedTrack.baseUrl);
+    // Parse segments
+    const segments = parseCaptionXml(captionXml);
+    const fullText = segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim();
+    return {
+        videoId,
+        title,
+        channel,
+        duration: formatDuration(lengthSeconds),
+        language: selectedTrack.languageCode,
+        segments,
+        fullText,
+        availableLanguages,
+    };
+}
+/**
+ * Extract the ytInitialPlayerResponse JSON object from page HTML.
+ */
+export function extractPlayerResponse(html) {
+    // Try a few patterns YouTube uses to embed this data
+    const patterns = [
+        // Modern: var ytInitialPlayerResponse = {...};
+        /var ytInitialPlayerResponse\s*=\s*(\{.+?\});\s*(?:var|<\/script>)/s,
+        // Also try without trailing var (some pages end differently)
+        /ytInitialPlayerResponse\s*=\s*(\{.+?\})(?:;|\s*<\/script>)/s,
+    ];
+    for (const pattern of patterns) {
+        const match = html.match(pattern);
+        if (match) {
+            try {
+                return JSON.parse(match[1]);
+            }
+            catch {
+                // Try to find a valid JSON boundary by walking the string
+                const start = html.indexOf('ytInitialPlayerResponse');
+                if (start === -1)
+                    continue;
+                const braceStart = html.indexOf('{', start);
+                if (braceStart === -1)
+                    continue;
+                const jsonStr = extractJsonObject(html, braceStart);
+                if (jsonStr) {
+                    try {
+                        return JSON.parse(jsonStr);
+                    }
+                    catch {
+                        /* fall through to next pattern */
+                    }
+                }
+            }
+        }
+    }
+    // Fallback: search for captionTracks directly
+    const captionIdx = html.indexOf('"captionTracks"');
+    if (captionIdx !== -1) {
+        // Walk back to find the enclosing object
+        const braceStart = html.lastIndexOf('{', captionIdx);
+        if (braceStart !== -1) {
+            const jsonStr = extractJsonObject(html, braceStart);
+            if (jsonStr) {
+                try {
+                    return JSON.parse(jsonStr);
+                }
+                catch { /* ignore */ }
+            }
+        }
+    }
+    return null;
+}
+/**
+ * Extract a complete JSON object starting at position `start` in `str`.
+ * Handles nested objects/arrays and string literals.
+ */
+function extractJsonObject(str, start) {
+    let depth = 0;
+    let inString = false;
+    let escape = false;
+    for (let i = start; i < str.length; i++) {
+        const ch = str[i];
+        if (escape) {
+            escape = false;
+            continue;
+        }
+        if (ch === '\\' && inString) {
+            escape = true;
+            continue;
+        }
+        if (ch === '"') {
+            inString = !inString;
+            continue;
+        }
+        if (inString)
+            continue;
+        if (ch === '{' || ch === '[')
+            depth++;
+        else if (ch === '}' || ch === ']') {
+            depth--;
+            if (depth === 0) {
+                return str.slice(start, i + 1);
+            }
+        }
+    }
+    return null;
+}
+/**
+ * Extract caption tracks from the player response.
+ */
+function extractCaptionTracks(playerResponse) {
+    try {
+        const tracks = playerResponse?.captions?.playerCaptionsTracklistRenderer?.captionTracks;
+        if (!Array.isArray(tracks))
+            return [];
+        return tracks.map((t) => ({
+            baseUrl: t.baseUrl ?? '',
+            languageCode: (t.languageCode ?? 'unknown').toLowerCase(),
+            name: t.name?.simpleText ?? t.name?.runs?.[0]?.text ?? t.languageCode ?? '',
+            isAutoGenerated: (t.kind === 'asr') ||
+                (t.vssId?.startsWith('a.') ?? false) ||
+                String(t.name?.simpleText ?? '').toLowerCase().includes('auto') ||
+                false,
+        })).filter(t => t.baseUrl);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Pick the best caption track for the requested language.
+ * Priority: manual track in preferred language > auto-generated in preferred language > any manual > any
+ */
+function selectBestTrack(tracks, preferredLang) {
+    const lang = preferredLang.toLowerCase().split('-')[0]; // "en-US" → "en"
+    // 1. Manual in preferred language
+    const manualPref = tracks.find(t => !t.isAutoGenerated && t.languageCode.startsWith(lang));
+    if (manualPref)
+        return manualPref;
+    // 2. Auto-generated in preferred language
+    const autoPref = tracks.find(t => t.isAutoGenerated && t.languageCode.startsWith(lang));
+    if (autoPref)
+        return autoPref;
+    // 3. Any manual track
+    const anyManual = tracks.find(t => !t.isAutoGenerated);
+    if (anyManual)
+        return anyManual;
+    // 4. Fall back to first available
+    return tracks[0];
+}
+/**
+ * Fetch the caption XML from YouTube's timedtext API.
+ */
+async function fetchCaptionXml(baseUrl) {
+    // Ensure we request plain text (not ASS format)
+    const url = new URL(baseUrl);
+    url.searchParams.set('fmt', 'srv3'); // srv3 is a clean XML format
+    // Some older tracks need fmt=xml
+    url.searchParams.delete('fmt');
+    const result = await simpleFetch(url.toString(), undefined, 15000);
+    return result.html;
+}
+/**
+ * Parse YouTube caption XML into transcript segments.
+ *
+ * Format: <transcript><text start="0.5" dur="2.1">Hello &amp; world</text>...</transcript>
+ */
+export function parseCaptionXml(xml) {
+    const segments = [];
+    // Match all <text> elements with their attributes
+    const textRegex = /<text\s+([^>]*)>([\s\S]*?)<\/text>/g;
+    let match;
+    while ((match = textRegex.exec(xml)) !== null) {
+        const attrs = match[1];
+        const rawText = match[2];
+        const start = parseFloat(extractAttr(attrs, 'start') ?? '0');
+        const duration = parseFloat(extractAttr(attrs, 'dur') ?? '0');
+        const text = decodeHtmlEntities(rawText.trim());
+        if (text) {
+            segments.push({ text, start, duration });
+        }
+    }
+    return segments;
+}
+/**
+ * Extract an attribute value from an HTML/XML attribute string.
+ */
+function extractAttr(attrs, name) {
+    const regex = new RegExp(`${name}="([^"]*)"`, 'i');
+    const m = attrs.match(regex);
+    return m ? m[1] : null;
+}
+/**
+ * Decode common HTML entities found in YouTube caption XML.
+ *
+ * Order of operations:
+ * 1. Strip real HTML tags (e.g. <font color="...">) — these appear literally in the XML
+ * 2. Decode all HTML entities (including &lt; → < which represents literal angle brackets)
+ */
+export function decodeHtmlEntities(text) {
+    return text
+        // Step 1: strip real inline HTML tags (literal <...> in the text, not entities)
+        .replace(/<[^>]+>/g, '')
+        // Step 2: decode HTML entities
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&amp;/g, '&')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&apos;/g, "'")
+        .replace(/&#x27;/g, "'")
+        .replace(/&#x2F;/g, '/')
+        .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(parseInt(code, 10)))
+        .replace(/&#x([0-9A-Fa-f]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
+        .trim();
+}
+/**
+ * Format seconds into MM:SS or HH:MM:SS.
+ */
+export function formatDuration(seconds) {
+    if (!seconds || isNaN(seconds))
+        return '0:00';
+    const h = Math.floor(seconds / 3600);
+    const m = Math.floor((seconds % 3600) / 60);
+    const s = Math.floor(seconds % 60);
+    if (h > 0) {
+        return `${h}:${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`;
+    }
+    return `${m}:${String(s).padStart(2, '0')}`;
+}
+/**
+ * Extract a meta tag value from HTML (og:title, og:description, etc.)
+ */
+function extractMetaTag(html, property) {
+    const regex = new RegExp(`<meta[^>]+(?:property|name)=["']${property.replace(/:/g, '\\:')}["'][^>]+content=["']([^"']+)["']`, 'i');
+    const m = html.match(regex) ?? html.match(new RegExp(`<meta[^>]+content=["']([^"']+)["'][^>]+(?:property|name)=["']${property.replace(/:/g, '\\:')}["']`, 'i'));
+    return m ? decodeHtmlEntities(m[1]) : null;
+}
+//# sourceMappingURL=youtube.js.map

package/dist/core/youtube.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../src/core/youtube.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AA4C3C,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAEjD,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAErF,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACxB,4BAA4B;QAC5B,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClD,OAAO,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACxC,CAAC;IAED,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,oBAAoB;QACpB,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAClE,MAAM,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAED,kBAAkB;QAClB,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAED,mBAAmB;QACnB,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3C,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAED,iCAAiC;QACjC,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,EAAU;IAChC,OAAO,OAAO,EAAE,KAAK,QAAQ,IAAI,qBAAqB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAEnD,MAAM,YAAY,GAAG,cAAc,EAAE,YAAY,IAAI,EAAE,CAAC;IACxD,MAAM,WAAW,GAAG,cAAc,EAAE,WAAW,EAAE,yBAAyB,IAAI,EAAE,CAAC;IAEjF,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,IAAI,EAAE,CAAC;IAC3C,MAAM,KAAK,GACT,YAAY,CAAC,KAAK;QAClB,WAAW,CAAC,KAAK,EAAE,UAAU;QAC7B,cAAc,CAAC,IAAI,EAAE,UAAU,CAAC;QAChC,EAAE,CAAC;IACL,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,IAAI,WAAW,CAAC,gBAAgB,IAAI,EAAE,CAAC;IAC1E,MAAM,aAAa,GAAG,QAAQ,CAAC,YAAY,CAAC,aAAa,IAAI,WAAW,CAAC,aAAa,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IACnG,MAAM,SAAS,GAAG,YAAY,CAAC,SAAS,IAAI,WAAW,CAAC,SAAS,IAAI,EAAE,CAAC;IACxE,MAAM,WAAW,GAAG,WAAW,CAAC,WAAW,IAAI,WAAW,CAAC,UAAU,IAAI,EAAE,CAAC;IAC5E,MAAM,WAAW,GACf,YAAY,CAAC,gBAAgB;QAC7B,WAAW,CAAC,WAAW,EAAE,UAAU;QACnC,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC;QACtC,EAAE,CAAC;IACL,MAAM,SAAS,GACb,YAAY,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG;QACrD,WAAW,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG;QACpD,8BAA8B,OAAO,oBAAoB,CAAC;IAE5D,gDAAgD;IAChD,MAAM,SAAS,GAAG,YAAY,CAAC,SAAS,IAAI,EAAE,CAAC;IAE/C,OAAO;QACL,OAAO;QACP,KAAK;QACL,OAAO;QACP,WAAW;QACX,QAAQ,EAAE,cAAc,CAAC,aAAa,CAAC;QACvC,WAAW;QACX,SAAS;QACT,SAAS;QACT,SAAS;KACV,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,GAAW,EACX,UAAiC,EAAE;IAEnC,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;IAE/C,uBAAuB;IACvB,MAAM,QAAQ,GAAG,mCAAmC,OAAO,EAAE,CAAC;IAC9D,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAClE,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;IAE9B,0BAA0B;IAC1B,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,+CAA+C,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,qBAAqB;IACrB,MAAM,YAAY,GAAG,cAAc,CAAC,YAAY,IAAI,EAAE,CAAC;IACvD,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC;IACvC,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,IAAI,EAAE,CAAC;IAC1C,MAAM,aAAa,GAAG,QAAQ,CAAC,YAAY,CAAC,aAAa,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IAEtE,yBAAyB;IACzB,MAAM,aAAa,GAAmB,oBAAoB,CAAC,cAAc,CAAC,CAAC;IAC3E,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,mCAAmC,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,kBAAkB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IAElE,kFAAkF;IAClF,MAAM,aAAa,GAAG,eAAe,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;IAEpE,wBAAwB;IACxB,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAEhE,iBAAiB;IACjB,MAAM,QAAQ,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAE7C,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAEjF,OAAO;QACL,OAAO;QACP,KAAK;QACL,OAAO;QACP,QAAQ,EAAE,cAAc,CAAC,aAAa,CAAC;QACvC,QAAQ,EAAE,aAAa,CAAC,YAAY;QACpC,QAAQ;QACR,QAAQ;QACR,kBAAkB;KACnB,CAAC;AACJ,CAAC;AAaD;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,qDAAqD;IACrD,MAAM,QAAQ,GAAG;QACf,+CAA+C;QAC/C,oEAAoE;QACpE,6DAA6D;QAC7D,6DAA6D;KAC9D,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC;gBACH,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,0DAA0D;gBAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;gBACtD,IAAI,KAAK,KAAK,CAAC,CAAC;oBAAE,SAAS;gBAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC5C,IAAI,UAAU,KAAK,CAAC,CAAC;oBAAE,SAAS;gBAChC,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACpD,IAAI,OAAO,EAAE,CAAC;oBACZ,IAAI,CAAC;wBACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;oBAAC,MAAM,CAAC;wBACP,kCAAkC;oBACpC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACnD,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QACtB,yCAAyC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACrD,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACtB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACpD,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,CAAC;oBACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC7B,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,GAAW,EAAE,KAAa;IACnD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAElB,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,GAAG,KAAK,CAAC;YACf,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,GAAG,IAAI,CAAC;YACd,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACf,QAAQ,GAAG,CAAC,QAAQ,CAAC;YACrB,SAAS;QACX,CAAC;QAED,IAAI,QAAQ;YAAE,SAAS;QAEvB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG;YAAE,KAAK,EAAE,CAAC;aACjC,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAClC,KAAK,EAAE,CAAC;YACR,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,cAAmC;IAC/D,IAAI,CAAC;QACH,MAAM,MAAM,GACV,cAAc,EAAE,QAAQ,EAAE,+BAA+B,EAAE,aAAa,CAAC;QAC3E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YAC7B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;YACxB,YAAY,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC,WAAW,EAAE;YACzD,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,YAAY,IAAI,EAAE;YAC3E,eAAe,EACb,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;gBAClB,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC;gBACpC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC/D,KAAK;SACR,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,MAAsB,EAAE,aAAqB;IACpE,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB;IAEzE,kCAAkC;IAClC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3F,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC;IAElC,0CAA0C;IAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACxF,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAE9B,sBAAsB;IACtB,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;IACvD,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhC,kCAAkC;IAClC,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,eAAe,CAAC,OAAe;IAC5C,gDAAgD;IAChD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7B,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAE,6BAA6B;IACnE,iCAAiC;IACjC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAE/B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IACnE,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,QAAQ,GAAwB,EAAE,CAAC;IAEzC,kDAAkD;IAClD,MAAM,SAAS,GAAG,qCAAqC,CAAC;IACxD,IAAI,KAA6B,CAAC;IAElC,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEzB,MAAM,KAAK,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAEhD,IAAI,IAAI,EAAE,CAAC;YACT,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,KAAa,EAAE,IAAY;IAC9C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,IAAI,YAAY,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,IAAI;QACT,gFAAgF;SAC/E,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;QACxB,+BAA+B;SAC9B,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;SAC1E,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;SAClF,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IAEnC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IAC5E,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AAC9C,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY,EAAE,QAAgB;IACpD,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,mCAAmC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,mCAAmC,EACnG,GAAG,CACJ,CAAC;IACF,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CACvC,IAAI,MAAM,CAAC,gEAAgE,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CACrH,CAAC;IACF,OAAO,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"}

package/dist/index.d.ts

@@ -6,6 +6,7 @@
 import { cleanup, warmup, closePool, scrollAndWait, closeProfileBrowser } from './core/fetcher.js';
 import type { PeelOptions, PeelResult } from './types.js';
 export * from './types.js';
+export { getDomainExtractor, extractDomainData, type DomainExtractResult, type DomainExtractor } from './core/domain-extractors.js';
 export { crawl, type CrawlOptions, type CrawlResult, type CrawlProgress } from './core/crawler.js';
 export { discoverSitemap, type SitemapUrl, type SitemapResult } from './core/sitemap.js';
 export { mapDomain, type MapOptions, type MapResult } from './core/map.js';
@@ -23,11 +24,14 @@ export { RateGovernor, formatDuration, type RateConfig, type RateState, type Can
 export { ApplicationTracker, type ApplicationRecord, type ApplicationFilter, type ApplicationStats, type ApplicationStatus, } from './core/application-tracker.js';
 export { applyToJob, loadApplications, saveApplication, getApplicationsToday, updateApplicationStatus, type ApplyProfile, type ApplyOptions, type ApplyProgressEvent, type DetectedField, type ApplyResult, type ApplicationRecord as ApplyApplicationRecord, } from './core/apply.js';
 export { extractListings, type ListingItem } from './core/extract-listings.js';
+export { parseYouTubeUrl, extractVideoInfo, extractPlayerResponse, parseCaptionXml, decodeHtmlEntities, getYouTubeTranscript, type TranscriptSegment, type YouTubeTranscript, type YouTubeVideoInfo, } from './core/youtube.js';
 export { formatTable } from './core/table-format.js';
 export { findNextPageUrl } from './core/paginate.js';
 export { distillToBudget, budgetListings, TOKENS_PER_LISTING_ITEM } from './core/budget.js';
 export { watch, parseDuration, parseAssertion, type WatchOptions, type Assertion, type WatchCheckResult, type AssertionResult, } from './core/watch.js';
 export { diffUrl, type DiffOptions, type DiffResult, type DiffChange, } from './core/diff.js';
+export { extractReadableContent, type ReadabilityResult, type ReadabilityOptions } from './core/readability.js';
+export { quickAnswer, type QuickAnswerOptions, type QuickAnswerResult } from './core/quick-answer.js';
 /**
  * Fetch and extract content from a URL
  *

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAQH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAInG,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAa,MAAM,YAAY,CAAC;AAErE,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,eAAe,EAAE,KAAK,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,KAAK,YAAY,EAAE,KAAK,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACvH,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,KAAK,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAClI,OAAO,EAAE,iBAAiB,EAAE,KAAK,oBAAoB,EAAE,KAAK,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAClH,OAAO,EAAE,QAAQ,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,KAAK,gBAAgB,EAAE,KAAK,UAAU,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7J,OAAO,EAAE,gBAAgB,EAAE,KAAK,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,mBAAmB,EACnB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,gBAAgB,GACtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,cAAc,EACd,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,UAAU,GAChB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,UAAU,EAAE,KAAK,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACvH,OAAO,EACL,YAAY,EACZ,cAAc,EACd,KAAK,UAAU,EACf,KAAK,SAAS,EACd,KAAK,cAAc,GACpB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,kBAAkB,EAClB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,GACvB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,iBAAiB,IAAI,sBAAsB,GACjD,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,eAAe,EAAE,KAAK,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAC/E,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EACL,KAAK,EACL,aAAa,EACb,cAAc,EACd,KAAK,YAAY,EACjB,KAAK,SAAS,EACd,KAAK,gBAAgB,EACrB,KAAK,eAAe,GACrB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,OAAO,EACP,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,UAAU,GAChB,MAAM,gBAAgB,CAAC;AAExB;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAwYtF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,WAAW,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACnD,OAAO,CAAC,CAAC,UAAU,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAAE,CAAC,CAwB1D;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,mBAAmB,EAAE,CAAC;AAC1E,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAChF,OAAO,EACL,qBAAqB,EACrB,WAAW,EACX,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,UAAU,EACV,cAAc,EACd,SAAS,EACT,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,oBAAoB,EACpB,aAAa,EACb,WAAW,EACX,eAAe,EACf,WAAW,EACX,KAAK,WAAW,GACjB,MAAM,iBAAiB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAQH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AASnG,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAa,MAAM,YAAY,CAAC;AAErE,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,KAAK,mBAAmB,EAAE,KAAK,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACpI,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,eAAe,EAAE,KAAK,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,KAAK,YAAY,EAAE,KAAK,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACvH,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,KAAK,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAClI,OAAO,EAAE,iBAAiB,EAAE,KAAK,oBAAoB,EAAE,KAAK,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAClH,OAAO,EAAE,QAAQ,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,KAAK,gBAAgB,EAAE,KAAK,UAAU,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7J,OAAO,EAAE,gBAAgB,EAAE,KAAK,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,mBAAmB,EACnB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,gBAAgB,GACtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,cAAc,EACd,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,UAAU,GAChB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,UAAU,EAAE,KAAK,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACvH,OAAO,EACL,YAAY,EACZ,cAAc,EACd,KAAK,UAAU,EACf,KAAK,SAAS,EACd,KAAK,cAAc,GACpB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,kBAAkB,EAClB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,GACvB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,iBAAiB,IAAI,sBAAsB,GACjD,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,eAAe,EAAE,KAAK,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAC/E,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,qBAAqB,EACrB,eAAe,EACf,kBAAkB,EAClB,oBAAoB,EACpB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,GACtB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EACL,KAAK,EACL,aAAa,EACb,cAAc,EACd,KAAK,YAAY,EACjB,KAAK,SAAS,EACd,KAAK,gBAAgB,EACrB,KAAK,eAAe,GACrB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,OAAO,EACP,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,UAAU,GAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,sBAAsB,EAAE,KAAK,iBAAiB,EAAE,KAAK,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAChH,OAAO,EAAE,WAAW,EAAE,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEtG;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CA4etF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,WAAW,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACnD,OAAO,CAAC,CAAC,UAAU,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAAE,CAAC,CAwB1D;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,mBAAmB,EAAE,CAAC;AAC1E,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAChF,OAAO,EACL,qBAAqB,EACrB,WAAW,EACX,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,UAAU,EACV,cAAc,EACd,SAAS,EACT,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,oBAAoB,EACpB,aAAa,EACb,WAAW,EACX,eAAe,EACf,WAAW,EACX,KAAK,WAAW,GACjB,MAAM,iBAAiB,CAAC"}

package/dist/index.js

@@ -13,7 +13,12 @@ import { cleanup, warmup, closePool, scrollAndWait, closeProfileBrowser } from '
 import { autoScroll as runAutoScroll } from './core/actions.js';
 import { extractStructured } from './core/extract.js';
 import { isPdfContentType, isDocxContentType, extractDocumentToFormat } from './core/documents.js';
+import { parseYouTubeUrl, getYouTubeTranscript } from './core/youtube.js';
+import { extractDomainData, getDomainExtractor } from './core/domain-extractors.js';
+import { extractReadableContent } from './core/readability.js';
+import { quickAnswer as runQuickAnswer } from './core/quick-answer.js';
 export * from './types.js';
+export { getDomainExtractor, extractDomainData } from './core/domain-extractors.js';
 export { crawl } from './core/crawler.js';
 export { discoverSitemap } from './core/sitemap.js';
 export { mapDomain } from './core/map.js';
@@ -32,11 +37,14 @@ export { ApplicationTracker, } from './core/application-tracker.js';
 export { applyToJob, loadApplications, saveApplication, getApplicationsToday, updateApplicationStatus, } from './core/apply.js';
 // Human behavior exports — see bottom of file for full export
 export { extractListings } from './core/extract-listings.js';
+export { parseYouTubeUrl, extractVideoInfo, extractPlayerResponse, parseCaptionXml, decodeHtmlEntities, getYouTubeTranscript, } from './core/youtube.js';
 export { formatTable } from './core/table-format.js';
 export { findNextPageUrl } from './core/paginate.js';
 export { distillToBudget, budgetListings, TOKENS_PER_LISTING_ITEM } from './core/budget.js';
 export { watch, parseDuration, parseAssertion, } from './core/watch.js';
 export { diffUrl, } from './core/diff.js';
+export { extractReadableContent } from './core/readability.js';
+export { quickAnswer } from './core/quick-answer.js';
 /**
  * Fetch and extract content from a URL
  *
@@ -90,6 +98,59 @@ export async function peel(url, options = {}) {
     if (autoScrollOpts) {
         render = true;
     }
+    // ---------------------------------------------------------------------------
+    // YouTube special case — extract transcript instead of fetching the page HTML
+    // ---------------------------------------------------------------------------
+    const ytVideoId = parseYouTubeUrl(url);
+    if (ytVideoId) {
+        const ytStartTime = Date.now();
+        try {
+            const transcript = await getYouTubeTranscript(url, {
+                language: options.language ?? 'en',
+            });
+            // Build a clean markdown representation of the video + transcript
+            const videoInfoLines = [
+                `# ${transcript.title}`,
+                '',
+                `**Channel:** ${transcript.channel}`,
+                `**Duration:** ${transcript.duration}`,
+                `**Language:** ${transcript.language}`,
+                transcript.availableLanguages.length > 1
+                    ? `**Available Languages:** ${transcript.availableLanguages.join(', ')}`
+                    : '',
+                '',
+                '## Transcript',
+                '',
+                transcript.fullText,
+            ].filter(l => l !== undefined);
+            const videoInfoContent = videoInfoLines.join('\n');
+            const elapsed = Date.now() - ytStartTime;
+            const tokens = estimateTokens(videoInfoContent);
+            const fingerprint = createHash('sha256').update(videoInfoContent).digest('hex').slice(0, 16);
+            return {
+                url: `https://www.youtube.com/watch?v=${ytVideoId}`,
+                title: transcript.title,
+                content: videoInfoContent,
+                metadata: {
+                    description: `YouTube video by ${transcript.channel}, duration ${transcript.duration}`,
+                    author: transcript.channel,
+                },
+                links: [`https://www.youtube.com/watch?v=${ytVideoId}`],
+                tokens,
+                method: 'simple',
+                elapsed,
+                contentType: 'youtube',
+                quality: 1.0,
+                fingerprint,
+                extracted: undefined,
+                structured: transcript,
+            };
+        }
+        catch (_ytError) {
+            // If transcript extraction fails (no captions, page changed, etc.),
+            // fall through to the normal HTML fetch pipeline below.
+        }
+    }
     try {
         // Fetch the page (keep browser open if branding extraction or autoScroll is needed)
         const needsBranding = options.branding && render;
@@ -284,6 +345,20 @@ export async function peel(url, options = {}) {
             links = [...new Set(found)];
             quality = 1.0;
         }
+        // Readability mode
+        let readabilityResult;
+        if (options.readable && isHTML && fetchResult.html) {
+            const readResult = extractReadableContent(fetchResult.html, fetchResult.url);
+            readabilityResult = readResult;
+            content = readResult.content;
+            metadata = {
+                ...metadata,
+                title: readResult.title || metadata?.title,
+                author: readResult.author || undefined,
+                publishedDate: readResult.date || undefined,
+            };
+            title = readResult.title || title;
+        }
         // Extract images if requested
         let imagesList;
         if (extractImagesFlag && isHTML) {
@@ -314,6 +389,31 @@ export async function peel(url, options = {}) {
                 format === 'text' ? 'text' : 'markdown';
             content = distillToBudget(content, options.budget, budgetFormat);
         }
+        // Domain-aware structured extraction (Twitter, Reddit, GitHub, HN)
+        // Fires when URL matches a known domain. Replaces content with clean markdown.
+        let domainData;
+        if (getDomainExtractor(fetchResult.url)) {
+            try {
+                const ddResult = await extractDomainData(fetchResult.html, fetchResult.url);
+                if (ddResult) {
+                    domainData = ddResult;
+                    content = ddResult.cleanContent;
+                }
+            }
+            catch {
+                // Domain extraction failure is non-fatal; continue with normal content
+            }
+        }
+        // Quick answer (LLM-free)
+        let quickAnswerResult;
+        if (options.question && content) {
+            const qa = runQuickAnswer({
+                question: options.question,
+                content,
+                url: fetchResult.url,
+            });
+            quickAnswerResult = qa;
+        }
         // Calculate elapsed time, tokens, and fingerprint
         const elapsed = Date.now() - startTime;
         const tokens = estimateTokens(content);
@@ -390,6 +490,9 @@ export async function peel(url, options = {}) {
             summary: summaryText,
             images: imagesList,
             ...(prunedPercent !== undefined ? { prunedPercent } : {}),
+            ...(domainData !== undefined ? { domainData } : {}),
+            ...(readabilityResult !== undefined ? { readability: readabilityResult } : {}),
+            ...(quickAnswerResult !== undefined ? { quickAnswer: quickAnswerResult } : {}),
         };
     }
     catch (error) {