ralph-starter 0.0.1 → 0.1.1-beta.0

package/dist/sources/builtin/github-scraper.js

@@ -0,0 +1,324 @@
+/**
+ * GitHub HTML Scraper
+ *
+ * Parses GitHub pages to extract content without requiring authentication.
+ * Handles:
+ * - Single issues: /owner/repo/issues/123
+ * - Issue lists: /owner/repo/issues
+ * - File blobs: /owner/repo/blob/branch/path
+ * - README pages: /owner/repo (extracts readme)
+ */
+/**
+ * Detect if a URL is a GitHub URL and what type
+ */
+export function detectGitHubUrl(url) {
+    if (url.hostname !== 'github.com' && url.hostname !== 'raw.githubusercontent.com') {
+        return { type: null, match: null };
+    }
+    // Raw file - already handled well by URL source
+    if (url.hostname === 'raw.githubusercontent.com') {
+        return { type: 'raw', match: null };
+    }
+    const pathname = url.pathname;
+    // Single issue: /owner/repo/issues/123
+    const issueMatch = pathname.match(/^\/([^/]+)\/([^/]+)\/issues\/(\d+)\/?$/);
+    if (issueMatch) {
+        return { type: 'issue', match: issueMatch };
+    }
+    // Issue list: /owner/repo/issues
+    const issuesMatch = pathname.match(/^\/([^/]+)\/([^/]+)\/issues\/?$/);
+    if (issuesMatch) {
+        return { type: 'issues', match: issuesMatch };
+    }
+    // File blob: /owner/repo/blob/branch/path
+    const blobMatch = pathname.match(/^\/([^/]+)\/([^/]+)\/blob\/([^/]+)\/(.+)$/);
+    if (blobMatch) {
+        return { type: 'blob', match: blobMatch };
+    }
+    // Repository root: /owner/repo
+    const repoMatch = pathname.match(/^\/([^/]+)\/([^/]+)\/?$/);
+    if (repoMatch) {
+        return { type: 'repo', match: repoMatch };
+    }
+    return { type: null, match: null };
+}
+/**
+ * Parse a single GitHub issue page
+ */
+export function parseGitHubIssue(html, issueNumber) {
+    // Extract title
+    const titleMatch = html.match(/<bdi class="js-issue-title[^"]*"[^>]*>([^<]+)<\/bdi>/);
+    const title = titleMatch ? decodeHtmlEntities(titleMatch[1].trim()) : `Issue #${issueNumber}`;
+    // Extract state (open/closed)
+    const stateOpen = html.includes('State State--open') || html.includes('status-open');
+    const state = stateOpen ? 'open' : 'closed';
+    // Extract author
+    const authorMatch = html.match(/class="author[^"]*"[^>]*>([^<]+)<\/a>/);
+    const author = authorMatch ? authorMatch[1].trim() : 'unknown';
+    // Extract labels
+    const labels = [];
+    const labelRegex = /IssueLabel[^>]*>([^<]+)<\/span>/g;
+    let labelMatch;
+    while ((labelMatch = labelRegex.exec(html)) !== null) {
+        labels.push(decodeHtmlEntities(labelMatch[1].trim()));
+    }
+    // Extract issue body
+    const bodyMatch = html.match(/class="edit-comment-hide"[^>]*>[\s\S]*?<td[^>]*class="[^"]*comment-body[^"]*"[^>]*>([\s\S]*?)<\/td>/);
+    let body = '';
+    if (bodyMatch) {
+        body = htmlToMarkdown(bodyMatch[1]);
+    }
+    // Extract created date
+    const dateMatch = html.match(/<relative-time[^>]*datetime="([^"]+)"[^>]*>/);
+    const createdAt = dateMatch ? dateMatch[1] : '';
+    // Extract comments
+    const comments = parseGitHubComments(html);
+    return {
+        number: issueNumber,
+        title,
+        body,
+        state,
+        labels,
+        author,
+        createdAt,
+        comments,
+    };
+}
+/**
+ * Parse comments from a GitHub issue page
+ */
+function parseGitHubComments(html) {
+    const comments = [];
+    // Find all comment bodies (skip the first one which is the issue body)
+    const commentRegex = /class="timeline-comment[^"]*"[\s\S]*?<a[^>]*class="author[^"]*"[^>]*>([^<]+)<\/a>[\s\S]*?<relative-time[^>]*datetime="([^"]+)"[\s\S]*?<td[^>]*class="[^"]*comment-body[^"]*"[^>]*>([\s\S]*?)<\/td>/g;
+    let match;
+    let isFirst = true;
+    while ((match = commentRegex.exec(html)) !== null) {
+        // Skip the first match (issue body)
+        if (isFirst) {
+            isFirst = false;
+            continue;
+        }
+        comments.push({
+            author: match[1].trim(),
+            createdAt: match[2],
+            body: htmlToMarkdown(match[3]),
+        });
+    }
+    return comments;
+}
+/**
+ * Parse GitHub issue list page
+ */
+export function parseGitHubIssueList(html, _owner, _repo) {
+    const issues = [];
+    // Find issue rows
+    const issueRegex = /<a[^>]*id="issue_(\d+)_link"[^>]*href="[^"]*"[^>]*>([^<]+)<\/a>/g;
+    let match;
+    while ((match = issueRegex.exec(html)) !== null) {
+        issues.push({
+            number: parseInt(match[1], 10),
+            title: decodeHtmlEntities(match[2].trim()),
+        });
+    }
+    // Check for pagination
+    const hasMore = html.includes('next_page') || html.includes('pagination');
+    return { issues, hasMore };
+}
+/**
+ * Parse GitHub file blob page
+ */
+export function parseGitHubBlob(html, filepath) {
+    // Try to find raw content in the page
+    // GitHub includes the raw content in a data attribute or script
+    let content = '';
+    // Method 1: Look for raw-blob-content
+    const blobMatch = html.match(/<div[^>]*class="[^"]*blob-code-content[^"]*"[^>]*>([\s\S]*?)<\/div>/);
+    if (blobMatch) {
+        content = extractCodeFromBlob(blobMatch[1]);
+    }
+    // Method 2: Look for data-plain attribute
+    if (!content) {
+        const plainMatch = html.match(/data-plain="([^"]*)"/);
+        if (plainMatch) {
+            content = decodeHtmlEntities(plainMatch[1]);
+        }
+    }
+    // Method 3: Extract line by line from blob-code elements
+    if (!content) {
+        const lines = [];
+        const lineRegex = /<td[^>]*class="[^"]*blob-code[^"]*"[^>]*>([\s\S]*?)<\/td>/g;
+        let lineMatch;
+        while ((lineMatch = lineRegex.exec(html)) !== null) {
+            lines.push(extractTextFromHtml(lineMatch[1]));
+        }
+        if (lines.length > 0) {
+            content = lines.join('\n');
+        }
+    }
+    // Detect language from file extension
+    const ext = filepath.split('.').pop()?.toLowerCase() || '';
+    const languageMap = {
+        js: 'javascript',
+        ts: 'typescript',
+        py: 'python',
+        rb: 'ruby',
+        go: 'go',
+        rs: 'rust',
+        java: 'java',
+        md: 'markdown',
+        json: 'json',
+        yaml: 'yaml',
+        yml: 'yaml',
+    };
+    return {
+        path: filepath,
+        content,
+        language: languageMap[ext] || ext,
+    };
+}
+/**
+ * Convert a GitHub issue to markdown
+ */
+export function issueToMarkdown(issue) {
+    const lines = [];
+    // Title with state badge
+    const stateBadge = issue.state === 'open' ? '🟢 Open' : '🔴 Closed';
+    lines.push(`# ${issue.title}`);
+    lines.push('');
+    lines.push(`**Issue #${issue.number}** | ${stateBadge} | by @${issue.author}`);
+    if (issue.labels.length > 0) {
+        lines.push(`**Labels:** ${issue.labels.join(', ')}`);
+    }
+    if (issue.createdAt) {
+        lines.push(`**Created:** ${new Date(issue.createdAt).toLocaleDateString()}`);
+    }
+    lines.push('');
+    lines.push('---');
+    lines.push('');
+    // Body
+    if (issue.body) {
+        lines.push(issue.body);
+        lines.push('');
+    }
+    // Comments
+    if (issue.comments.length > 0) {
+        lines.push('---');
+        lines.push('');
+        lines.push('## Comments');
+        lines.push('');
+        for (const comment of issue.comments) {
+            lines.push(`### @${comment.author}`);
+            if (comment.createdAt) {
+                lines.push(`*${new Date(comment.createdAt).toLocaleDateString()}*`);
+            }
+            lines.push('');
+            lines.push(comment.body);
+            lines.push('');
+        }
+    }
+    return lines.join('\n');
+}
+/**
+ * Convert a GitHub file to markdown
+ */
+export function fileToMarkdown(file) {
+    const lines = [];
+    lines.push(`# ${file.path}`);
+    lines.push('');
+    // If it's already markdown, just include it
+    if (file.language === 'markdown') {
+        lines.push(file.content);
+    }
+    else {
+        // Wrap in code block
+        lines.push(`\`\`\`${file.language || ''}`);
+        lines.push(file.content);
+        lines.push('```');
+    }
+    return lines.join('\n');
+}
+/**
+ * Convert issue list to markdown
+ */
+export function issueListToMarkdown(issues, owner, repo) {
+    const lines = [];
+    lines.push(`# Issues: ${owner}/${repo}`);
+    lines.push('');
+    for (const issue of issues) {
+        const link = `https://github.com/${owner}/${repo}/issues/${issue.number}`;
+        lines.push(`- [#${issue.number}](${link}) ${issue.title}`);
+    }
+    return lines.join('\n');
+}
+// Helper functions
+function decodeHtmlEntities(text) {
+    return text
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&#x27;/g, "'")
+        .replace(/&#(\d+);/g, (_, num) => String.fromCharCode(parseInt(num, 10)));
+}
+function htmlToMarkdown(html) {
+    return (html
+        // Code blocks
+        .replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, '\n```\n$1\n```\n')
+        // Inline code
+        .replace(/<code[^>]*>(.*?)<\/code>/gi, '`$1`')
+        // Headers
+        .replace(/<h1[^>]*>(.*?)<\/h1>/gi, '# $1\n')
+        .replace(/<h2[^>]*>(.*?)<\/h2>/gi, '## $1\n')
+        .replace(/<h3[^>]*>(.*?)<\/h3>/gi, '### $1\n')
+        .replace(/<h4[^>]*>(.*?)<\/h4>/gi, '#### $1\n')
+        // Paragraphs
+        .replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, '$1\n\n')
+        // Lists
+        .replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n')
+        .replace(/<ul[^>]*>|<\/ul>|<ol[^>]*>|<\/ol>/gi, '\n')
+        // Links
+        .replace(/<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gi, '[$2]($1)')
+        // Bold
+        .replace(/<strong[^>]*>(.*?)<\/strong>/gi, '**$1**')
+        .replace(/<b[^>]*>(.*?)<\/b>/gi, '**$1**')
+        // Italic
+        .replace(/<em[^>]*>(.*?)<\/em>/gi, '*$1*')
+        .replace(/<i[^>]*>(.*?)<\/i>/gi, '*$1*')
+        // Line breaks
+        .replace(/<br\s*\/?>/gi, '\n')
+        // Remove remaining tags
+        .replace(/<[^>]+>/g, '')
+        // Decode entities
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        // Clean up whitespace
+        .replace(/\n{3,}/g, '\n\n')
+        .trim());
+}
+function extractCodeFromBlob(html) {
+    // Remove span tags but keep content
+    return html
+        .replace(/<span[^>]*>/g, '')
+        .replace(/<\/span>/g, '')
+        .replace(/<[^>]+>/g, '')
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .trim();
+}
+function extractTextFromHtml(html) {
+    return html
+        .replace(/<[^>]+>/g, '')
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .trim();
+}
+//# sourceMappingURL=github-scraper.js.map

package/dist/sources/builtin/github-scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github-scraper.js","sourceRoot":"","sources":["../../../src/sources/builtin/github-scraper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAyBH;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,GAAQ;IAItC,IAAI,GAAG,CAAC,QAAQ,KAAK,YAAY,IAAI,GAAG,CAAC,QAAQ,KAAK,2BAA2B,EAAE,CAAC;QAClF,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACrC,CAAC;IAED,gDAAgD;IAChD,IAAI,GAAG,CAAC,QAAQ,KAAK,2BAA2B,EAAE,CAAC;QACjD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACtC,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAE9B,uCAAuC;IACvC,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5E,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IAC9C,CAAC;IAED,iCAAiC;IACjC,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACtE,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;IAChD,CAAC;IAED,0CAA0C;IAC1C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC9E,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;IAED,+BAA+B;IAC/B,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC5D,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,WAAmB;IAChE,gBAAgB;IAChB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;IACtF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,WAAW,EAAE,CAAC;IAE9F,8BAA8B;IAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;IAE5C,iBAAiB;IACjB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAE/D,iBAAiB;IACjB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,UAAU,GAAG,kCAAkC,CAAC;IACtD,IAAI,UAAU,CAAC;IACf,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACrD,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,qBAAqB;IACrB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAC1B,qGAAqG,CACtG,CAAC;IACF,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,uBAAuB;IACvB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAC5E,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAEhD,mBAAmB;IACnB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAE3C,OAAO;QACL,MAAM,EAAE,WAAW;QACnB,KAAK;QACL,IAAI;QACJ,KAAK;QACL,MAAM;QACN,MAAM;QACN,SAAS;QACT,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,QAAQ,GAAoB,EAAE,CAAC;IAErC,uEAAuE;IACvE,MAAM,YAAY,GAChB,qMAAqM,CAAC;IAExM,IAAI,KAAK,CAAC;IACV,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAClD,oCAAoC;QACpC,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,GAAG,KAAK,CAAC;YAChB,SAAS;QACX,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;YACvB,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;YACnB,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,IAAY,EACZ,MAAc,EACd,KAAa;IAEb,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,kBAAkB;IAClB,MAAM,UAAU,GAAG,kEAAkE,CAAC;IAEtF,IAAI,KAAK,CAAC;IACV,OAAO,CAAC,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAChD,MAAM,CAAC,IAAI,CAAC;YACV,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YAC9B,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SAC3C,CAAC,CAAC;IACL,CAAC;IAED,uBAAuB;IACvB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IAE1E,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,QAAgB;IAC5D,sCAAsC;IACtC,gEAAgE;IAChE,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,sCAAsC;IACtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAC1B,qEAAqE,CACtE,CAAC;IACF,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,0CAA0C;IAC1C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtD,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,yDAAyD;IACzD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,4DAA4D,CAAC;QAC/E,IAAI,SAAS,CAAC;QACd,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACnD,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC3D,MAAM,WAAW,GAA2B;QAC1C,EAAE,EAAE,YAAY;QAChB,EAAE,EAAE,YAAY;QAChB,EAAE,EAAE,QAAQ;QACZ,EAAE,EAAE,MAAM;QACV,EAAE,EAAE,IAAI;QACR,EAAE,EAAE,MAAM;QACV,IAAI,EAAE,MAAM;QACZ,EAAE,EAAE,UAAU;QACd,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,MAAM;QACZ,GAAG,EAAE,MAAM;KACZ,CAAC;IAEF,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,OAAO;QACP,QAAQ,EAAE,WAAW,CAAC,GAAG,CAAC,IAAI,GAAG;KAClC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,yBAAyB;IACzB,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC;IACpE,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,MAAM,QAAQ,UAAU,UAAU,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAE/E,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO;IACP,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,WAAW;IACX,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,QAAQ,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YACrC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBACtB,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;YACtE,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAgB;IAC7C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,4CAA4C;IAC5C,IAAI,IAAI,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAA8B,EAC9B,KAAa,EACb,IAAY;IAEZ,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,sBAAsB,KAAK,IAAI,IAAI,WAAW,KAAK,CAAC,MAAM,EAAE,CAAC;QAC1E,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,MAAM,KAAK,IAAI,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,mBAAmB;AAEnB,SAAS,kBAAkB,CAAC,IAAY;IACtC,OAAO,IAAI;SACR,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,CACL,IAAI;QACF,cAAc;SACb,OAAO,CAAC,kDAAkD,EAAE,kBAAkB,CAAC;QAChF,cAAc;SACb,OAAO,CAAC,4BAA4B,EAAE,MAAM,CAAC;QAC9C,UAAU;SACT,OAAO,CAAC,wBAAwB,EAAE,QAAQ,CAAC;SAC3C,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC;SAC5C,OAAO,CAAC,wBAAwB,EAAE,UAAU,CAAC;SAC7C,OAAO,CAAC,wBAAwB,EAAE,WAAW,CAAC;QAC/C,aAAa;SACZ,OAAO,CAAC,2BAA2B,EAAE,QAAQ,CAAC;QAC/C,QAAQ;SACP,OAAO,CAAC,6BAA6B,EAAE,QAAQ,CAAC;SAChD,OAAO,CAAC,qCAAqC,EAAE,IAAI,CAAC;QACrD,QAAQ;SACP,OAAO,CAAC,yCAAyC,EAAE,UAAU,CAAC;QAC/D,OAAO;SACN,OAAO,CAAC,gCAAgC,EAAE,QAAQ,CAAC;SACnD,OAAO,CAAC,sBAAsB,EAAE,QAAQ,CAAC;QAC1C,SAAS;SACR,OAAO,CAAC,wBAAwB,EAAE,MAAM,CAAC;SACzC,OAAO,CAAC,sBAAsB,EAAE,MAAM,CAAC;QACxC,cAAc;SACb,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC;QAC9B,wBAAwB;SACvB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;QACxB,kBAAkB;SACjB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;QACtB,sBAAsB;SACrB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CACV,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,oCAAoC;IACpC,OAAO,IAAI;SACR,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;SACxB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI;SACR,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,IAAI,EAAE,CAAC;AACZ,CAAC"}

package/dist/sources/builtin/pdf.d.ts

@@ -0,0 +1,24 @@
+import { BuiltinSource } from '../base.js';
+import type { SourceOptions, SourceResult } from '../types.js';
+/**
+ * PDF source - extracts text from PDF documents
+ *
+ * Supports:
+ * - Local PDF files
+ * - Remote PDF URLs
+ *
+ * Requires: pdf-parse package
+ */
+export declare class PdfSource extends BuiltinSource {
+    name: string;
+    description: string;
+    private pdfParse;
+    isAvailable(): Promise<boolean>;
+    fetch(identifier: string, options?: SourceOptions): Promise<SourceResult>;
+    private fetchFromFile;
+    private fetchFromUrl;
+    private formatPdfContent;
+    private addMarkdownStructure;
+    getHelp(): string;
+}
+//# sourceMappingURL=pdf.d.ts.map

package/dist/sources/builtin/pdf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf.d.ts","sourceRoot":"","sources":["../../../src/sources/builtin/pdf.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE/D;;;;;;;;GAQG;AACH,qBAAa,SAAU,SAAQ,aAAa;IAC1C,IAAI,SAAS;IACb,WAAW,SAAsC;IAEjD,OAAO,CAAC,QAAQ,CAA8D;IAExE,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAU/B,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;YAiDjE,aAAa;YAqBb,YAAY;IA2B1B,OAAO,CAAC,gBAAgB;IAkBxB,OAAO,CAAC,oBAAoB;IAgD5B,OAAO,IAAI,MAAM;CAmBlB"}

package/dist/sources/builtin/pdf.js

@@ -0,0 +1,174 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { basename, resolve } from 'node:path';
+import { BuiltinSource } from '../base.js';
+/**
+ * PDF source - extracts text from PDF documents
+ *
+ * Supports:
+ * - Local PDF files
+ * - Remote PDF URLs
+ *
+ * Requires: pdf-parse package
+ */
+export class PdfSource extends BuiltinSource {
+    name = 'pdf';
+    description = 'Extract specs from PDF documents';
+    pdfParse = null;
+    async isAvailable() {
+        try {
+            const pdfParseModule = await import('pdf-parse');
+            this.pdfParse = pdfParseModule.default;
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async fetch(identifier, options) {
+        // Ensure pdf-parse is loaded
+        if (!this.pdfParse) {
+            const available = await this.isAvailable();
+            if (!available) {
+                this.error('pdf-parse package not installed. Run: npm install pdf-parse');
+            }
+        }
+        let buffer;
+        let source;
+        let title;
+        // Check if it's a URL or local file
+        if (identifier.startsWith('http://') || identifier.startsWith('https://')) {
+            const result = await this.fetchFromUrl(identifier, options);
+            buffer = result.buffer;
+            source = identifier;
+            title = result.title;
+        }
+        else {
+            const result = await this.fetchFromFile(identifier);
+            buffer = result.buffer;
+            source = result.path;
+            title = result.title;
+        }
+        // Parse PDF
+        const data = await this.pdfParse?.(buffer);
+        if (!data) {
+            throw new Error('PDF parsing failed - pdf-parse not available');
+        }
+        // Clean and format the text
+        const content = this.formatPdfContent(data.text, title);
+        return {
+            content,
+            source,
+            title,
+            metadata: {
+                type: 'pdf',
+                pages: data.numpages,
+                info: data.info,
+                version: data.version,
+            },
+        };
+    }
+    async fetchFromFile(identifier) {
+        const path = resolve(process.cwd(), identifier);
+        if (!existsSync(path)) {
+            this.error(`PDF file not found: ${identifier}`);
+        }
+        if (!path.toLowerCase().endsWith('.pdf')) {
+            this.error(`Not a PDF file: ${identifier}`);
+        }
+        const buffer = readFileSync(path);
+        const title = basename(path, '.pdf');
+        return { buffer, path, title };
+    }
+    async fetchFromUrl(url, options) {
+        const response = await fetch(url, {
+            headers: {
+                'User-Agent': 'ralph-starter/0.1.0',
+                ...options?.headers,
+            },
+        });
+        if (!response.ok) {
+            this.error(`HTTP error ${response.status}: ${response.statusText}`);
+        }
+        const arrayBuffer = await response.arrayBuffer();
+        const buffer = Buffer.from(arrayBuffer);
+        // Try to get title from URL
+        const urlObj = new URL(url);
+        const pathSegments = urlObj.pathname.split('/');
+        const filename = pathSegments[pathSegments.length - 1] || 'document';
+        const title = filename.replace(/\.pdf$/i, '');
+        return { buffer, title };
+    }
+    formatPdfContent(text, title) {
+        // Clean up the text
+        let cleaned = text
+            // Normalize whitespace
+            .replace(/\r\n/g, '\n')
+            .replace(/\r/g, '\n')
+            // Remove excessive blank lines
+            .replace(/\n{4,}/g, '\n\n\n')
+            // Clean up spacing
+            .replace(/[ \t]+/g, ' ')
+            .trim();
+        // Try to detect sections and add markdown headers
+        cleaned = this.addMarkdownStructure(cleaned);
+        return `# ${title}\n\n${cleaned}`;
+    }
+    addMarkdownStructure(text) {
+        const lines = text.split('\n');
+        const result = [];
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i].trim();
+            // Skip empty lines
+            if (!line) {
+                result.push('');
+                continue;
+            }
+            // Detect potential headers (all caps, short, followed by content)
+            if (line.length < 60 &&
+                line === line.toUpperCase() &&
+                /^[A-Z][A-Z\s\d]+$/.test(line) &&
+                lines[i + 1]?.trim()) {
+                // Convert to title case
+                const titleCase = line.toLowerCase().replace(/\b\w/g, (c) => c.toUpperCase());
+                result.push(`\n## ${titleCase}\n`);
+                continue;
+            }
+            // Detect numbered sections
+            const numberedMatch = line.match(/^(\d+\.?\d*\.?)\s+(.+)$/);
+            if (numberedMatch && line.length < 80) {
+                const [, number, text] = numberedMatch;
+                const depth = (number.match(/\./g) || []).length;
+                const prefix = '#'.repeat(Math.min(depth + 2, 4));
+                result.push(`\n${prefix} ${text}\n`);
+                continue;
+            }
+            // Detect bullet points
+            if (line.startsWith('•') || line.startsWith('-') || line.startsWith('*')) {
+                result.push(`- ${line.slice(1).trim()}`);
+                continue;
+            }
+            result.push(line);
+        }
+        return result.join('\n');
+    }
+    getHelp() {
+        return `
+pdf: Extract specs from PDF documents
+
+Usage:
+  ralph-starter run --from ./document.pdf
+  ralph-starter run --from https://example.com/spec.pdf
+
+Examples:
+  ralph-starter run --from requirements.pdf
+  ralph-starter run --from ./docs/PRD.pdf
+  ralph-starter run --from https://company.com/api-spec.pdf
+
+Notes:
+  - Requires pdf-parse package (npm install pdf-parse)
+  - Text extraction quality depends on PDF structure
+  - Scanned PDFs (images) are not supported
+`.trim();
+    }
+}
+//# sourceMappingURL=pdf.js.map

package/dist/sources/builtin/pdf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../src/sources/builtin/pdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAG3C;;;;;;;;GAQG;AACH,MAAM,OAAO,SAAU,SAAQ,aAAa;IAC1C,IAAI,GAAG,KAAK,CAAC;IACb,WAAW,GAAG,kCAAkC,CAAC;IAEzC,QAAQ,GAAyD,IAAI,CAAC;IAE9E,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;YACjD,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC;YACvC,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAkB,EAAE,OAAuB;QACrD,6BAA6B;QAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,IAAI,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QAED,IAAI,MAAc,CAAC;QACnB,IAAI,MAAc,CAAC;QACnB,IAAI,KAAa,CAAC;QAElB,oCAAoC;QACpC,IAAI,UAAU,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YAC5D,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YACvB,MAAM,GAAG,UAAU,CAAC;YACpB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YACpD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YACvB,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC;YACrB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACvB,CAAC;QAED,YAAY;QACZ,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,CAAC;QAE3C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QAED,4BAA4B;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAExD,OAAO;YACL,OAAO;YACP,MAAM;YACN,KAAK;YACL,QAAQ,EAAE;gBACR,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,IAAI,CAAC,QAAQ;gBACpB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,UAAkB;QAK5C,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;QAEhD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,KAAK,CAAC,uBAAuB,UAAU,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,mBAAmB,UAAU,EAAE,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAErC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,GAAW,EACX,OAAuB;QAEvB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,OAAO,EAAE;gBACP,YAAY,EAAE,qBAAqB;gBACnC,GAAG,OAAO,EAAE,OAAO;aACpB;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,IAAI,CAAC,KAAK,CAAC,cAAc,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAExC,4BAA4B;QAC5B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,UAAU,CAAC;QACrE,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAE9C,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC3B,CAAC;IAEO,gBAAgB,CAAC,IAAY,EAAE,KAAa;QAClD,oBAAoB;QACpB,IAAI,OAAO,GAAG,IAAI;YAChB,uBAAuB;aACtB,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;aACtB,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;YACrB,+BAA+B;aAC9B,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC;YAC7B,mBAAmB;aAClB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,IAAI,EAAE,CAAC;QAEV,kDAAkD;QAClD,OAAO,GAAG,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAE7C,OAAO,KAAK,KAAK,OAAO,OAAO,EAAE,CAAC;IACpC,CAAC;IAEO,oBAAoB,CAAC,IAAY;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAE7B,mBAAmB;YACnB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,kEAAkE;YAClE,IACE,IAAI,CAAC,MAAM,GAAG,EAAE;gBAChB,IAAI,KAAK,IAAI,CAAC,WAAW,EAAE;gBAC3B,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC9B,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EACpB,CAAC;gBACD,wBAAwB;gBACxB,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;gBAC9E,MAAM,CAAC,IAAI,CAAC,QAAQ,SAAS,IAAI,CAAC,CAAC;gBACnC,SAAS;YACX,CAAC;YAED,2BAA2B;YAC3B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC5D,IAAI,aAAa,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACtC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,aAAa,CAAC;gBACvC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;gBACjD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,CAAC;gBACrC,SAAS;YACX,CAAC;YAED,uBAAuB;YACvB,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzE,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,OAAO;;;;;;;;;;;;;;;;CAgBV,CAAC,IAAI,EAAE,CAAC;IACP,CAAC;CACF"}

package/dist/sources/builtin/url.d.ts

@@ -0,0 +1,47 @@
+import { BuiltinSource } from '../base.js';
+import type { SourceOptions, SourceResult } from '../types.js';
+/**
+ * URL source - fetches specs from remote URLs
+ *
+ * Supports:
+ * - Raw markdown files (GitHub raw, etc.)
+ * - HTML pages (basic conversion)
+ * - JSON responses
+ * - GitHub issues and files (specialized parsing)
+ * - Notion public pages (specialized parsing)
+ */
+export declare class UrlSource extends BuiltinSource {
+    name: string;
+    description: string;
+    isAvailable(): Promise<boolean>;
+    fetch(identifier: string, options?: SourceOptions): Promise<SourceResult>;
+    /**
+     * Fetch and parse GitHub URLs (issues, blobs, etc.)
+     */
+    private fetchGitHub;
+    /**
+     * Check if URL is a Notion public page
+     */
+    private isNotionUrl;
+    /**
+     * Fetch and parse Notion public pages
+     */
+    private fetchNotion;
+    /**
+     * Parse Notion public page HTML
+     * Note: Notion uses client-side rendering, so public pages have limited HTML content
+     */
+    private parseNotionHtml;
+    /**
+     * Extract content from Notion's preloaded data
+     */
+    private extractNotionContent;
+    private isMarkdown;
+    private jsonToMarkdown;
+    private arrayToMarkdown;
+    private objectToMarkdown;
+    private htmlToMarkdown;
+    private getTitleFromUrl;
+    getHelp(): string;
+}
+//# sourceMappingURL=url.d.ts.map

package/dist/sources/builtin/url.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"url.d.ts","sourceRoot":"","sources":["../../../src/sources/builtin/url.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAW/D;;;;;;;;;GASG;AACH,qBAAa,SAAU,SAAQ,aAAa;IAC1C,IAAI,SAAS;IACb,WAAW,SAAkC;IAEvC,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAI/B,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAoE/E;;OAEG;YACW,WAAW;IAuGzB;;OAEG;IACH,OAAO,CAAC,WAAW;IAQnB;;OAEG;YACW,WAAW;IAoCzB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAsDvB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAkB5B,OAAO,CAAC,UAAU;IAsBlB,OAAO,CAAC,cAAc;IAoBtB,OAAO,CAAC,eAAe;IAsBvB,OAAO,CAAC,gBAAgB;IAwCxB,OAAO,CAAC,cAAc;IAyBtB,OAAO,CAAC,eAAe;IAiBvB,OAAO,IAAI,MAAM;CAiClB"}