summd 0.1.4 → 0.1.5

package/dist/api.js

@@ -26,16 +26,12 @@ function text(result) {
     return result.content.map(c => c.text).join('');
 }
 // ── Public API ────────────────────────────────────────────────────────────────
-export async function add(content, tags, opts, note) {
+export async function add(content, tags, opts, note, sourceUrl) {
     const args = { content, tags, source_type: 'cli' };
     if (note)
         args.note = note;
-    return text(await tool('add_entry', args, opts));
-}
-export async function addUrl(url, tags, opts, note) {
-    const args = { url, tags, source_type: 'cli' };
-    if (note)
-        args.note = note;
+    if (sourceUrl)
+        args.source_url = sourceUrl;
     return text(await tool('add_entry', args, opts));
 }
 export async function search(query, limit, opts) {

package/dist/index.js

@@ -5,6 +5,7 @@ import { createInterface } from 'readline';
 import { program } from 'commander';
 import { saveKey } from './config.js';
 import * as api from './api.js';
+import { urlToMarkdown } from './url-to-md.js';
 const CODE_EXTS = new Set([
     'js', 'ts', 'jsx', 'tsx', 'mjs', 'cjs',
     'py', 'rb', 'go', 'rs', 'java', 'kt', 'swift', 'c', 'cpp', 'cs',
@@ -67,9 +68,18 @@ program
     .option('-n, --note <text>', 'Personal annotation (not included in AI summarization)')
     .action(async (content, cmd) => {
     const tags = cmd.tag ? cmd.tag.split(',').map(t => t.trim()) : [];
-    // URL mode: delegate fetch + convert to server
+    // URL mode: fetch + convert locally (local IP), then send content to server
     if (cmd.url) {
-        console.log(await api.addUrl(cmd.url, tags, globalOpts(), cmd.note));
+        let result;
+        try {
+            process.stderr.write('Fetching…\n');
+            result = await urlToMarkdown(cmd.url);
+        }
+        catch (err) {
+            console.error(`URL fetch failed: ${err.message}`);
+            process.exit(1);
+        }
+        console.log(await api.add(result.markdown, tags, globalOpts(), cmd.note, cmd.url));
         return;
     }
     let body = content;

package/dist/url-to-md.js

@@ -0,0 +1,209 @@
+/**
+ * URL → Markdown conversion.
+ *
+ * YouTube:  yt-dlp (install: pip install yt-dlp | brew install yt-dlp)
+ * Twitter:  fxtwitter API
+ * Article:  fetch → jsdom + Readability → turndown
+ */
+import { execFile } from 'child_process';
+import { readFileSync, readdirSync, unlinkSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { promisify } from 'util';
+import { JSDOM } from 'jsdom';
+import { Readability } from '@mozilla/readability';
+import TurndownService from 'turndown';
+const execFileAsync = promisify(execFile);
+// ── Public entry point ────────────────────────────────────────────────────────
+export async function urlToMarkdown(url) {
+    const videoId = extractYouTubeId(url);
+    if (videoId)
+        return fetchYouTubeTranscript(url, videoId);
+    const twitter = extractTwitterStatusId(url);
+    if (twitter)
+        return fetchTwitterPost(url, twitter.user, twitter.id);
+    // Non-tweet Twitter URLs (profile, home, etc.) fall through to fetchArticle.
+    // fetch() returns a JS-SPA shell so Readability will likely throw,
+    // but that error is more informative than a pre-emptive rejection.
+    return fetchArticle(url);
+}
+// ── URL detection ─────────────────────────────────────────────────────────────
+function extractYouTubeId(url) {
+    try {
+        const u = new URL(url);
+        if (/youtube\.com/.test(u.hostname)) {
+            const v = u.searchParams.get('v');
+            if (v)
+                return v;
+            const m = u.pathname.match(/^\/(shorts|live|embed)\/([^/?#]+)/);
+            if (m)
+                return m[2];
+        }
+        if (/youtu\.be/.test(u.hostname))
+            return u.pathname.slice(1).split('?')[0] || null;
+    }
+    catch { }
+    return null;
+}
+function extractTwitterStatusId(url) {
+    try {
+        const u = new URL(url);
+        if (!/(?:twitter|x)\.com/.test(u.hostname))
+            return null;
+        const m = u.pathname.match(/^\/([^/]+)\/status\/(\d+)/);
+        if (m)
+            return { user: m[1], id: m[2] };
+    }
+    catch { }
+    return null;
+}
+// ── YouTube ───────────────────────────────────────────────────────────────────
+async function fetchYouTubeTranscript(url, videoId) {
+    // Fetch title and transcript in parallel — they're independent
+    const [title, ytDlp] = await Promise.all([
+        fetchYouTubeTitle(url, videoId),
+        ytDlpTranscript(url, videoId),
+    ]);
+    const body = ytDlp.transcript ?? (ytDlp.reason === 'not-installed'
+        ? '> Transcript unavailable.\n> Install yt-dlp to enable: `pip install yt-dlp` or `brew install yt-dlp`'
+        : '> Transcript unavailable for this video.');
+    return { title, markdown: `# ${title}\n\n${body}\n\nsource: ${url}` };
+}
+async function fetchYouTubeTitle(url, videoId) {
+    try {
+        const res = await fetch(`https://www.youtube.com/oembed?url=${encodeURIComponent(url)}&format=json`, { signal: AbortSignal.timeout(5_000) });
+        if (res.ok)
+            return (await res.json()).title ?? `YouTube: ${videoId}`;
+    }
+    catch { }
+    return `YouTube: ${videoId}`;
+}
+// yt-dlp — works from any IP, supports auth/geo-block via --cookies-from-browser
+async function ytDlpTranscript(url, videoId) {
+    const dir = tmpdir();
+    const outTemplate = join(dir, videoId); // output: {videoId}.en.json3
+    try {
+        await execFileAsync('yt-dlp', [
+            '--write-subs',
+            '--write-auto-subs',
+            '--sub-langs', 'en',
+            '--sub-format', 'json3',
+            '--skip-download',
+            '--quiet',
+            '--no-progress',
+            '-o', outTemplate,
+            url,
+        ], { timeout: 30_000 });
+    }
+    catch (e) {
+        // Distinguish "yt-dlp not in PATH" from "ran but video has no captions"
+        const isNotFound = e.code === 'ENOENT';
+        return { transcript: null, reason: isNotFound ? 'not-installed' : 'no-transcript' };
+    }
+    // Find output file: {videoId}.en.json3, {videoId}.en-US.json3, etc.
+    try {
+        const files = readdirSync(dir).filter(f => f.startsWith(videoId) && f.endsWith('.json3'));
+        for (const file of files) {
+            const filePath = join(dir, file);
+            try {
+                const text = parseTimedText(JSON.parse(readFileSync(filePath, 'utf8')));
+                if (text)
+                    return { transcript: text };
+            }
+            finally {
+                try {
+                    unlinkSync(filePath);
+                }
+                catch { }
+            }
+        }
+    }
+    catch { }
+    return { transcript: null, reason: 'no-transcript' };
+}
+// json3 parser — shared algorithm with browser extension
+function parseTimedText(data) {
+    const paragraphs = [];
+    let current = [];
+    for (const event of data.events ?? []) {
+        if (!event.segs)
+            continue;
+        const text = event.segs.map(s => s.utf8 ?? '').join('');
+        if (text === '\n') {
+            const p = current.join('').replace(/\s+/g, ' ').trim();
+            if (p)
+                paragraphs.push(p);
+            current = [];
+        }
+        else {
+            current.push(text.replace(/\n/g, ' '));
+        }
+    }
+    const last = current.join('').replace(/\s+/g, ' ').trim();
+    if (last)
+        paragraphs.push(last);
+    return paragraphs.join('\n\n');
+}
+// ── Twitter / X ───────────────────────────────────────────────────────────────
+async function fetchTwitterPost(url, user, id) {
+    const res = await fetch(`https://api.fxtwitter.com/${user}/status/${id}`, {
+        signal: AbortSignal.timeout(8_000),
+    });
+    if (!res.ok)
+        throw new Error(`fxtwitter HTTP ${res.status}`);
+    const data = await res.json();
+    const tweet = data.tweet;
+    if (!tweet)
+        throw new Error('No tweet data from fxtwitter');
+    const author = tweet.author?.name ?? tweet.author?.screen_name ?? 'Unknown';
+    const title = `${author} on X`;
+    const photos = (tweet.media?.photos ?? []).map(p => `![](${p.url})`).join('\n');
+    const markdown = [
+        `# ${title}`,
+        tweet.text || null,
+        photos || null,
+        tweet.created_at?.trim() ?? null,
+        `source: ${url}`,
+    ].filter(Boolean).join('\n\n');
+    return { title, markdown };
+}
+// ── Article ───────────────────────────────────────────────────────────────────
+async function fetchArticle(url) {
+    const res = await fetch(url, {
+        headers: {
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+        },
+        signal: AbortSignal.timeout(10_000),
+    });
+    if (!res.ok)
+        throw new Error(`HTTP ${res.status}`);
+    const ct = res.headers.get('content-type') ?? '';
+    if (!ct.includes('html'))
+        throw new Error(`Unsupported content type: ${ct}`);
+    const html = await res.text();
+    const dom = new JSDOM(html, { url });
+    const article = new Readability(dom.window.document).parse();
+    if (!article?.content)
+        throw new Error('Could not extract article content');
+    const title = article.title || new URL(url).hostname;
+    const body = buildTurndown().turndown(article.content)
+        .replace(/\[]\([^)]*\)/g, '') // remove empty links (e.g. <a><img data:…></a>)
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+    return { title, markdown: `# ${title}\n\n${body}\n\nsource: ${url}` };
+}
+function buildTurndown() {
+    const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+    td.addRule('images', {
+        filter: 'img',
+        replacement: (_content, node) => {
+            const n = node;
+            if (!n.src || n.src.startsWith('data:'))
+                return '';
+            return `![${n.alt || ''}](${n.src})`;
+        },
+    });
+    return td;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "summd",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "CLI for sum.md — Sum to anything.",
   "license": "MIT",
   "bin": {
@@ -17,10 +17,15 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "commander": "^12.0.0"
+    "@mozilla/readability": "^0.6.0",
+    "commander": "^12.0.0",
+    "jsdom": "^29.0.2",
+    "turndown": "^7.2.4"
   },
   "devDependencies": {
+    "@types/jsdom": "^28.0.1",
     "@types/node": "^20.0.0",
+    "@types/turndown": "^5.0.6",
     "typescript": "^5.0.0"
   },
   "engines": {