tokwise 0.1.0 → 0.1.2

package/README.md

@@ -43,7 +43,7 @@ npm link
 ```bash
 # Optional: save a browser cookie for private collections or liked videos.
 # Easiest on macOS: pull it straight from a logged-in Chromium browser.
-tokwise auth from-browser            # auto-detects Chrome, Brave, Edge, Arc, or Chromium
+tokwise auth from-browser            # auto-detects Chrome, Brave, Edge, Arc, Dia, or Chromium
 tokwise auth refresh                 # re-pull later when the session goes stale
 
 # Or paste a cookie manually (works everywhere).
@@ -52,7 +52,12 @@ tokwise auth set --cookie "YOUR_COOKIE"
 # Tokwise tries to detect the @handle tied to your cookie. Set it manually if needed.
 tokwise auth set-username your-handle
 
-# Sync a collection, download audio, transcribe, classify, and index.
+# Easiest start: run sync with no source. Tokwise pulls your TikTok bookmarks
+# (Collections + Favorites), shows an interactive checklist, then asks what to do
+# with the ones you pick (download / transcribe / classify).
+tokwise sync
+
+# Or target a specific collection directly.
 # --collection accepts a full URL, an @user/collection/slug path, or a bare slug.
 tokwise sync --collection "name-123" \
   --limit 200 \
@@ -132,9 +137,50 @@ export TOKWISE_COMMANDS_DIR=/path/to/commands
 
 Legacy `TT_*` environment variables and `~/.tiktoktheory` are still read so existing local archives keep working after the rename.
 
+## Interactive sync
+
+Run `tokwise sync` with no source flags to discover and pick your bookmarks:
+
+```bash
+tokwise sync                  # checklist of your Collections + Favorites, then a pipeline checklist
+tokwise sync --download --classify   # skip the pipeline prompt; apply these to whatever you pick
+```
+
+Requires a saved cookie (`tokwise auth from-browser`). Tokwise resolves your
+`secUid` once and caches it. The pipeline checklist pre-checks Download and
+Transcribe only when `yt-dlp` / Whisper are installed; Classify is always on.
+
+When you pick more than one bookmark, sync runs and reports progress per
+collection (each with its own download / transcribe / classify status) and
+builds the search index once at the end.
+
+For scripts and agents, the same discovery is non-interactive:
+
+```bash
+tokwise sync --list           # print discovered bookmarks as a table
+tokwise sync --list --json    # machine-readable: id, name, itemCount, url, kind
+tokwise sync --all            # sync every bookmark without prompting
+```
+
+When `tokwise sync` runs without an interactive terminal and no flags, it prints
+the bookmark list plus a hint and exits without syncing.
+
+### Rate limits
+
+TikTok rate-limits rapid requests. Tokwise paces calls and retries on HTTP 429
+(and 5xx) with exponential backoff, honoring any `Retry-After` header. If a
+collection still fails after retries, it is skipped and reported at the end so
+the collections that did sync are kept.
+
+```bash
+tokwise sync --request-delay 1500   # ms between requests (default 500); raise if you still hit 429s
+tokwise sync --max-retries 5        # retries per request on 429/5xx (default 3)
+```
+
 ## Sources
 
 ```bash
+tokwise sync                  # interactive: pick from your bookmarks
 tokwise sync --collection <url | @user/collection/slug | slug-or-id>
 tokwise sync --playlist <playlist-id-or-url>
 tokwise sync --liked <username>
@@ -155,7 +201,7 @@ tokwise sync --collection "name-123"   # uses the @handle saved with your cookie
 
 The bare-slug form needs the username tied to your cookie. Tokwise tries to detect it automatically when you run `tokwise auth set` or `tokwise auth from-browser`; you can also set it explicitly with `tokwise auth set-username <handle>` or `--username` on those commands. `tokwise auth show` reports the saved handle.
 
-Private collections usually require a fresh browser cookie from a logged-in session. On macOS, `tokwise auth from-browser` reads and decrypts it straight from a logged-in Chromium browser (Chrome, Brave, Edge, Arc, or Chromium) via the macOS Keychain, and `tokwise auth refresh` re-pulls it when the session goes stale. On other platforms or browsers, paste it manually with `tokwise auth set`. Cookies are stored locally only (`auth.json`, chmod 600).
+Private collections usually require a fresh browser cookie from a logged-in session. On macOS, `tokwise auth from-browser` reads and decrypts it straight from a logged-in Chromium browser (Chrome, Brave, Edge, Arc, Dia, or Chromium) via the macOS Keychain, and `tokwise auth refresh` re-pulls it when the session goes stale. On other platforms or browsers, paste it manually with `tokwise auth set`. Cookies are stored locally only (`auth.json`, chmod 600).
 
 ## Transcription
 

package/dist/ask.js

@@ -1,3 +1,4 @@
+import { videoReference } from "./reference.js";
 export async function answerQuestion(question, results, options) {
     if (results.length === 0)
         return "No local evidence matched that question.";
@@ -13,9 +14,8 @@ function answerExtractively(question, results) {
         "",
         ...results.slice(0, 8).map((result, idx) => {
             const video = result.video;
-            const author = video.author?.username ? `@${video.author.username}` : "unknown";
             const summary = video.classification?.summary ?? result.highlights[0] ?? video.description ?? "No text.";
-            return `${idx + 1}. ${video.id} ${author}\n   ${summary}\n   ${video.canonicalUrl ?? video.url}`;
+            return `${idx + 1}. ${videoReference(video)}\n   ${summary}\n   ${video.canonicalUrl ?? video.url}`;
         }),
         "",
         "Use --engine ollama --model <model> for a synthesized answer from a local model.",
@@ -29,8 +29,8 @@ async function answerWithOllama(question, results, options) {
         .map((result, idx) => {
         const video = result.video;
         return [
-            `[${idx + 1}] ${video.id} ${video.canonicalUrl ?? video.url}`,
-            `Author: ${video.author?.username ?? "unknown"}`,
+            `[${idx + 1}] ${videoReference(video)}`,
+            `Source: ${video.canonicalUrl ?? video.url}`,
             `Category: ${video.classification?.category ?? "unknown"}`,
             `Summary: ${video.classification?.summary ?? ""}`,
             `Transcript: ${(video.transcript?.text ?? video.description ?? "").slice(0, 2500)}`,
@@ -39,7 +39,7 @@ async function answerWithOllama(question, results, options) {
         .join("\n\n");
     const prompt = [
         "Answer the user's question using only the saved clip evidence below.",
-        "Cite video ids in brackets when making claims. If evidence is thin, say so.",
+        "Cite clips by their readable reference (e.g. @author \u00b7 Mon YYYY \u2014 \"title\") when making claims. If evidence is thin, say so.",
         "",
         `Question: ${question}`,
         "",

package/dist/browser-cookies.js

@@ -3,12 +3,13 @@ import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { runProcess } from "./process.js";
-export const SUPPORTED_BROWSERS = ["chrome", "brave", "edge", "arc", "chromium"];
+export const SUPPORTED_BROWSERS = ["chrome", "brave", "edge", "arc", "dia", "chromium"];
 const CHANNELS = {
     chrome: { dir: "Google/Chrome", service: "Chrome Safe Storage", account: "Chrome" },
     brave: { dir: "BraveSoftware/Brave-Browser", service: "Brave Safe Storage", account: "Brave" },
     edge: { dir: "Microsoft Edge", service: "Microsoft Edge Safe Storage", account: "Microsoft Edge" },
     arc: { dir: "Arc/User Data", service: "Arc Safe Storage", account: "Arc" },
+    dia: { dir: "Dia/User Data", service: "Dia Safe Storage", account: "Dia" },
     chromium: { dir: "Chromium", service: "Chromium Safe Storage", account: "Chromium" },
 };
 export function isChromiumBrowser(value) {

package/dist/cli.js

@@ -13,11 +13,15 @@ import { commandsDir, dataDir, ensureDataDirs, libraryDir, searchIndexPath, toDi
 import { compileWiki, exportMarkdown, lintWiki } from "./markdown.js";
 import { downloadMedia } from "./media.js";
 import { formatSearchResults, loadSearchIndex, saveSearchIndex, searchWithIndex } from "./search.js";
-import { detectLoggedInUsername, fetchCollection, fetchLiked, fetchPlaylist, fetchSingleUrl, fetchUserPosts, fetchVideoSearch, videosFromImport, videosFromUrls } from "./tiktok.js";
+import { detectLoggedInUsername, fetchBookmarkFolders, fetchCollection, fetchLiked, fetchPlaylist, fetchSingleUrl, fetchUserPosts, fetchVideoSearch, resolveSecUid, videosFromImport, videosFromUrls } from "./tiktok.js";
 import { transcribeVideo } from "./transcribe.js";
+import { detectEngines } from "./engines.js";
+import { sleep } from "./http.js";
+import { cancelled, intro, isInteractive, promptFolderSelection, promptPipelineSelection } from "./interactive.js";
 import { createCommand, createLibraryPage, deleteLibraryPage, listCommands, searchLibrary, showLibraryPage, updateLibraryPage, validateCommands } from "./library.js";
 import { installSkill, skillContent, uninstallSkill } from "./skill.js";
-import { barChart, box, c, kvList, setColorEnabled, truncate } from "./render.js";
+import { barChart, box, c, kvList, setColorEnabled, table, truncate } from "./render.js";
+import { formatReference } from "./reference.js";
 import { createProgress } from "./progress.js";
 const require = createRequire(import.meta.url);
 function version() {
@@ -184,6 +188,9 @@ export function buildCli() {
     program
         .command("sync")
         .description("Sync short-form video sources into the local archive")
+        .option("--list", "List discovered bookmarks instead of syncing", false)
+        .option("--json", "With --list, print bookmarks as JSON", false)
+        .option("--all", "Sync every discovered bookmark without prompting", false)
         .option("--collection <idOrUrl>", "Collection URL, @user/collection/slug, or bare slug/id", collect, [])
         .option("--playlist <idOrUrl>", "Playlist id or URL", collect, [])
         .option("--liked <username>", "Sync a user's liked videos; usually requires cookie", collect, [])
@@ -198,6 +205,8 @@ export function buildCli() {
         .option("--limit <n>", "Max items per source", parseNumber, 30)
         .option("--page <n>", "Start page", parseNumber, 1)
         .option("--pages <n>", "Max pages per paged source", parseNumber)
+        .option("--request-delay <ms>", "Delay between API requests to avoid rate limits", parseNumber, 500)
+        .option("--max-retries <n>", "Retries per request on rate limit (429) or server errors", parseNumber, 3)
         .option("--rebuild", "Replace archive with this sync result", false)
         .option("--download", "Download media after syncing", false)
         .option("--audio", "When downloading, extract audio only", false)
@@ -217,14 +226,21 @@ export function buildCli() {
         ensureDataDirs();
         const cookie = await loadCookie({ cookie: options.cookie, cookieFile: options.cookieFile });
         const auth = await loadAuth();
+        if (!hasExplicitSource(options)) {
+            await runInteractiveSync(options, cookie, auth);
+            return;
+        }
         const discovered = [];
         const fetchOptions = {
             cookie,
             username: auth.username,
+            secUid: auth.secUid,
             proxy: options.proxy,
             limit: Number(options.limit),
             page: Number(options.page),
             pages: options.pages == null ? undefined : Number(options.pages),
+            requestDelayMs: Number(options.requestDelay),
+            maxRetries: Number(options.maxRetries),
         };
         for (const value of options.collection)
             discovered.push(...(await fetchCollection(value, fetchOptions)));
@@ -250,36 +266,7 @@ export function buildCli() {
         }
         if (discovered.length === 0)
             throw new Error("No source supplied. Try --collection, --url, --urls-file, or --input.");
-        const sync = await mergeVideos(discovered, { rebuild: Boolean(options.rebuild) });
-        console.log(`Synced ${sync.added} new, ${sync.updated} updated, ${sync.unchanged} unchanged (${sync.total} total).`);
-        let videos = await loadVideos();
-        const touched = new Set(sync.ids);
-        if (options.download) {
-            videos = await runDownloads(videos, touched, {
-                ytDlp: options.ytDlp,
-                proxy: options.proxy,
-                cookiesFile: options.ytDlpCookies,
-                cookiesFromBrowser: options.cookiesFromBrowser,
-                audioOnly: Boolean(options.audio),
-            });
-        }
-        if (options.transcribe) {
-            videos = await runTranscription(videos, touched, {
-                engine: options.sttEngine,
-                command: options.sttCommand,
-                model: options.sttModel,
-                language: options.language,
-            });
-        }
-        if (options.classify) {
-            videos = await runClassification(videos, touched, {
-                engine: options.engine,
-                model: options.model,
-                ollamaBaseUrl: options.ollamaUrl,
-            });
-        }
-        const index = await saveSearchIndex(videos);
-        console.log(`Indexed ${index.recordCount} videos at ${toDisplayPath(searchIndexPath())}.`);
+        await runSyncPipeline([{ label: "sync", videos: discovered }], options);
     }));
     program
         .command("fetch-media")
@@ -548,10 +535,13 @@ export function buildCli() {
     }));
     return program;
 }
-async function runDownloads(videos, touched, options) {
+export function progressLabel(base, label) {
+    return label ? `${base} \u00b7 ${label}` : base;
+}
+async function runDownloads(videos, touched, options, label) {
     const next = [...videos];
     const total = next.filter((video) => touched.has(video.id)).length;
-    const progress = createProgress({ total, label: "media" });
+    const progress = createProgress({ total, label: progressLabel("media", label) });
     let downloaded = 0;
     let present = 0;
     const failed = [];
@@ -574,13 +564,14 @@ async function runDownloads(videos, touched, options) {
     }
     progress.done();
     await saveVideos(next);
-    console.log(`Media: ${downloaded} downloaded, ${present} already present${failed.length ? `, ${failed.length} failed (${failed.join(", ")})` : ""} (${total} total).`);
+    const indent = label ? "  " : "";
+    console.log(`${indent}Media: ${downloaded} downloaded, ${present} already present${failed.length ? `, ${failed.length} failed (${failed.join(", ")})` : ""} (${total} total).`);
     return next;
 }
-async function runTranscription(videos, touched, options) {
+async function runTranscription(videos, touched, options, label) {
     const next = [...videos];
     const total = next.filter((video) => touched.has(video.id)).length;
-    const progress = createProgress({ total, label: "transcribe" });
+    const progress = createProgress({ total, label: progressLabel("transcribe", label) });
     let transcribed = 0;
     let present = 0;
     const failed = [];
@@ -604,13 +595,14 @@ async function runTranscription(videos, touched, options) {
     }
     progress.done();
     await saveVideos(next);
-    console.log(`Transcripts: ${transcribed} transcribed, ${present} already present${failed.length ? `, ${failed.length} failed (${failed.join(", ")})` : ""} (${total} total).`);
+    const indent = label ? "  " : "";
+    console.log(`${indent}Transcripts: ${transcribed} transcribed, ${present} already present${failed.length ? `, ${failed.length} failed (${failed.join(", ")})` : ""} (${total} total).`);
     return next;
 }
-async function runClassification(videos, touched, options) {
+async function runClassification(videos, touched, options, label) {
     const next = [...videos];
     const total = next.filter((video) => touched.has(video.id)).length;
-    const progress = createProgress({ total, label: "classify" });
+    const progress = createProgress({ total, label: progressLabel("classify", label) });
     let classified = 0;
     const failed = [];
     for (const [idx, video] of next.entries()) {
@@ -629,9 +621,188 @@ async function runClassification(videos, touched, options) {
     }
     progress.done();
     await saveVideos(next);
-    console.log(`Classified ${classified} videos${failed.length ? `, ${failed.length} failed (${failed.join(", ")})` : ""} (${total} total).`);
+    const indent = label ? "  " : "";
+    console.log(`${indent}Classified ${classified} videos${failed.length ? `, ${failed.length} failed (${failed.join(", ")})` : ""} (${total} total).`);
     return next;
 }
+export function hasExplicitSource(options) {
+    return (options.collection.length > 0 ||
+        options.playlist.length > 0 ||
+        options.liked.length > 0 ||
+        options.user.length > 0 ||
+        options.searchVideo.length > 0 ||
+        options.url.length > 0 ||
+        Boolean(options.urlsFile) ||
+        Boolean(options.input));
+}
+async function runSyncPipeline(groups, options) {
+    const showPerGroup = groups.length > 1;
+    let videos = await loadVideos();
+    for (const [index, group] of groups.entries()) {
+        const groupLabel = showPerGroup ? group.label : undefined;
+        if (showPerGroup)
+            console.log(c.heading(group.label));
+        const sync = await mergeVideos(group.videos, { rebuild: Boolean(options.rebuild) && index === 0 });
+        console.log(showPerGroup
+            ? `  Synced ${sync.added} new, ${sync.updated} updated, ${sync.unchanged} unchanged`
+            : `Synced ${sync.added} new, ${sync.updated} updated, ${sync.unchanged} unchanged (${sync.total} total).`);
+        videos = await loadVideos();
+        const touched = new Set(sync.ids);
+        if (options.download) {
+            videos = await runDownloads(videos, touched, {
+                ytDlp: options.ytDlp,
+                proxy: options.proxy,
+                cookiesFile: options.ytDlpCookies,
+                cookiesFromBrowser: options.cookiesFromBrowser,
+                audioOnly: Boolean(options.audio),
+            }, groupLabel);
+        }
+        if (options.transcribe) {
+            videos = await runTranscription(videos, touched, {
+                engine: options.sttEngine,
+                command: options.sttCommand,
+                model: options.sttModel,
+                language: options.language,
+            }, groupLabel);
+        }
+        if (options.classify) {
+            videos = await runClassification(videos, touched, {
+                engine: options.engine,
+                model: options.model,
+                ollamaBaseUrl: options.ollamaUrl,
+            }, groupLabel);
+        }
+        if (showPerGroup)
+            console.log("");
+    }
+    const index = await saveSearchIndex(videos);
+    console.log(`Indexed ${index.recordCount} videos at ${toDisplayPath(searchIndexPath())}.`);
+}
+function printBookmarkFolders(folders, options) {
+    if (options.json) {
+        console.log(JSON.stringify(folders.map((folder) => ({
+            id: folder.id,
+            name: folder.name,
+            itemCount: folder.itemCount ?? null,
+            url: folder.url ?? null,
+            kind: folder.kind,
+        })), null, 2));
+        return;
+    }
+    const rows = folders.map((folder) => [
+        folder.name,
+        folder.kind,
+        typeof folder.itemCount === "number" ? String(folder.itemCount) : "",
+        c.muted(folder.id),
+    ]);
+    console.log(table(["Name", "Kind", "Videos", "ID"], rows));
+}
+async function runInteractiveSync(options, cookie, auth) {
+    if (options.json)
+        setColorEnabled(false);
+    if (!cookie) {
+        throw new Error("No cookie saved. Run `tw auth from-browser` (or `tw auth set --cookie ...`) to enable bookmark sync.");
+    }
+    let secUid = auth.secUid;
+    if (!secUid) {
+        secUid = await resolveSecUid(cookie, auth.username);
+        if (secUid)
+            await saveAuth({ ...auth, secUid, updatedAt: new Date().toISOString() });
+    }
+    const folders = await fetchBookmarkFolders({
+        cookie,
+        username: auth.username,
+        secUid,
+        proxy: options.proxy,
+        limit: Number(options.limit),
+    });
+    if (folders.length === 0) {
+        console.log("No bookmarks found. Make sure your cookie is fresh (try `tw auth refresh`).");
+        return;
+    }
+    if (options.list) {
+        printBookmarkFolders(folders, { json: Boolean(options.json) });
+        return;
+    }
+    let selected;
+    if (options.all) {
+        selected = folders;
+    }
+    else if (isInteractive()) {
+        intro(`Found ${folders.length} bookmarks${auth.username ? ` for @${auth.username}` : ""}`);
+        const picked = await promptFolderSelection(folders);
+        if (!picked) {
+            cancelled("Sync cancelled.");
+            return;
+        }
+        selected = picked;
+    }
+    else {
+        printBookmarkFolders(folders, { json: false });
+        console.log("");
+        console.log(`${c.muted("Hint")} run \`tokwise sync --all\` to sync everything, \`--collection <id>\` for one, or \`--list --json\` for machine output.`);
+        return;
+    }
+    if (selected.length === 0) {
+        console.log("Nothing selected.");
+        return;
+    }
+    const pipelineFlagsGiven = Boolean(options.download || options.transcribe || options.classify);
+    if (isInteractive() && !options.all && !pipelineFlagsGiven) {
+        const engines = await detectEngines({ ytDlp: options.ytDlp });
+        const pipeline = await promptPipelineSelection({
+            download: engines.ytDlp,
+            transcribe: engines.ytDlp && engines.whisper,
+            classify: true,
+        });
+        if (!pipeline) {
+            cancelled("Sync cancelled.");
+            return;
+        }
+        options.download = pipeline.download;
+        options.transcribe = pipeline.transcribe;
+        options.classify = pipeline.classify;
+        if (pipeline.download && !options.audio)
+            options.audio = true;
+    }
+    const requestDelayMs = Number(options.requestDelay);
+    const fetchOptions = {
+        cookie,
+        username: auth.username,
+        secUid,
+        proxy: options.proxy,
+        limit: Number(options.limit),
+        page: Number(options.page),
+        pages: options.pages == null ? undefined : Number(options.pages),
+        requestDelayMs,
+        maxRetries: Number(options.maxRetries),
+    };
+    const groups = [];
+    const failed = [];
+    for (const [index, folder] of selected.entries()) {
+        if (index > 0 && requestDelayMs > 0)
+            await sleep(requestDelayMs);
+        try {
+            const videos = await fetchCollection(folder.url ?? folder.id, fetchOptions);
+            if (videos.length > 0)
+                groups.push({ label: folder.name, videos });
+        }
+        catch (error) {
+            failed.push(folder.name);
+            console.error(c.warn(`Skipped "${folder.name}": ${error.message}`));
+        }
+    }
+    if (failed.length > 0) {
+        const names = failed.map((name) => `"${name}"`).join(", ");
+        const suggestion = Math.max(1500, requestDelayMs * 2);
+        console.error(c.warn(`${failed.length} collection${failed.length === 1 ? "" : "s"} skipped after retries: ${names}. Re-run or try --request-delay ${suggestion}.`));
+    }
+    if (groups.length === 0) {
+        console.log("No videos found in the selected bookmarks.");
+        return;
+    }
+    await runSyncPipeline(groups, options);
+}
 async function readImport(filePath) {
     const text = await readTextInput(filePath);
     const trimmed = text.trim();
@@ -677,12 +848,11 @@ function formatList(videos) {
         return c.muted("No videos.");
     return videos
         .map((video) => {
-        const author = video.author?.username ? c.accent(`@${video.author.username}`) : c.muted("unknown");
         const category = video.classification?.category ? ` ${c.warn(`[${video.classification.category}]`)}` : "";
         const transcript = video.transcript?.text ? ` ${c.success("transcript")}` : "";
         const desc = truncate((video.description ?? "").replace(/\s+/g, " "), 160);
         return [
-            `${c.value(video.id)} ${author}${category}${transcript}`,
+            `${formatReference(video)}${category}${transcript}`,
             `  ${desc}`,
             `  ${c.muted(video.canonicalUrl ?? video.url)}`,
         ].join("\n");
@@ -691,7 +861,7 @@ function formatList(videos) {
 }
 function formatVideo(video) {
     return [
-        `${c.value(video.id)} ${video.author?.username ? c.accent(`@${video.author.username}`) : ""}`.trimEnd(),
+        formatReference(video),
         c.muted(video.canonicalUrl ?? video.url),
         "",
         video.description ?? "",
@@ -869,7 +1039,7 @@ async function showDashboard() {
             indexExists: await fileExists(searchIndexPath()),
         }),
         "",
-        `${c.muted("Next")}    tokwise sync --collection <url> --download --audio --transcribe --classify`,
+        `${c.muted("Next")}    tokwise sync ${c.muted("(pick your bookmarks)")} ${c.muted("|")} tokwise sync --collection <url> --download --audio --transcribe --classify`,
         `${c.muted("Explore")} tokwise search "life advice" ${c.muted("|")} tokwise viz ${c.muted("|")} tokwise wiki`,
     ].join("\n"));
 }

package/dist/engines.js

@@ -0,0 +1,36 @@
+import { access, constants } from "node:fs/promises";
+import path from "node:path";
+async function canExecute(file) {
+    try {
+        await access(file, constants.X_OK);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// Detect a command by resolving it on PATH rather than executing it with a
+// flag: tools disagree on whether `--version`/`--help` exit 0 (OpenAI whisper
+// errors on `--version`), so a PATH lookup is the only reliable presence check.
+async function isOnPath(command) {
+    if (command.includes(path.sep))
+        return canExecute(command);
+    const dirs = (process.env.PATH ?? "").split(path.delimiter).filter(Boolean);
+    const exts = process.platform === "win32" ? (process.env.PATHEXT ?? ".EXE;.CMD;.BAT").split(";") : [""];
+    for (const dir of dirs) {
+        for (const ext of exts) {
+            if (await canExecute(path.join(dir, command + ext)))
+                return true;
+        }
+    }
+    return false;
+}
+export async function detectEngines(options) {
+    const ytDlpCommand = options?.ytDlp ?? "yt-dlp";
+    const [ytDlp, whisper, whisperCpp] = await Promise.all([
+        isOnPath(ytDlpCommand),
+        isOnPath("whisper"),
+        isOnPath("whisper-cli"),
+    ]);
+    return { ytDlp, whisper: whisper || whisperCpp };
+}

package/dist/http.js

@@ -0,0 +1,42 @@
+const MAX_BACKOFF_MS = 30_000;
+const MAX_RETRY_AFTER_MS = 60_000;
+export function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
+}
+export function shouldRetry(status) {
+    return status === 429 || (status >= 500 && status <= 599);
+}
+function parseRetryAfterMs(retryAfter) {
+    if (!retryAfter)
+        return undefined;
+    const trimmed = retryAfter.trim();
+    const seconds = Number(trimmed);
+    if (Number.isFinite(seconds))
+        return Math.max(0, seconds * 1000);
+    const date = Date.parse(trimmed);
+    if (!Number.isNaN(date))
+        return Math.max(0, date - Date.now());
+    return undefined;
+}
+export function retryDelayMs(status, retryAfter, attempt, baseDelayMs = 1000) {
+    const backoff = Math.min(baseDelayMs * 2 ** Math.max(0, attempt), MAX_BACKOFF_MS);
+    const jitter = Math.random() * baseDelayMs;
+    const computed = Math.min(backoff + jitter, MAX_BACKOFF_MS);
+    const headerMs = parseRetryAfterMs(retryAfter);
+    if (headerMs != null)
+        return Math.min(Math.max(headerMs, computed), MAX_RETRY_AFTER_MS);
+    return computed;
+}
+export async function fetchWithRetry(input, init, options = {}) {
+    const maxRetries = options.maxRetries ?? 3;
+    const baseDelayMs = options.baseDelayMs ?? 1000;
+    const doFetch = options.fetchImpl ?? fetch;
+    let attempt = 0;
+    for (;;) {
+        const response = await doFetch(input, init);
+        if (!shouldRetry(response.status) || attempt >= maxRetries)
+            return response;
+        await sleep(retryDelayMs(response.status, response.headers.get("retry-after"), attempt, baseDelayMs));
+        attempt += 1;
+    }
+}

package/dist/interactive.js

@@ -0,0 +1,60 @@
+import * as p from "@clack/prompts";
+export function isInteractive() {
+    return Boolean(process.stdout.isTTY && process.stdin.isTTY);
+}
+export function intro(message) {
+    p.intro(message);
+}
+export function cancelled(message) {
+    p.cancel(message);
+}
+function folderHint(folder) {
+    const parts = [];
+    if (folder.kind === "favorites")
+        parts.push("saved videos");
+    if (typeof folder.itemCount === "number")
+        parts.push(`${folder.itemCount} videos`);
+    return parts.length ? parts.join(" \u00b7 ") : undefined;
+}
+export async function promptFolderSelection(folders) {
+    const selection = await p.multiselect({
+        message: "Select bookmarks to sync",
+        options: folders.map((folder) => ({
+            value: folder.id,
+            label: folder.name,
+            hint: folderHint(folder),
+        })),
+        required: false,
+    });
+    if (p.isCancel(selection))
+        return undefined;
+    const ids = new Set(selection);
+    return folders.filter((folder) => ids.has(folder.id));
+}
+export async function promptPipelineSelection(defaults) {
+    const initialValues = [];
+    if (defaults.download)
+        initialValues.push("download");
+    if (defaults.transcribe)
+        initialValues.push("transcribe");
+    if (defaults.classify)
+        initialValues.push("classify");
+    const selection = await p.multiselect({
+        message: "What should we do with these?",
+        options: [
+            { value: "download", label: "Download audio" },
+            { value: "transcribe", label: "Transcribe" },
+            { value: "classify", label: "Classify" },
+        ],
+        initialValues,
+        required: false,
+    });
+    if (p.isCancel(selection))
+        return undefined;
+    const chosen = new Set(selection);
+    return {
+        download: chosen.has("download"),
+        transcribe: chosen.has("transcribe"),
+        classify: chosen.has("classify"),
+    };
+}

package/dist/markdown.js

@@ -3,6 +3,7 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import { ensureDataDirs, libraryDir, markdownCategoriesDir, markdownDomainsDir, markdownVideosDir, } from "./paths.js";
 import { sanitizeFilePart } from "./store.js";
+import { videoTitle } from "./reference.js";
 export async function exportMarkdown(videos, options) {
     ensureDataDirs();
     let written = 0;
@@ -48,7 +49,7 @@ export function videoMarkdownPath(video) {
     return path.join(markdownVideosDir(), `${sanitizeFilePart(video.id)}.md`);
 }
 export function renderVideoMarkdown(video) {
-    const title = video.description?.split(/\r?\n/)[0]?.slice(0, 80) || `Clip ${video.id}`;
+    const title = videoTitle(video);
     const category = video.classification?.category ?? "uncategorized";
     const domain = video.classification?.domain ?? "general";
     const topics = video.classification?.topics ?? [];
@@ -160,9 +161,6 @@ function renderGroupPage(kind, name, videos) {
         "",
     ].join("\n");
 }
-function videoTitle(video) {
-    return video.description?.replace(/\s+/g, " ").slice(0, 80) || video.id;
-}
 function groupBy(items, keyFn) {
     const groups = new Map();
     for (const item of items) {

package/dist/reference.js

@@ -0,0 +1,59 @@
+import { c } from "./render.js";
+const TITLE_MAX = 60;
+const SHORT_ID_LENGTH = 8;
+export function videoTitle(video) {
+    const candidates = [
+        video.description?.split(/\r?\n/)[0],
+        video.classification?.summary,
+        video.classification?.topics?.[0],
+        video.classification?.category,
+    ];
+    for (const candidate of candidates) {
+        const cleaned = cleanTitle(candidate);
+        if (cleaned)
+            return cleaned;
+    }
+    return "Untitled clip";
+}
+export function shortId(video) {
+    return video.id.slice(-SHORT_ID_LENGTH);
+}
+export function videoReference(video) {
+    const author = video.author?.username
+        ? `@${video.author.username}`
+        : video.author?.displayName || "unknown";
+    const date = referenceDate(video);
+    const title = videoTitle(video);
+    const head = [author, date].filter(Boolean).join(" \u00b7 ");
+    return `${head} \u2014 "${title}"  #${shortId(video)}`;
+}
+export function formatReference(video) {
+    const author = video.author?.username
+        ? c.accent(`@${video.author.username}`)
+        : c.muted(video.author?.displayName || "unknown");
+    const date = referenceDate(video);
+    const datePart = date ? ` ${c.muted("\u00b7")} ${c.muted(date)}` : "";
+    const title = c.value(`"${videoTitle(video)}"`);
+    return `${author}${datePart} ${c.muted("\u2014")} ${title}  ${c.muted(`#${shortId(video)}`)}`;
+}
+function referenceDate(video) {
+    const iso = video.createdAt ?? video.savedAt;
+    if (!iso)
+        return "";
+    const date = new Date(iso);
+    if (Number.isNaN(date.getTime()))
+        return "";
+    return date.toLocaleDateString("en-US", { year: "numeric", month: "short" });
+}
+function cleanTitle(value) {
+    if (!value)
+        return "";
+    const collapsed = value.replace(/\s+/g, " ").trim();
+    const withoutTrailingTags = collapsed.replace(/(?:\s+#[\p{L}\p{N}_]+)+\s*$/u, "").trim();
+    const base = withoutTrailingTags || collapsed;
+    if (!base)
+        return "";
+    if (base.length <= TITLE_MAX)
+        return base;
+    return `${base.slice(0, TITLE_MAX - 1).trimEnd()}\u2026`;
+}

package/dist/search.js

@@ -2,6 +2,7 @@ import fs from "node:fs/promises";
 import { searchIndexPath } from "./paths.js";
 import { readJsonFile, writeJsonFile } from "./jsonl.js";
 import { c } from "./render.js";
+import { formatReference } from "./reference.js";
 const STOP_WORDS = new Set([
     "a",
     "an",
@@ -214,10 +215,9 @@ export function formatSearchResults(results, options) {
     return results
         .map((result, idx) => {
         const video = result.video;
-        const author = video.author?.username ? c.accent(`@${video.author.username}`) : c.muted("unknown");
         const category = video.classification?.category ? ` ${c.warn(`[${video.classification.category}]`)}` : "";
         const score = result.score > 0 ? ` ${c.muted(`score ${result.score.toFixed(2)}`)}` : "";
-        const line = `${c.muted(`${idx + 1}.`)} ${c.value(video.id)} ${author}${category}${score}`;
+        const line = `${c.muted(`${idx + 1}.`)} ${formatReference(video)}${category}${score}`;
         const desc = video.description ? `   ${video.description.replace(/\s+/g, " ").slice(0, 160)}` : "";
         const hit = result.highlights[0] ? `   ${c.muted(">")} ${c.success(result.highlights[0])}` : "";
         return [line, desc, hit, `   ${c.muted(video.canonicalUrl ?? video.url)}`].filter(Boolean).join("\n");

package/dist/skill.js

@@ -24,7 +24,7 @@ export function skillContent() {
         "",
         "## Grounding",
         "",
-        "Cite video ids or Markdown page paths when drawing conclusions. Treat transcripts as user-owned local context and do not assume videos are public.",
+        "Cite clips by their readable reference (`@author \u00b7 Mon YYYY \u2014 \"title\"`, optionally with the trailing short id like `#49952278`) or by Markdown page path when drawing conclusions. Run `tokwise show <short-id-or-url>` to pull a clip back up. Treat transcripts as user-owned local context and do not assume videos are public.",
         "",
     ].join("\n");
 }

package/dist/store.js

@@ -94,7 +94,8 @@ export function findVideo(videos, query) {
     const normalized = query.trim();
     return (videos.find((video) => video.id === normalized) ??
         videos.find((video) => video.id.startsWith(normalized)) ??
-        videos.find((video) => video.url === normalized || video.canonicalUrl === normalized));
+        videos.find((video) => video.url === normalized || video.canonicalUrl === normalized) ??
+        videos.find((video) => video.id.endsWith(normalized)));
 }
 export async function updateVideosById(updates) {
     const videos = await loadVideos();

package/dist/tiktok.js

@@ -1,4 +1,6 @@
 import { stableHash, uniqueStrings } from "./store.js";
+import { fetchWithRetry, sleep } from "./http.js";
+const WEB_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36";
 async function loadApi() {
     const mod = (await import("@tobyg74/tiktok-api-dl"));
     return mod.default ?? mod;
@@ -97,15 +99,19 @@ async function fetchPaged(fn, idOrUrl, options, context) {
     const limit = options.limit ?? 30;
     const pageStart = options.page ?? 1;
     const maxPages = options.pages ?? Math.ceil(limit / 30);
+    const requestDelayMs = options.requestDelayMs ?? 0;
     const videos = [];
     let page = pageStart;
     for (let i = 0; i < maxPages && videos.length < limit; i += 1) {
+        if (i > 0 && requestDelayMs > 0)
+            await sleep(requestDelayMs);
         const response = await fn(idOrUrl, {
             page,
             count: Math.min(30, limit - videos.length),
             postLimit: Math.min(30, limit - videos.length),
             cookie: options.cookie,
             proxy: options.proxy,
+            maxRetries: options.maxRetries,
         });
         const items = extractItemsFromSuccessfulResponse(response, context.source);
         videos.push(...items.map((item) => normalizeVideo(item, context)));
@@ -162,16 +168,16 @@ async function fetchCollectionWithCookie(idOrUrl, options) {
         user_is_login: "true",
         webcast_language: "en",
     });
-    const response = await fetch(`https://www.tiktok.com/api/collection/item_list/?${params.toString()}`, {
+    const response = await fetchWithRetry(`https://www.tiktok.com/api/collection/item_list/?${params.toString()}`, {
         method: "GET",
         headers: {
-            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+            "user-agent": WEB_USER_AGENT,
             accept: "application/json, text/plain, */*",
             "accept-language": "en-US,en;q=0.9",
             cookie: String(options.cookie ?? ""),
             referer: looksLikeUrl(idOrUrl) ? idOrUrl : "https://www.tiktok.com/",
         },
-    });
+    }, { maxRetries: numberValue(options.maxRetries) });
     if (!response.ok) {
         return {
             status: "error",
@@ -180,29 +186,42 @@ async function fetchCollectionWithCookie(idOrUrl, options) {
     }
     return response.json();
 }
-export function extractUsernameFromRehydrationHtml(html) {
+function parseRehydrationScope(html) {
     const match = html.match(/__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)<\/script>/s);
     if (!match || !match[1])
         return undefined;
     try {
         const data = asRecord(JSON.parse(match[1]));
-        const scope = asRecord(data?.["__DEFAULT_SCOPE__"]);
-        const appContext = asRecord(scope?.["webapp.app-context"]);
-        const user = asRecord(appContext?.user);
-        return stringValue(user?.uniqueId);
+        return asRecord(data?.["__DEFAULT_SCOPE__"]);
     }
     catch {
         return undefined;
     }
 }
-export async function detectLoggedInUsername(cookie, proxy) {
-    if (!cookie)
-        return undefined;
+export function extractUsernameFromRehydrationHtml(html) {
+    const scope = parseRehydrationScope(html);
+    const appContext = asRecord(scope?.["webapp.app-context"]);
+    const user = asRecord(appContext?.user);
+    return stringValue(user?.uniqueId);
+}
+export function extractSecUidFromRehydrationHtml(html) {
+    const scope = parseRehydrationScope(html);
+    const appContext = asRecord(scope?.["webapp.app-context"]);
+    const contextUser = asRecord(appContext?.user);
+    const fromContext = stringValue(contextUser?.secUid);
+    if (fromContext)
+        return fromContext;
+    const userDetail = asRecord(scope?.["webapp.user-detail"]);
+    const userInfo = asRecord(userDetail?.userInfo);
+    const detailUser = asRecord(userInfo?.user);
+    return stringValue(detailUser?.secUid);
+}
+async function fetchRehydrationHtml(url, cookie) {
     try {
-        const response = await fetch("https://www.tiktok.com/", {
+        const response = await fetch(url, {
             method: "GET",
             headers: {
-                "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+                "user-agent": WEB_USER_AGENT,
                 accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                 "accept-language": "en-US,en;q=0.9",
                 cookie,
@@ -210,12 +229,157 @@ export async function detectLoggedInUsername(cookie, proxy) {
         });
         if (!response.ok)
             return undefined;
-        return extractUsernameFromRehydrationHtml(await response.text());
+        return await response.text();
     }
     catch {
         return undefined;
     }
 }
+export async function detectLoggedInUsername(cookie, proxy) {
+    if (!cookie)
+        return undefined;
+    const html = await fetchRehydrationHtml("https://www.tiktok.com/", cookie);
+    return html ? extractUsernameFromRehydrationHtml(html) : undefined;
+}
+export async function resolveSecUid(cookie, username, proxy) {
+    if (!cookie)
+        return undefined;
+    const home = await fetchRehydrationHtml("https://www.tiktok.com/", cookie);
+    const fromHome = home ? extractSecUidFromRehydrationHtml(home) : undefined;
+    if (fromHome)
+        return fromHome;
+    if (username) {
+        const profile = await fetchRehydrationHtml(`https://www.tiktok.com/@${username}`, cookie);
+        const fromProfile = profile ? extractSecUidFromRehydrationHtml(profile) : undefined;
+        if (fromProfile)
+            return fromProfile;
+    }
+    return undefined;
+}
+const COLLECTION_LIST_PAGE_SIZE = 30;
+const MAX_BOOKMARK_FOLDERS = 500;
+const FAVORITES_NAMES = new Set(["favorites", "favorite", "favourites", "favourite"]);
+export async function fetchBookmarkFolders(options) {
+    const { cookie } = options;
+    if (!cookie) {
+        throw new Error("A browser cookie is required to list bookmarks. Run `tw auth from-browser` first.");
+    }
+    const secUid = options.secUid ?? (await resolveSecUid(cookie, options.username, options.proxy));
+    if (!secUid) {
+        throw new Error("Could not resolve your TikTok secUid. Run `tw auth from-browser` (or `tw auth set-username <handle>`) and try again.");
+    }
+    const folders = [];
+    const seen = new Set();
+    let cursor = "0";
+    const requestDelayMs = options.requestDelayMs ?? 0;
+    let firstPage = true;
+    while (folders.length < MAX_BOOKMARK_FOLDERS) {
+        if (!firstPage && requestDelayMs > 0)
+            await sleep(requestDelayMs);
+        firstPage = false;
+        const response = await fetchCollectionList(secUid, cursor, COLLECTION_LIST_PAGE_SIZE, cookie, options.maxRetries);
+        assertSuccessfulResponse(response, "collection");
+        const entries = extractCollectionListEntries(response);
+        for (const entry of entries) {
+            const folder = normalizeBookmarkFolder(entry, options.username);
+            if (!folder || seen.has(folder.id))
+                continue;
+            seen.add(folder.id);
+            folders.push(folder);
+        }
+        if (!hasMore(response) || entries.length === 0)
+            break;
+        const next = readCursor(response);
+        if (!next || next === cursor)
+            break;
+        cursor = next;
+    }
+    return folders;
+}
+async function fetchCollectionList(secUid, cursor, count, cookie, maxRetries) {
+    const params = new URLSearchParams({
+        WebIdLastTime: String(Date.now()),
+        aid: "1988",
+        app_language: "en",
+        app_name: "tiktok_web",
+        browser_language: "en-US",
+        browser_name: "Mozilla",
+        browser_online: "true",
+        browser_platform: "MacIntel",
+        browser_version: "5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+        channel: "tiktok_web",
+        cookie_enabled: "true",
+        count: String(count),
+        cursor,
+        device_platform: "web_pc",
+        focus_state: "true",
+        from_page: "user",
+        history_len: "3",
+        is_fullscreen: "false",
+        is_page_visible: "true",
+        language: "en",
+        needPinnedItemIds: "true",
+        os: "mac",
+        secUid,
+        tz_name: Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC",
+        user_is_login: "true",
+        webcast_language: "en",
+    });
+    const response = await fetchWithRetry(`https://www.tiktok.com/api/user/collection_list/?${params.toString()}`, {
+        method: "GET",
+        headers: {
+            "user-agent": WEB_USER_AGENT,
+            accept: "application/json, text/plain, */*",
+            "accept-language": "en-US,en;q=0.9",
+            cookie,
+            referer: "https://www.tiktok.com/",
+        },
+    }, { maxRetries });
+    if (!response.ok) {
+        return {
+            status: "error",
+            message: `Source returned HTTP ${response.status} ${response.statusText}`,
+        };
+    }
+    return response.json();
+}
+export function extractCollectionListEntries(response) {
+    const root = asRecord(response);
+    const result = asRecord(root?.result);
+    const candidates = [root?.collectionList, root?.collection_list, result?.collectionList, result?.collection_list];
+    for (const candidate of candidates) {
+        if (Array.isArray(candidate)) {
+            return candidate.flatMap((item) => (asRecord(item) ? [asRecord(item)] : []));
+        }
+    }
+    return [];
+}
+export function normalizeBookmarkFolder(entry, username) {
+    const id = stringValue(entry.collectionId) ?? stringValue(entry.id) ?? stringValue(entry.collection_id);
+    if (!id)
+        return undefined;
+    const name = stringValue(entry.name) ?? stringValue(entry.collectionName) ?? "Untitled";
+    const itemCount = numberValue(entry.total) ??
+        numberValue(entry.itemCount) ??
+        numberValue(entry.itemTotal) ??
+        numberValue(entry.videoCount);
+    const cover = firstString(entry.cover) ?? stringValue(entry.cover) ?? firstString(entry.coverUrl) ?? stringValue(entry.coverUrl);
+    const kind = FAVORITES_NAMES.has(name.trim().toLowerCase()) ? "favorites" : "collection";
+    const url = username ? `https://www.tiktok.com/@${username}/collection/${slugifyCollection(name, id)}` : undefined;
+    return { id, name, url, itemCount, cover, kind };
+}
+function slugifyCollection(name, id) {
+    const slug = name
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "");
+    return slug ? `${slug}-${id}` : id;
+}
+function readCursor(response) {
+    const root = asRecord(response);
+    const result = asRecord(root?.result);
+    return stringValue(result?.cursor) ?? stringValue(root?.cursor);
+}
 export function extractItemsFromSuccessfulResponse(response, source) {
     assertSuccessfulResponse(response, source);
     return extractItems(response).filter(isLikelyVideoItem);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tokwise",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Local-first CLI for syncing, downloading, transcribing, searching, and analyzing saved short-form videos.",
   "type": "module",
   "author": "Sebastian Crossa",
@@ -45,6 +45,7 @@
     "whisper"
   ],
   "dependencies": {
+    "@clack/prompts": "^1.6.0",
     "@tobyg74/tiktok-api-dl": "^1.3.7",
     "commander": "^15.0.0",
     "picocolors": "^1.1.1"