tokwise 0.1.0

package/dist/progress.js

@@ -0,0 +1,56 @@
+import { c } from "./render.js";
+const FRAMES = ["\u280b", "\u2819", "\u2839", "\u2838", "\u283c", "\u2834", "\u2826", "\u2827", "\u2807", "\u280f"];
+const CLEAR_LINE = "\r\u001b[2K";
+const FRAME_INTERVAL_MS = 80;
+export function createProgress(options) {
+    const { total, label } = options;
+    const stream = process.stderr;
+    const tty = Boolean(stream.isTTY);
+    let count = 0;
+    let frame = 0;
+    let current = "";
+    let failed = false;
+    let timer;
+    function draw() {
+        const spinner = c.accent(FRAMES[frame % FRAMES.length] ?? "");
+        const counter = c.muted(`${count}/${total}`);
+        const status = failed ? c.danger(current) : current;
+        stream.write(`${CLEAR_LINE}${spinner} ${label} ${counter} ${status}`.trimEnd());
+    }
+    if (tty && total > 0) {
+        draw();
+        timer = setInterval(() => {
+            frame += 1;
+            draw();
+        }, FRAME_INTERVAL_MS);
+        timer.unref();
+    }
+    function update(message, isFailure) {
+        count += 1;
+        current = message;
+        failed = isFailure;
+        if (!tty) {
+            stream.write(`${label} ${count}/${total}: ${message}\n`);
+            return;
+        }
+        draw();
+    }
+    return {
+        tick(message = "") {
+            update(message, false);
+        },
+        fail(message = "") {
+            update(message, true);
+        },
+        done(summary) {
+            if (timer) {
+                clearInterval(timer);
+                timer = undefined;
+            }
+            if (tty)
+                stream.write(CLEAR_LINE);
+            if (summary)
+                stream.write(`${summary}\n`);
+        },
+    };
+}

package/dist/render.js

@@ -0,0 +1,114 @@
+import pc from "picocolors";
+const raw = pc.createColors(true);
+let colorEnabled = pc.isColorSupported;
+export function setColorEnabled(on) {
+    colorEnabled = on;
+}
+export function isColorEnabled() {
+    return colorEnabled;
+}
+function style(fn) {
+    return (text) => (colorEnabled ? fn(text) : text);
+}
+export const c = {
+    heading: style((text) => raw.bold(raw.cyan(text))),
+    label: style((text) => text),
+    value: style((text) => raw.bold(text)),
+    muted: style((text) => raw.dim(text)),
+    accent: style((text) => raw.cyan(text)),
+    success: style((text) => raw.green(text)),
+    warn: style((text) => raw.yellow(text)),
+    danger: style((text) => raw.red(text)),
+};
+const PALETTE = [raw.cyan, raw.green, raw.yellow, raw.magenta, raw.blue, raw.red];
+function paletteColor(index) {
+    const fn = PALETTE[index % PALETTE.length] ?? raw.cyan;
+    return style(fn);
+}
+const ANSI_PATTERN = /\u001b\[[0-9;]*m/g;
+export function stripAnsi(text) {
+    return text.replace(ANSI_PATTERN, "");
+}
+export function visibleWidth(text) {
+    return stripAnsi(text).length;
+}
+export function padEndVisible(text, width) {
+    const pad = width - visibleWidth(text);
+    return pad > 0 ? text + " ".repeat(pad) : text;
+}
+export function truncate(text, max) {
+    if (text.length <= max)
+        return text;
+    if (max <= 1)
+        return text.slice(0, Math.max(0, max));
+    return `${text.slice(0, max - 1)}\u2026`;
+}
+export function rule(width = 60) {
+    return c.muted("\u2500".repeat(width));
+}
+export function box(title, lines) {
+    const titleWidth = visibleWidth(title);
+    const bodyWidth = Math.max(titleWidth + 1, 0, ...lines.map(visibleWidth));
+    const interior = bodyWidth + 2;
+    const dashCount = Math.max(0, interior - (titleWidth + 3));
+    const top = c.muted("\u256d\u2500 ") + c.heading(title) + c.muted(` ${"\u2500".repeat(dashCount)}\u256e`);
+    const bottom = c.muted(`\u2570${"\u2500".repeat(interior)}\u256f`);
+    const body = lines.map((line) => `${c.muted("\u2502")} ${padEndVisible(line, bodyWidth)} ${c.muted("\u2502")}`);
+    return [top, ...body, bottom].join("\n");
+}
+const EIGHTHS = ["", "\u258f", "\u258e", "\u258d", "\u258c", "\u258b", "\u258a", "\u2589"];
+const FULL_BLOCK = "\u2588";
+function bar(value, max, width) {
+    if (max <= 0 || value <= 0)
+        return "";
+    const eighths = Math.round((value / max) * width * 8);
+    const full = Math.floor(eighths / 8);
+    const remainder = eighths % 8;
+    let result = FULL_BLOCK.repeat(full);
+    if (remainder > 0)
+        result += EIGHTHS[remainder];
+    if (result === "")
+        result = EIGHTHS[1] ?? FULL_BLOCK;
+    return result;
+}
+export function barChart(entries, options = {}) {
+    const width = options.width ?? 24;
+    const limit = options.limit ?? entries.length;
+    const sorted = [...entries].sort((a, b) => b[1] - a[1]).slice(0, limit);
+    if (sorted.length === 0)
+        return c.muted("(none)");
+    const total = options.total ?? entries.reduce((sum, [, count]) => sum + count, 0);
+    const max = Math.max(1, ...sorted.map(([, count]) => count));
+    const labelWidth = Math.max(...sorted.map(([label]) => label.length));
+    const countWidth = Math.max(...sorted.map(([, count]) => String(count).length));
+    return sorted
+        .map(([label, count], index) => {
+        const barText = bar(count, max, width);
+        const coloredBar = options.color === false ? barText : paletteColor(index)(barText);
+        const percent = total > 0 ? `${Math.round((count / total) * 100)}%` : "";
+        return [
+            "  ",
+            c.label(padEndVisible(label, labelWidth)),
+            "  ",
+            padEndVisible(coloredBar, width),
+            "  ",
+            c.value(String(count).padStart(countWidth)),
+            "  ",
+            c.muted(percent),
+        ].join("");
+    })
+        .join("\n");
+}
+export function kvList(pairs) {
+    if (pairs.length === 0)
+        return "";
+    const keyWidth = Math.max(...pairs.map(([key]) => visibleWidth(key)));
+    return pairs.map(([key, value]) => `${c.label(padEndVisible(key, keyWidth))}  ${value}`).join("\n");
+}
+export function table(headers, rows) {
+    const widths = headers.map((header, index) => Math.max(visibleWidth(header), ...rows.map((row) => visibleWidth(row[index] ?? ""))));
+    const renderRow = (cells) => cells.map((cell, index) => padEndVisible(cell, widths[index] ?? 0)).join("  ");
+    const head = c.heading(renderRow(headers));
+    const body = rows.map((row) => renderRow(row));
+    return [head, ...body].join("\n");
+}

package/dist/search.js

@@ -0,0 +1,226 @@
+import fs from "node:fs/promises";
+import { searchIndexPath } from "./paths.js";
+import { readJsonFile, writeJsonFile } from "./jsonl.js";
+import { c } from "./render.js";
+const STOP_WORDS = new Set([
+    "a",
+    "an",
+    "and",
+    "are",
+    "as",
+    "at",
+    "be",
+    "but",
+    "by",
+    "for",
+    "from",
+    "how",
+    "i",
+    "in",
+    "into",
+    "is",
+    "it",
+    "of",
+    "on",
+    "or",
+    "that",
+    "the",
+    "this",
+    "to",
+    "was",
+    "with",
+    "you",
+    "your",
+]);
+export function tokenize(input) {
+    if (!input)
+        return [];
+    return input
+        .toLowerCase()
+        .normalize("NFKD")
+        .replace(/['']/g, "")
+        .split(/[^a-z0-9]+/g)
+        .filter((token) => token.length >= 2 && !STOP_WORDS.has(token));
+}
+export function searchableText(video) {
+    return [
+        video.description,
+        video.transcript?.text,
+        video.classification?.summary,
+        video.classification?.category,
+        video.classification?.domain,
+        ...(video.classification?.topics ?? []),
+        ...video.hashtags,
+        video.author?.username,
+        video.author?.displayName,
+        video.music?.title,
+        video.music?.author,
+    ]
+        .filter(Boolean)
+        .join("\n");
+}
+export function buildSearchIndex(videos) {
+    const docs = videos.map(toSearchDocument);
+    const termDocFreq = {};
+    for (const doc of docs) {
+        for (const term of Object.keys(doc.terms)) {
+            termDocFreq[term] = (termDocFreq[term] ?? 0) + 1;
+        }
+    }
+    const totalLength = docs.reduce((sum, doc) => sum + doc.length, 0);
+    return {
+        version: 1,
+        builtAt: new Date().toISOString(),
+        recordCount: docs.length,
+        avgDocLength: docs.length === 0 ? 0 : totalLength / docs.length,
+        termDocFreq,
+        docs,
+    };
+}
+function toSearchDocument(video) {
+    const weightedTerms = {};
+    addTerms(weightedTerms, tokenize(video.description), 3);
+    addTerms(weightedTerms, tokenize(video.transcript?.text), 2);
+    addTerms(weightedTerms, tokenize(video.classification?.summary), 3);
+    addTerms(weightedTerms, tokenize(video.classification?.category), 4);
+    addTerms(weightedTerms, tokenize(video.classification?.domain), 4);
+    addTerms(weightedTerms, tokenize((video.classification?.topics ?? []).join(" ")), 4);
+    addTerms(weightedTerms, tokenize(video.hashtags.join(" ")), 4);
+    addTerms(weightedTerms, tokenize([video.author?.username, video.author?.displayName].filter(Boolean).join(" ")), 2);
+    addTerms(weightedTerms, tokenize([video.music?.title, video.music?.author].filter(Boolean).join(" ")), 1);
+    const length = Object.values(weightedTerms).reduce((sum, count) => sum + count, 0);
+    return {
+        id: video.id,
+        length,
+        terms: weightedTerms,
+        title: video.description?.split(/\r?\n/)[0]?.slice(0, 100) ?? video.id,
+        preview: makePreview(video),
+    };
+}
+function addTerms(target, terms, weight) {
+    for (const term of terms) {
+        target[term] = (target[term] ?? 0) + weight;
+    }
+}
+function makePreview(video) {
+    const text = video.transcript?.text || video.description || "";
+    return text.replace(/\s+/g, " ").trim().slice(0, 220);
+}
+export async function saveSearchIndex(videos) {
+    const index = buildSearchIndex(videos);
+    await writeJsonFile(searchIndexPath(), index);
+    return index;
+}
+export async function loadSearchIndex() {
+    try {
+        await fs.access(searchIndexPath());
+    }
+    catch {
+        return undefined;
+    }
+    return readJsonFile(searchIndexPath(), undefined);
+}
+export function filterVideos(videos, filters) {
+    const author = filters.author?.replace(/^@/, "").toLowerCase();
+    const category = filters.category?.toLowerCase();
+    const domain = filters.domain?.toLowerCase();
+    const collection = filters.collection?.toLowerCase();
+    const after = filters.after ? new Date(filters.after) : undefined;
+    const before = filters.before ? new Date(filters.before) : undefined;
+    return videos.filter((video) => {
+        if (author && !video.author?.username?.toLowerCase().includes(author))
+            return false;
+        if (category && video.classification?.category?.toLowerCase() !== category)
+            return false;
+        if (domain && video.classification?.domain?.toLowerCase() !== domain)
+            return false;
+        if (collection) {
+            const haystack = [video.collection?.id, video.collection?.name, video.collection?.url].filter(Boolean).join(" ");
+            if (!haystack.toLowerCase().includes(collection))
+                return false;
+        }
+        if (filters.source && video.source !== filters.source)
+            return false;
+        if (filters.hasTranscript != null && Boolean(video.transcript?.text) !== filters.hasTranscript)
+            return false;
+        const date = video.createdAt ? new Date(video.createdAt) : undefined;
+        if (after && date && date < after)
+            return false;
+        if (before && date && date > before)
+            return false;
+        return true;
+    });
+}
+export function searchWithIndex(videos, index, filters) {
+    const filtered = filterVideos(videos, filters);
+    const allowedIds = new Set(filtered.map((video) => video.id));
+    const byId = new Map(videos.map((video) => [video.id, video]));
+    const query = filters.query?.trim() ?? "";
+    const limit = filters.limit ?? 20;
+    const offset = filters.offset ?? 0;
+    if (!query) {
+        return filtered.slice(offset, offset + limit).map((video) => ({ video, score: 0, highlights: highlights(video, []) }));
+    }
+    const queryTerms = tokenize(query);
+    const phrase = query.toLowerCase();
+    const scored = [];
+    for (const doc of index.docs) {
+        if (!allowedIds.has(doc.id))
+            continue;
+        const video = byId.get(doc.id);
+        if (!video)
+            continue;
+        let score = 0;
+        for (const term of queryTerms) {
+            score += bm25(term, doc, index);
+        }
+        if (searchableText(video).toLowerCase().includes(phrase))
+            score += 1.5;
+        if (score > 0)
+            scored.push({ video, score, highlights: highlights(video, queryTerms) });
+    }
+    return scored.sort((a, b) => b.score - a.score).slice(offset, offset + limit);
+}
+function bm25(term, doc, index) {
+    const tf = doc.terms[term] ?? 0;
+    if (tf <= 0)
+        return 0;
+    const n = index.recordCount;
+    const df = index.termDocFreq[term] ?? 0;
+    const idf = Math.log(1 + (n - df + 0.5) / (df + 0.5));
+    const k1 = 1.4;
+    const b = 0.75;
+    const avg = index.avgDocLength || 1;
+    return idf * ((tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (doc.length / avg))));
+}
+function highlights(video, terms) {
+    const parts = [video.description, video.transcript?.text].filter(Boolean);
+    if (terms.length === 0)
+        return parts.slice(0, 1).map((part) => part.replace(/\s+/g, " ").slice(0, 220));
+    const normalizedTerms = new Set(terms);
+    for (const part of parts) {
+        const sentences = part.split(/(?<=[.!?])\s+|\n+/);
+        const matched = sentences.find((sentence) => tokenize(sentence).some((term) => normalizedTerms.has(term)));
+        if (matched)
+            return [matched.replace(/\s+/g, " ").slice(0, 240)];
+    }
+    return parts.slice(0, 1).map((part) => part.replace(/\s+/g, " ").slice(0, 220));
+}
+export function formatSearchResults(results, options) {
+    if (options?.json)
+        return JSON.stringify(results, null, 2);
+    if (results.length === 0)
+        return c.muted("No matches.");
+    return results
+        .map((result, idx) => {
+        const video = result.video;
+        const author = video.author?.username ? c.accent(`@${video.author.username}`) : c.muted("unknown");
+        const category = video.classification?.category ? ` ${c.warn(`[${video.classification.category}]`)}` : "";
+        const score = result.score > 0 ? ` ${c.muted(`score ${result.score.toFixed(2)}`)}` : "";
+        const line = `${c.muted(`${idx + 1}.`)} ${c.value(video.id)} ${author}${category}${score}`;
+        const desc = video.description ? `   ${video.description.replace(/\s+/g, " ").slice(0, 160)}` : "";
+        const hit = result.highlights[0] ? `   ${c.muted(">")} ${c.success(result.highlights[0])}` : "";
+        return [line, desc, hit, `   ${c.muted(video.canonicalUrl ?? video.url)}`].filter(Boolean).join("\n");
+    })
+        .join("\n\n");
+}

package/dist/skill.js

@@ -0,0 +1,57 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+export function skillContent() {
+    return [
+        "---",
+        "name: tokwise",
+        "description: Search and use the user's local Tokwise archive of saved short-form video transcripts.",
+        "---",
+        "",
+        "# Tokwise",
+        "",
+        "Use this skill when the user asks about saved short-form videos, life advice clips, transcript patterns, or similarities across saved clips.",
+        "",
+        "## Commands",
+        "",
+        "- `tokwise status` shows whether the archive exists and how many transcripts are available.",
+        "- `tokwise search \"query\" --limit 8` searches descriptions, hashtags, summaries, and transcripts.",
+        "- `tokwise show <id>` prints full metadata and transcript for one video.",
+        "- `tokwise similar <id>` finds related videos by transcript and metadata overlap.",
+        "- `tokwise ask \"question\"` returns top local evidence; add `--engine ollama --model <model>` only when the user wants local synthesis.",
+        "- `tokwise md` and `tokwise wiki` export Markdown pages under the local library.",
+        "- `tw` is the short alias for `tokwise`.",
+        "",
+        "## Grounding",
+        "",
+        "Cite video ids or Markdown page paths when drawing conclusions. Treat transcripts as user-owned local context and do not assume videos are public.",
+        "",
+    ].join("\n");
+}
+export async function installSkill(target = "all") {
+    const destinations = [];
+    if (target === "codex" || target === "all") {
+        destinations.push(path.join(os.homedir(), ".codex", "skills", "tokwise", "SKILL.md"));
+    }
+    if (target === "claude" || target === "all") {
+        destinations.push(path.join(os.homedir(), ".claude", "skills", "tokwise", "SKILL.md"));
+    }
+    for (const destination of destinations) {
+        await fs.mkdir(path.dirname(destination), { recursive: true });
+        await fs.writeFile(destination, skillContent(), "utf8");
+    }
+    return destinations;
+}
+export async function uninstallSkill(target = "all") {
+    const destinations = [];
+    if (target === "codex" || target === "all") {
+        destinations.push(path.join(os.homedir(), ".codex", "skills", "tokwise"));
+    }
+    if (target === "claude" || target === "all") {
+        destinations.push(path.join(os.homedir(), ".claude", "skills", "tokwise"));
+    }
+    for (const destination of destinations) {
+        await fs.rm(destination, { recursive: true, force: true });
+    }
+    return destinations;
+}

package/dist/store.js

@@ -0,0 +1,158 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { authPath, ensureDataDirs, preferencesPath, videosJsonlPath } from "./paths.js";
+import { readJsonFile, readJsonl, writeJsonFile, writeJsonl } from "./jsonl.js";
+export function stableHash(input) {
+    return crypto.createHash("sha1").update(input).digest("hex").slice(0, 16);
+}
+export function sanitizeFilePart(input) {
+    return input
+        .replace(/[^a-zA-Z0-9._-]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 120) || "untitled";
+}
+export async function loadVideos() {
+    ensureDataDirs();
+    return readJsonl(videosJsonlPath());
+}
+export async function saveVideos(videos) {
+    ensureDataDirs();
+    const sorted = [...videos].sort((a, b) => {
+        const aTime = a.savedAt ?? a.createdAt ?? a.syncedAt;
+        const bTime = b.savedAt ?? b.createdAt ?? b.syncedAt;
+        return bTime.localeCompare(aTime);
+    });
+    await writeJsonl(videosJsonlPath(), sorted);
+}
+export function mergeVideo(existing, incoming) {
+    if (!existing)
+        return incoming;
+    const classification = mergeClassification(existing.classification, incoming.classification);
+    return {
+        ...existing,
+        ...incoming,
+        savedAt: incoming.savedAt ?? existing.savedAt,
+        syncedAt: incoming.syncedAt,
+        author: { ...existing.author, ...incoming.author },
+        stats: { ...existing.stats, ...incoming.stats },
+        media: {
+            ...existing.media,
+            ...incoming.media,
+            videoPath: existing.media?.videoPath ?? incoming.media?.videoPath,
+            audioPath: existing.media?.audioPath ?? incoming.media?.audioPath,
+            infoJsonPath: existing.media?.infoJsonPath ?? incoming.media?.infoJsonPath,
+            downloadedAt: existing.media?.downloadedAt ?? incoming.media?.downloadedAt,
+        },
+        transcript: existing.transcript ?? incoming.transcript,
+        classification,
+        hashtags: uniqueStrings([...existing.hashtags, ...incoming.hashtags]),
+        raw: incoming.raw ?? existing.raw,
+    };
+}
+function mergeClassification(existing, incoming) {
+    if (!existing)
+        return incoming;
+    if (!incoming)
+        return existing;
+    return {
+        ...existing,
+        ...incoming,
+        topics: uniqueStrings([...(existing.topics ?? []), ...(incoming.topics ?? [])]),
+    };
+}
+export function uniqueStrings(values) {
+    return [...new Set(values.map((value) => value?.trim()).filter((value) => Boolean(value)))];
+}
+export async function mergeVideos(incoming, options) {
+    const existing = options?.rebuild ? [] : await loadVideos();
+    const existingById = new Map(existing.map((video) => [video.id, video]));
+    const before = new Map(existing.map((video) => [video.id, JSON.stringify(video)]));
+    for (const video of incoming) {
+        existingById.set(video.id, mergeVideo(existingById.get(video.id), video));
+    }
+    const merged = [...existingById.values()];
+    await saveVideos(merged);
+    let added = 0;
+    let updated = 0;
+    let unchanged = 0;
+    for (const video of incoming) {
+        const old = before.get(video.id);
+        if (old == null) {
+            added += 1;
+        }
+        else if (old === JSON.stringify(existingById.get(video.id))) {
+            unchanged += 1;
+        }
+        else {
+            updated += 1;
+        }
+    }
+    return { added, updated, unchanged, total: merged.length, ids: incoming.map((video) => video.id) };
+}
+export function findVideo(videos, query) {
+    const normalized = query.trim();
+    return (videos.find((video) => video.id === normalized) ??
+        videos.find((video) => video.id.startsWith(normalized)) ??
+        videos.find((video) => video.url === normalized || video.canonicalUrl === normalized));
+}
+export async function updateVideosById(updates) {
+    const videos = await loadVideos();
+    const next = videos.map((video) => {
+        const update = updates.get(video.id);
+        if (!update)
+            return video;
+        if (typeof update === "function")
+            return update(video);
+        return mergeVideo(video, { ...video, ...update, syncedAt: update.syncedAt ?? video.syncedAt });
+    });
+    await saveVideos(next);
+    return next;
+}
+export async function readTextInput(filePath) {
+    if (filePath === "-") {
+        return new Promise((resolve, reject) => {
+            let body = "";
+            process.stdin.setEncoding("utf8");
+            process.stdin.on("data", (chunk) => {
+                body += chunk;
+            });
+            process.stdin.on("end", () => resolve(body));
+            process.stdin.on("error", reject);
+        });
+    }
+    return fs.readFile(filePath, "utf8");
+}
+export async function loadPreferences() {
+    return readJsonFile(preferencesPath(), {});
+}
+export async function savePreferences(preferences) {
+    await writeJsonFile(preferencesPath(), preferences, 0o600);
+}
+export async function loadAuth() {
+    return readJsonFile(authPath(), {});
+}
+export async function saveAuth(auth) {
+    await writeJsonFile(authPath(), auth, 0o600);
+}
+export async function clearAuth() {
+    try {
+        await fs.rm(authPath());
+    }
+    catch (error) {
+        if (error.code !== "ENOENT")
+            throw error;
+    }
+}
+export async function loadCookie(options) {
+    if (options?.cookie)
+        return options.cookie;
+    if (options?.cookieFile)
+        return (await fs.readFile(options.cookieFile, "utf8")).trim();
+    return (await loadAuth()).cookie;
+}
+export function resolveMaybeRelative(filePath, cwd = process.cwd()) {
+    if (path.isAbsolute(filePath))
+        return filePath;
+    return path.resolve(cwd, filePath);
+}