tokwise 0.1.2 → 0.1.4

package/dist/ask.js

@@ -39,7 +39,7 @@ async function answerWithOllama(question, results, options) {
         .join("\n\n");
     const prompt = [
         "Answer the user's question using only the saved clip evidence below.",
-        "Cite clips by their readable reference (e.g. @author \u00b7 Mon YYYY \u2014 \"title\") when making claims. If evidence is thin, say so.",
+        "Cite clips by their readable reference (e.g. @author (Mon YYYY)) when making claims; include the clip's Source URL when a claim needs to be traceable. If evidence is thin, say so.",
         "",
         `Question: ${question}`,
         "",
@@ -49,10 +49,56 @@ async function answerWithOllama(question, results, options) {
     const response = await fetch(`${baseUrl.replace(/\/+$/, "")}/api/generate`, {
         method: "POST",
         headers: { "content-type": "application/json" },
-        body: JSON.stringify({ model, prompt, stream: false }),
+        body: JSON.stringify({ model, prompt, stream: true }),
     });
     if (!response.ok)
         throw new Error(`Ollama ask failed: ${response.status} ${response.statusText}`);
-    const body = (await response.json());
-    return body.response?.trim() || "Ollama returned an empty answer.";
+    if (!response.body)
+        throw new Error("Ollama ask failed: empty response body");
+    let full = "";
+    let firstToken = false;
+    for await (const chunk of iterateNdjson(response.body)) {
+        const piece = chunk.response ?? "";
+        if (!piece) {
+            if (chunk.done)
+                break;
+            continue;
+        }
+        if (!firstToken) {
+            firstToken = true;
+            options.onFirstToken?.();
+        }
+        full += piece;
+        options.onToken?.(piece);
+        if (chunk.done)
+            break;
+    }
+    return full.trim() || "Ollama returned an empty answer.";
+}
+async function* iterateNdjson(body) {
+    const reader = body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    try {
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            buffer += decoder.decode(value, { stream: true });
+            const lines = buffer.split("\n");
+            buffer = lines.pop() ?? "";
+            for (const line of lines) {
+                const trimmed = line.trim();
+                if (!trimmed)
+                    continue;
+                yield JSON.parse(trimmed);
+            }
+        }
+        const trailing = buffer.trim();
+        if (trailing)
+            yield JSON.parse(trailing);
+    }
+    finally {
+        reader.releaseLock();
+    }
 }

package/dist/cli.js

@@ -22,7 +22,7 @@ import { createCommand, createLibraryPage, deleteLibraryPage, listCommands, sear
 import { installSkill, skillContent, uninstallSkill } from "./skill.js";
 import { barChart, box, c, kvList, setColorEnabled, table, truncate } from "./render.js";
 import { formatReference } from "./reference.js";
-import { createProgress } from "./progress.js";
+import { createProgress, createSpinner } from "./progress.js";
 const require = createRequire(import.meta.url);
 function version() {
     try {
@@ -482,12 +482,30 @@ export function buildCli() {
         const prefs = await loadPreferences();
         const { videos, index } = await requireIndex();
         const results = searchWithIndex(videos, index, { query: question, limit: Number(options.limit) });
-        const answer = await answerQuestion(question, results, {
-            engine: options.engine ?? prefs.askEngine ?? "extractive",
-            model: options.model ?? prefs.model,
-            ollamaBaseUrl: options.ollamaUrl ?? prefs.ollamaBaseUrl,
-        });
-        console.log(answer);
+        const engine = options.engine ?? prefs.askEngine ?? "extractive";
+        const spinner = engine === "ollama" ? createSpinner() : undefined;
+        spinner?.start();
+        let streamed = false;
+        let answer;
+        try {
+            answer = await answerQuestion(question, results, {
+                engine,
+                model: options.model ?? prefs.model,
+                ollamaBaseUrl: options.ollamaUrl ?? prefs.ollamaBaseUrl,
+                onFirstToken: () => spinner?.stop(),
+                onToken: (chunk) => {
+                    streamed = true;
+                    process.stdout.write(chunk);
+                },
+            });
+        }
+        finally {
+            spinner?.stop();
+        }
+        if (streamed)
+            process.stdout.write("\n");
+        else
+            console.log(answer);
         if (options.save) {
             const file = path.join(libraryDir(), "answers", `${Date.now()}-${slug(question)}.md`);
             await fs.mkdir(path.dirname(file), { recursive: true });

package/dist/progress.js

@@ -54,3 +54,72 @@ export function createProgress(options) {
         },
     };
 }
+const DEFAULT_SPINNER_MESSAGES = [
+    "Thinking",
+    "Pondering",
+    "Rummaging through your bookmarks",
+    "Connecting the dots",
+    "Cogitating",
+    "Synthesizing",
+    "Consulting the archive",
+    "Musing",
+    "Distilling insights",
+];
+const MESSAGE_INTERVAL_MS = 2500;
+export function createSpinner(options = {}) {
+    const messages = options.messages ?? DEFAULT_SPINNER_MESSAGES;
+    const messageIntervalMs = options.messageIntervalMs ?? MESSAGE_INTERVAL_MS;
+    const stream = process.stderr;
+    const tty = Boolean(stream.isTTY);
+    let running = false;
+    let frame = 0;
+    let messageIndex = 0;
+    let startedAt = 0;
+    let frameTimer;
+    let messageTimer;
+    function draw() {
+        const elapsedSec = Math.floor((Date.now() - startedAt) / 1000);
+        const spinner = c.accent(FRAMES[frame % FRAMES.length] ?? "");
+        const message = messages[messageIndex % messages.length] ?? "Thinking";
+        stream.write(`${CLEAR_LINE}${spinner} ${message}\u2026 ${c.muted(`(${elapsedSec}s)`)}`);
+    }
+    function clearTimers() {
+        if (frameTimer) {
+            clearInterval(frameTimer);
+            frameTimer = undefined;
+        }
+        if (messageTimer) {
+            clearInterval(messageTimer);
+            messageTimer = undefined;
+        }
+    }
+    return {
+        start() {
+            if (running || !tty)
+                return;
+            running = true;
+            startedAt = Date.now();
+            frame = 0;
+            messageIndex = 0;
+            draw();
+            frameTimer = setInterval(() => {
+                frame += 1;
+                draw();
+            }, FRAME_INTERVAL_MS);
+            frameTimer.unref();
+            messageTimer = setInterval(() => {
+                messageIndex += 1;
+                draw();
+            }, messageIntervalMs);
+            messageTimer.unref();
+        },
+        stop() {
+            if (!running)
+                return;
+            running = false;
+            clearTimers();
+            if (tty)
+                stream.write(CLEAR_LINE);
+        },
+    };
+}

package/dist/reference.js

@@ -23,18 +23,14 @@ export function videoReference(video) {
         ? `@${video.author.username}`
         : video.author?.displayName || "unknown";
     const date = referenceDate(video);
-    const title = videoTitle(video);
-    const head = [author, date].filter(Boolean).join(" \u00b7 ");
-    return `${head} \u2014 "${title}"  #${shortId(video)}`;
+    return date ? `${author} (${date})` : author;
 }
 export function formatReference(video) {
     const author = video.author?.username
         ? c.accent(`@${video.author.username}`)
         : c.muted(video.author?.displayName || "unknown");
     const date = referenceDate(video);
-    const datePart = date ? ` ${c.muted("\u00b7")} ${c.muted(date)}` : "";
-    const title = c.value(`"${videoTitle(video)}"`);
-    return `${author}${datePart} ${c.muted("\u2014")} ${title}  ${c.muted(`#${shortId(video)}`)}`;
+    return date ? `${author} ${c.muted(`(${date})`)}` : author;
 }
 function referenceDate(video) {
     const iso = video.createdAt ?? video.savedAt;

package/dist/skill.js

@@ -24,7 +24,7 @@ export function skillContent() {
         "",
         "## Grounding",
         "",
-        "Cite clips by their readable reference (`@author \u00b7 Mon YYYY \u2014 \"title\"`, optionally with the trailing short id like `#49952278`) or by Markdown page path when drawing conclusions. Run `tokwise show <short-id-or-url>` to pull a clip back up. Treat transcripts as user-owned local context and do not assume videos are public.",
+        "Cite clips by their readable reference (`@author (Mon YYYY)`) and the clip URL, or by Markdown page path, when drawing conclusions. Run `tokwise show <id-or-url>` to pull a clip back up. Treat transcripts as user-owned local context and do not assume videos are public.",
         "",
     ].join("\n");
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tokwise",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Local-first CLI for syncing, downloading, transcribing, searching, and analyzing saved short-form videos.",
   "type": "module",
   "author": "Sebastian Crossa",