vidistill 0.6.2 → 0.6.3

package/dist/index.js

@@ -242,6 +242,23 @@ var LANGUAGE_NAMES = {
   ar: "Arabic",
   hi: "Hindi"
 };
+var SYSTEM_INSTRUCTION_DEDUP = `
+You are a transcript deduplication reviewer. You will receive a list of numbered transcript entries. Your task is to identify entries that are SEMANTIC DUPLICATES of an earlier entry.
+
+A duplicate means:
+- The entry says essentially the same thing as an earlier entry (same meaning, possibly different wording)
+- The entry is a subset of an earlier entry (the earlier one already covers this content)
+- The entry repeats the same quote or statement that already appeared earlier
+
+NOT a duplicate:
+- Entries where the speaker genuinely repeats themselves for emphasis (common in lectures/presentations)
+- Similar topics discussed at different points with different details
+- Call-and-response patterns (e.g. Q&A)
+
+Return ONLY the indices of entries to REMOVE (the later duplicate, not the original).
+If no duplicates are found, return an empty array.
+Be conservative \u2014 only flag clear duplicates. When in doubt, keep the entry.
+`;
 function withLanguage(prompt, lang) {
   if (!lang || lang === "en") return prompt;
   const languageName = LANGUAGE_NAMES[lang] ?? lang;
@@ -1545,6 +1562,17 @@ var SCHEMA_SYNTHESIS = {
   },
   required: ["overview", "key_decisions", "key_concepts", "action_items", "questions_raised", "suggestions", "topics", "files_to_generate", "prerequisites"]
 };
+var SCHEMA_DEDUP_REVIEW = {
+  type: Type.OBJECT,
+  properties: {
+    duplicate_indices: {
+      type: Type.ARRAY,
+      items: { type: Type.INTEGER },
+      description: "Zero-based indices of transcript entries that are semantic duplicates of an earlier entry and should be removed"
+    }
+  },
+  required: ["duplicate_indices"]
+};
 
 // src/lib/utils.ts
 import { readFile } from "fs/promises";
@@ -3286,6 +3314,55 @@ async function runPipeline(config) {
     const msg = e instanceof Error ? e.message : String(e);
     log5.warn(`speaker reconciliation failed, continuing with original labels: ${msg}`);
   }
+  const allEntries = [];
+  for (let segIdx = 0; segIdx < results.length; segIdx++) {
+    const p1 = results[segIdx].pass1;
+    if (p1 == null) continue;
+    for (let entryIdx = 0; entryIdx < p1.transcript_entries.length; entryIdx++) {
+      allEntries.push({ segIdx, entryIdx, entry: p1.transcript_entries[entryIdx] });
+    }
+  }
+  if (allEntries.length > 20) {
+    try {
+      const numbered = allEntries.map(
+        (e, i) => `[${i}] ${e.entry.timestamp} ${e.entry.speaker}: ${e.entry.text}`
+      ).join("\n");
+      const dedupResult = await rateLimiter.execute(
+        () => client.generate({
+          model: MODELS.flash,
+          contents: [{ role: "user", parts: [{ text: numbered }] }],
+          config: {
+            systemInstruction: SYSTEM_INSTRUCTION_DEDUP,
+            responseMimeType: "application/json",
+            responseSchema: SCHEMA_DEDUP_REVIEW,
+            temperature: 0
+          }
+        }),
+        { onWait }
+      );
+      const parsed = dedupResult;
+      if (parsed != null && Array.isArray(parsed.duplicate_indices)) {
+        const indices = parsed.duplicate_indices;
+        const toRemove = new Set(
+          indices.filter((v) => typeof v === "number" && v >= 0 && v < allEntries.length)
+        );
+        if (toRemove.size > 0) {
+          const segRemovals = /* @__PURE__ */ new Map();
+          for (const globalIdx of toRemove) {
+            const { segIdx, entryIdx } = allEntries[globalIdx];
+            if (!segRemovals.has(segIdx)) segRemovals.set(segIdx, /* @__PURE__ */ new Set());
+            segRemovals.get(segIdx).add(entryIdx);
+          }
+          for (const [segIdx, entryIndices] of segRemovals) {
+            const p1 = results[segIdx].pass1;
+            if (p1 == null) continue;
+            p1.transcript_entries = p1.transcript_entries.filter((_, i) => !entryIndices.has(i));
+          }
+        }
+      }
+    } catch {
+    }
+  }
   let peopleExtraction = null;
   if (strategy.passes.includes("people")) {
     onProgress?.({ phase: "pass3b", segment: 0, totalSegments: 1, status: "running", totalSteps });
@@ -3580,6 +3657,15 @@ function writeTranscript(params) {
     sections.push("_No transcript data available._");
     return sections.join("\n");
   }
+  for (let i = 1; i < segmentsWithPass1.length; i++) {
+    const prev = segmentsWithPass1[i - 1].pass1;
+    const curr = segmentsWithPass1[i].pass1;
+    if (prev == null || curr == null) continue;
+    const tail = prev.transcript_entries.slice(-5);
+    curr.transcript_entries = curr.transcript_entries.filter(
+      (entry) => !tail.some((prevEntry) => isNearDuplicate(entry, prevEntry))
+    );
+  }
   for (const seg of segmentsWithPass1) {
     if (seg.pass1 != null) {
       sections.push(renderPass1(seg.pass1, speakerMapping));
@@ -5902,7 +5988,7 @@ async function run2(args) {
 }
 
 // src/cli/index.ts
-var version = "0.6.2";
+var version = "0.6.3";
 var DEFAULT_OUTPUT2 = "./vidistill-output/";
 var SUBCOMMANDS = {
   mcp: run,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vidistill",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "Video intelligence distiller — extract structured notes, transcripts, and insights from any video using Gemini",
   "type": "module",
   "license": "MIT",