yt-briefing 0.12.2 → 0.13.0

package/.claude/skills/yt/SKILL.md

@@ -22,10 +22,12 @@ Read `data/config.json` → `output_lang` once at the start of the loop. Phrase
 ```
 out = JSON.parse(`bun run src/yt-sweep.ts --reset`)   // Bash — first call: --reset rebuilds the queue fresh
 while true:
+  if out.skipped: FIRST, in output_lang, tell the user one line — "Skipped {out.skipped}:" + a `; `-joined list of `{channel} «{title}» — {reason}` from out.skips (videos the title/content filter dropped on the way here, including your channel directives). Then handle out.status below.
   out.status:
     "done"          → sweep finished — tell the user, stop
     "error"         → setup/config problem (e.g. missing API key) — show `out.error` to the user verbatim, stop
-    "rate_limited"  → transcript fetch blocked (usually a blocked egress IP) — tell the user, stop; recovery in README.md → Proxy
+    "rate_limited"  → YouTube is blocking the egress IP (429 / captcha) — tell the user, stop; recovery in README.md → Proxy
+    "tooling_error" → transcript toolchain failed (proxy down / yt-dlp / network) — tell the user, stop; check proxy health, then README.md → Proxy
     "rating_needed" → steps A–E
 ```
 

package/dist/lib/yt-lib.js

@@ -87,6 +87,16 @@ export function bumpStatePointer(content, handle, type, videoId, date) {
         throw new Error(`bumpStatePointer: row for ${handle} not found`);
     return bumpStateFrontmatterDate(lines.join('\n'), date);
 }
+/**
+ * Is this video already resolved for the current run? True when the per-(channel,type)
+ * state pointer has landed on it, OR it is in the run's `seen` set. The `seen` arm is the
+ * race-proof guard: `bumpStatePointer` is not monotonic, so a detached `--fill` re-bumping
+ * a stale pointer can transiently regress it and make a pointer-only check re-emit an
+ * already-rated video. `seen` is authoritative and pointer-independent.
+ */
+export function isResolved(pointer, videoId, seen) {
+    return pointer === videoId || seen.includes(videoId);
+}
 // ---------- channel profile: durable signal writes ----------
 /**
  * Append `line` under `## <section>`, creating the section if missing (before `## Notes`

package/dist/yt-rating.js

@@ -19,7 +19,7 @@
 import { readFileSync, writeFileSync, existsSync } from 'fs';
 import { loadEnv } from "./lib/env.js";
 import { parseChannels, appendSkipTitle, appendNote, bumpStatePointer } from "./lib/yt-lib.js";
-import { CHANNELS_MD, STATE_MD, PENDING_FILE, profilePath } from "./lib/paths.js";
+import { CHANNELS_MD, STATE_MD, PENDING_FILE, QUEUE_FILE, profilePath } from "./lib/paths.js";
 loadEnv();
 function getArg(args, name) {
     const idx = args.indexOf(name);
@@ -103,6 +103,20 @@ if (!args.noState) {
         stateBumped = true;
     }
 }
+// 3. Mark the rated video resolved in the run queue (`seen`) — independent of the state
+//    pointer, so a cursor-regression race in yt-sweep (a detached `--fill` re-bumping a
+//    stale pointer) can't re-emit an already-rated video. Best-effort: the queue is a
+//    same-day throwaway and no sweep process writes it while the user rates.
+if (!args.noState && existsSync(QUEUE_FILE)) {
+    try {
+        const q = JSON.parse(readFileSync(QUEUE_FILE, 'utf8'));
+        if (q?.built_at === date && Array.isArray(q.seen) && !q.seen.includes(args.id)) {
+            q.seen.push(args.id);
+            writeFileSync(QUEUE_FILE, JSON.stringify(q), 'utf8');
+        }
+    }
+    catch { /* corrupt / foreign queue → ignore; the next sweep rebuilds it */ }
+}
 console.log(JSON.stringify({
     ok: true,
     profile: `channels/${ch.slug}.md`,

package/dist/yt-sweep.js

@@ -50,7 +50,7 @@
 import { readFileSync, writeFileSync, existsSync, rmSync, mkdirSync, renameSync, appendFileSync } from 'node:fs';
 import { spawn } from 'node:child_process';
 import { loadEnv, missingEnv, missingEnvMessage, REQUIRED_LLM, REQUIRED_YOUTUBE } from "./lib/env.js";
-import { parseChannels, parseState, bumpStatePointer } from "./lib/yt-lib.js";
+import { parseChannels, parseState, bumpStatePointer, isResolved } from "./lib/yt-lib.js";
 import { chat, getModel } from "./lib/llm.js";
 import { outputLang } from "./lib/config.js";
 import { PKG_ROOT, CHANNELS_MD, STATE_MD, CACHE_DIR, QUEUE_FILE, REST_FILE, PENDING_FILE, PREFETCH_FILE, LOG_FILE, profilePath, script, } from "./lib/paths.js";
@@ -77,6 +77,15 @@ const DEBUG = !!process.env.YT_BRIEFING_DEBUG;
 const T0 = Date.now();
 const log = (msg) => { if (DEBUG)
     appendFileSync(LOG_FILE, `⏱ ${msg} (+${Date.now() - T0}ms)\n`); };
+const skipLedger = [];
+const recordedSkips = new Set();
+/** Record a skip once per videoId — the fg/bg expansion race can apply a title-skip twice. */
+function recordSkip(log) {
+    if (recordedSkips.has(log.videoId))
+        return;
+    recordedSkips.add(log.videoId);
+    skipLedger.push(log);
+}
 // ---------- subprocess helper ----------
 function run(cmd) {
     return new Promise((resolve, reject) => {
@@ -114,8 +123,10 @@ function persistSkip(item, status) {
  * Idempotent — re-applying the same skip is a no-op. ONLY the foreground calls this.
  */
 function applyTitleSkips(skips) {
-    for (const s of skips)
+    for (const s of skips) {
         bumpState(s.channel, s.type, s.videoId);
+        recordSkip({ stage: 'title', channel: s.channel, videoId: s.videoId, title: s.title, reason: s.reason || 'title filter' });
+    }
 }
 /** Append items, skipping any videoId already queued OR already resolved (seen). */
 function enqueue(queue, add) {
@@ -207,17 +218,18 @@ function spawnBackgroundFill() {
 }
 function emit(obj) {
     log(`EXIT status=${obj.status}`);
-    process.stdout.write(JSON.stringify(obj));
+    const ledger = skipLedger.length ? { skipped: skipLedger.length, skips: skipLedger } : {};
+    process.stdout.write(JSON.stringify({ ...obj, ...ledger }));
     process.exit(0);
 }
 // ---------- LLM gates ----------
 /** Title filter: batch-classify a channel's non-baseline titles. Falls back to keep-all on any error. */
 async function runTitleFilter(profilePathAbs, videos) {
-    const skip = new Set();
+    const skip = new Map();
     if (!existsSync(profilePathAbs))
         return skip;
     const profile = readFileSync(profilePathAbs, 'utf8');
-    if (!/##\s*Skip titles/.test(profile))
+    if (!/##\s*(Skip titles|Notes|Channel policy)/.test(profile))
         return skip;
     const toClassify = videos.filter(v => !v.is_baseline);
     if (toClassify.length === 0)
@@ -227,8 +239,8 @@ async function runTitleFilter(profilePathAbs, videos) {
 Channel profile:
 ${profile}
 
-Focus on the '## Skip titles' section (titles to skip) and any '## Notes' rules.
-Keep by default — only skip a title that clearly matches the worthless pattern. If that section is missing or empty: classify all as keep.
+Skip a title when it clearly matches a skip rule from the profile — a '## Skip titles' example, or a skip/reject directive in '## Notes' or '## Channel policy' that is judgeable from the title alone (defer transcript-dependent rejections to the content stage).
+Keep by default: skip only on a clear match, and when unsure, keep.
 
 Videos:
 ${JSON.stringify(toClassify.map(v => ({ id: v.videoId, title: v.title, type: v.type })))}
@@ -253,7 +265,7 @@ Output ONLY a raw JSON array (no markdown fences, no explanation):
         const parsed = JSON.parse(out.slice(start, end + 1));
         for (const r of parsed)
             if (r.result === 'skip')
-                skip.add(r.id);
+                skip.set(r.id, (r.reason ?? '').trim());
     }
     catch { /* keep-all */ }
     return skip;
@@ -263,7 +275,7 @@ async function runContentFilter(item, transcript) {
     const profile = existsSync(item.profile_path) ? readFileSync(item.profile_path, 'utf8') : '';
     const baselineNote = item.is_baseline ? ' · baseline' : '';
     const profileSection = profile
-        ? `\nChannel profile (sections to honor: Channel policy, Summary format, Cut sections, Episode types, Notes):\n${profile}\n`
+        ? `\nChannel directives — standing instructions the user set for THIS channel (sections: Channel policy, Summary format, Cut sections, Episode types, Notes). Carry them out as written: a directive may reject the video outright (see the skip check in step 1), or — for a video that passes — add a section, shift emphasis or tone, or scrutinize/flag specific things within the briefing format (never changing its skeleton). If a directive asks you to verify or scrutinize claims, assess them against your own background knowledge: state what you can corroborate, what looks overstated, cherry-picked or misattributed, and what you cannot place. Always mark your confidence, and say plainly this is a from-memory assessment, not live verification — for anything recent or outside your knowledge, flag that instead of guessing. Keep "what the video claims" separate from your own assessment.\n${profile}\n`
         : '';
     const prompt = `Write a summary of this YouTube video in ${LANG}, OR return 'OFFTOPIC: <reason>' if the transcript clearly does not match what the title/channel promises.
 
@@ -279,17 +291,18 @@ Transcript:
 ${transcript}
 ${profileSection}
 Steps:
-1. Substance check: does the transcript actually deliver what the title promises? If clearly not → output ONLY 'OFFTOPIC: <short reason>' and stop.
+1. Substance / skip check — output ONLY 'OFFTOPIC: <short reason>' and stop if EITHER (a) the transcript clearly does not deliver what the title promises, OR (b) a channel directive clearly instructs rejecting this kind of content and the transcript clearly meets that rejection criterion. When unsure, keep. Otherwise continue.
 2. Otherwise write the summary:
    - Header: ### ${item.channel} — "${item.title}"
    - Subtitle: _${item.publishedAt} · ${item.type} · https://youtube.com/watch?v=${item.videoId}${baselineNote}_
    - 2-5 numbered thematic sections × 2-5 sentences each
    - At most 5-8 short quotes from the transcript
    - No timestamps
-3. Language: natural ${LANG}. Avoid calques/anglicisms; use foreign words only for proper nouns or established technical terms. Section headers should be verb phrases, not noun stacks.
-4. Output: ONLY the summary OR 'OFFTOPIC: ...'. No preamble, no trailing commentary.`;
+3. Apply the channel directives above, if any — they shape the briefing's content, emphasis, and tone, but not the header/subtitle/section skeleton. Put any directive-driven assessment in its own short, clearly-labeled section.
+4. Language: natural ${LANG}. Avoid calques/anglicisms; use foreign words only for proper nouns or established technical terms. Section headers should be verb phrases, not noun stacks.
+5. Output: ONLY the briefing itself OR 'OFFTOPIC: ...'. No preamble, and no meta-commentary about these instructions.`;
     return await chat(prompt, {
-        system: `You are a video summarizer writing in ${LANG}. Follow the task instructions exactly. Output only the summary or 'OFFTOPIC: <reason>'. No preamble, no commentary.`,
+        system: `You write channel briefings in ${LANG}, following the task instructions and the channel's standing directives exactly. Output only the briefing or 'OFFTOPIC: <reason>' — no preamble, no meta-commentary about the instructions.`,
         model: getModel(),
     });
 }
@@ -321,7 +334,7 @@ async function expandChannel(ref) {
     const skips = [];
     for (const it of candidates) {
         if (titleSkip.has(it.videoId))
-            skips.push({ channel: it.channel, type: it.type, videoId: it.videoId });
+            skips.push({ channel: it.channel, type: it.type, videoId: it.videoId, title: it.title, reason: titleSkip.get(it.videoId) || 'title filter' });
         else
             items.push(it);
     }
@@ -418,15 +431,19 @@ async function processItem(item) {
     log(`transcript ${item.videoId} ${Date.now() - tT}ms (exit ${t.code})`);
     if (t.code === 2)
         return { kind: 'rate_limited' };
+    // exit 3 = tooling/proxy failure (503, tunnel down, missing yt-dlp) — NOT a missing-captions
+    // case. Surface it distinctly so the operator looks at the toolchain, not at YouTube/IP.
+    if (t.code === 3)
+        return { kind: 'tooling_error' };
     if (t.code !== 0)
         return { kind: 'skip', status: 'no_transcript' };
     if (!t.stdout.trim())
-        return { kind: 'skip', status: 'content_skip' };
+        return { kind: 'skip', status: 'content_skip', reason: 'empty transcript' };
     const tC = Date.now();
     const summary = await runContentFilter(item, t.stdout);
     log(`content ${item.videoId} ${Date.now() - tC}ms`);
     if (summary.startsWith('OFFTOPIC:'))
-        return { kind: 'skip', status: 'content_skip' };
+        return { kind: 'skip', status: 'content_skip', reason: summary.slice(9).trim() };
     return { kind: 'ratable', summary };
 }
 async function advance(queue) {
@@ -437,8 +454,9 @@ async function advance(queue) {
         if (queue.items.length === 0)
             break;
         const item = queue.items[0];
-        // Head already resolved last round (rated/skipped → pointer landed on it) → drop.
-        if (statePointer(item) === item.videoId) {
+        // Head already resolved last round (rated/skipped → pointer landed on it, or it's in
+        // `seen`) → drop. The `seen` arm survives a --fill pointer-regression race (isResolved).
+        if (isResolved(statePointer(item), item.videoId, queue.seen)) {
             dropHead(queue);
             continue;
         }
@@ -453,8 +471,13 @@ async function advance(queue) {
             writeFileSync(QUEUE_FILE, JSON.stringify(queue));
             emit({ status: 'rate_limited' });
         }
+        if (result.kind === 'tooling_error') {
+            writeFileSync(QUEUE_FILE, JSON.stringify(queue));
+            emit({ status: 'tooling_error' });
+        }
         if (result.kind === 'skip') {
             persistSkip(item, result.status);
+            recordSkip({ stage: 'content', channel: item.channel, videoId: item.videoId, title: item.title, reason: result.reason ?? (result.status === 'no_transcript' ? 'no transcript' : 'off-topic') });
             dropHead(queue);
             continue;
         }
@@ -489,7 +512,7 @@ async function runPrefetch(videoId) {
     const item = queue.items.find(i => i.videoId === videoId);
     if (!item)
         process.exit(0);
-    if (statePointer(item) === item.videoId)
+    if (isResolved(statePointer(item), item.videoId, queue.seen))
         process.exit(0); // already resolved
     if (loadPrefetch(item.videoId))
         process.exit(0); // already warm

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yt-briefing",
-  "version": "0.12.2",
+  "version": "0.13.0",
   "description": "A self-learning YouTube briefing engine: it sweeps the channels you follow, filters noise in two stages (title, then transcript), summarizes the rest in your language, and adapts to your ratings — one video at a time.",
   "type": "module",
   "bin": {