yt-briefing 0.12.3 → 0.13.1

package/.claude/skills/yt/SKILL.md

@@ -22,10 +22,12 @@ Read `data/config.json` → `output_lang` once at the start of the loop. Phrase
 ```
 out = JSON.parse(`bun run src/yt-sweep.ts --reset`)   // Bash — first call: --reset rebuilds the queue fresh
 while true:
+  if out.skipped: FIRST, in output_lang, tell the user one line — "Skipped {out.skipped}:" + a `; `-joined list of `{channel} «{title}» — {reason}` from out.skips (videos the title/content filter dropped on the way here, including your channel directives). Then handle out.status below.
   out.status:
     "done"          → sweep finished — tell the user, stop
     "error"         → setup/config problem (e.g. missing API key) — show `out.error` to the user verbatim, stop
-    "rate_limited"  → transcript fetch blocked (usually a blocked egress IP) — tell the user, stop; recovery in README.md → Proxy
+    "rate_limited"  → YouTube is blocking the egress IP (429 / captcha) — tell the user, stop; recovery in README.md → Proxy
+    "tooling_error" → transcript toolchain failed (proxy down / yt-dlp / network) — tell the user, stop; check proxy health, then README.md → Proxy
     "rating_needed" → steps A–E
 ```
 

package/README.md

@@ -150,7 +150,7 @@ YT_BRIEFING_LLM_MODEL=gemini-2.5-flash
 
 Want something else? Change those three lines for OpenRouter (`https://openrouter.ai/api/v1`),
 OpenAI (`https://api.openai.com/v1`), Anthropic (`https://api.anthropic.com/v1/`,
-e.g. `claude-sonnet-4-6`), or a local Ollama (`http://localhost:11434/v1`). Set
+e.g. `claude-sonnet-5`), or a local Ollama (`http://localhost:11434/v1`). Set
 `YT_BRIEFING_LLM_BASE_URL`, `_API_KEY`, and `_MODEL` in your root `.env` (see [Setup](#setup)).
 
 ## Why an API, not the agent's native model

package/dist/yt-channel-pending.js

@@ -2,8 +2,11 @@
 /**
  * Usage: bun src/yt-channel-pending.ts @HANDLE
  *
- * For one channel: reads state.md pointers + last-updated date,
- * fetches the channel's videos in-process via lib/yt-api.ts (--since updated),
+ * For one channel: reads state.md pointers, fetches the channel's videos in-process via
+ * lib/yt-api.ts (--since a fixed lookback, NOT state.md's `updated` — that column is a
+ * write-timestamp, not the pointer video's publish date; using it as the fetch cutoff
+ * silently orphaned backlog videos on channels swept more than once per day, see
+ * CHANGELOG),
  * filters to videos NEWER than each type's pointer (or baseline if pointer null),
  * sorts ASC by publishedAt (process oldest first → state pointer advances monotonically),
  * outputs JSON array: [{videoId, title, publishedAt, type, is_baseline}].
@@ -28,7 +31,14 @@ if (!row) {
     console.error(`Channel ${handle} not found in state.md`);
     process.exit(1);
 }
-const since = row.updated ?? '2020-01-01';
+// `row.updated` is when state.md was last WRITTEN (stamped to today on every bump),
+// not when the pointer video was PUBLISHED — reusing it as the fetch cutoff shrinks the
+// window to "today" after the first bump of the day, so a second same-day sweep can
+// silently drop any not-yet-processed backlog between the pointer and today. A fixed
+// lookback avoids that: the pointer-cutoff logic below still does the real filtering,
+// this just has to be wide enough to always re-include the pointer video itself.
+const LOOKBACK_DAYS = 45;
+const since = new Date(Date.now() - LOOKBACK_DAYS * 24 * 60 * 60 * 1000).toISOString().slice(0, 10);
 let videos;
 try {
     videos = await fetchChannelVideos(handle, { since });

package/dist/yt-sweep.js

@@ -77,6 +77,15 @@ const DEBUG = !!process.env.YT_BRIEFING_DEBUG;
 const T0 = Date.now();
 const log = (msg) => { if (DEBUG)
     appendFileSync(LOG_FILE, `⏱ ${msg} (+${Date.now() - T0}ms)\n`); };
+const skipLedger = [];
+const recordedSkips = new Set();
+/** Record a skip once per videoId — the fg/bg expansion race can apply a title-skip twice. */
+function recordSkip(log) {
+    if (recordedSkips.has(log.videoId))
+        return;
+    recordedSkips.add(log.videoId);
+    skipLedger.push(log);
+}
 // ---------- subprocess helper ----------
 function run(cmd) {
     return new Promise((resolve, reject) => {
@@ -114,8 +123,10 @@ function persistSkip(item, status) {
  * Idempotent — re-applying the same skip is a no-op. ONLY the foreground calls this.
  */
 function applyTitleSkips(skips) {
-    for (const s of skips)
+    for (const s of skips) {
         bumpState(s.channel, s.type, s.videoId);
+        recordSkip({ stage: 'title', channel: s.channel, videoId: s.videoId, title: s.title, reason: s.reason || 'title filter' });
+    }
 }
 /** Append items, skipping any videoId already queued OR already resolved (seen). */
 function enqueue(queue, add) {
@@ -207,17 +218,18 @@ function spawnBackgroundFill() {
 }
 function emit(obj) {
     log(`EXIT status=${obj.status}`);
-    process.stdout.write(JSON.stringify(obj));
+    const ledger = skipLedger.length ? { skipped: skipLedger.length, skips: skipLedger } : {};
+    process.stdout.write(JSON.stringify({ ...obj, ...ledger }));
     process.exit(0);
 }
 // ---------- LLM gates ----------
 /** Title filter: batch-classify a channel's non-baseline titles. Falls back to keep-all on any error. */
 async function runTitleFilter(profilePathAbs, videos) {
-    const skip = new Set();
+    const skip = new Map();
     if (!existsSync(profilePathAbs))
         return skip;
     const profile = readFileSync(profilePathAbs, 'utf8');
-    if (!/##\s*Skip titles/.test(profile))
+    if (!/##\s*(Skip titles|Notes|Channel policy)/.test(profile))
         return skip;
     const toClassify = videos.filter(v => !v.is_baseline);
     if (toClassify.length === 0)
@@ -227,8 +239,8 @@ async function runTitleFilter(profilePathAbs, videos) {
 Channel profile:
 ${profile}
 
-Focus on the '## Skip titles' section (titles to skip) and any '## Notes' rules.
-Keep by default — only skip a title that clearly matches the worthless pattern. If that section is missing or empty: classify all as keep.
+Skip a title when it clearly matches a skip rule from the profile — a '## Skip titles' example, or a skip/reject directive in '## Notes' or '## Channel policy' that is judgeable from the title alone (defer transcript-dependent rejections to the content stage).
+Keep by default: skip only on a clear match, and when unsure, keep.
 
 Videos:
 ${JSON.stringify(toClassify.map(v => ({ id: v.videoId, title: v.title, type: v.type })))}
@@ -253,7 +265,7 @@ Output ONLY a raw JSON array (no markdown fences, no explanation):
         const parsed = JSON.parse(out.slice(start, end + 1));
         for (const r of parsed)
             if (r.result === 'skip')
-                skip.add(r.id);
+                skip.set(r.id, (r.reason ?? '').trim());
     }
     catch { /* keep-all */ }
     return skip;
@@ -263,7 +275,7 @@ async function runContentFilter(item, transcript) {
     const profile = existsSync(item.profile_path) ? readFileSync(item.profile_path, 'utf8') : '';
     const baselineNote = item.is_baseline ? ' · baseline' : '';
     const profileSection = profile
-        ? `\nChannel profile (sections to honor: Channel policy, Summary format, Cut sections, Episode types, Notes):\n${profile}\n`
+        ? `\nChannel directives — standing instructions the user set for THIS channel (sections: Channel policy, Summary format, Cut sections, Episode types, Notes). Carry them out as written: a directive may reject the video outright (see the skip check in step 1), or — for a video that passes — add a section, shift emphasis or tone, or scrutinize/flag specific things within the briefing format (never changing its skeleton). If a directive asks you to verify or scrutinize claims, assess them against your own background knowledge: state what you can corroborate, what looks overstated, cherry-picked or misattributed, and what you cannot place. Always mark your confidence, and say plainly this is a from-memory assessment, not live verification — for anything recent or outside your knowledge, flag that instead of guessing. Keep "what the video claims" separate from your own assessment.\n${profile}\n`
         : '';
     const prompt = `Write a summary of this YouTube video in ${LANG}, OR return 'OFFTOPIC: <reason>' if the transcript clearly does not match what the title/channel promises.
 
@@ -279,17 +291,18 @@ Transcript:
 ${transcript}
 ${profileSection}
 Steps:
-1. Substance check: does the transcript actually deliver what the title promises? If clearly not → output ONLY 'OFFTOPIC: <short reason>' and stop.
+1. Substance / skip check — output ONLY 'OFFTOPIC: <short reason>' and stop if EITHER (a) the transcript clearly does not deliver what the title promises, OR (b) a channel directive clearly instructs rejecting this kind of content and the transcript clearly meets that rejection criterion. When unsure, keep. Otherwise continue.
 2. Otherwise write the summary:
    - Header: ### ${item.channel} — "${item.title}"
    - Subtitle: _${item.publishedAt} · ${item.type} · https://youtube.com/watch?v=${item.videoId}${baselineNote}_
    - 2-5 numbered thematic sections × 2-5 sentences each
    - At most 5-8 short quotes from the transcript
    - No timestamps
-3. Language: natural ${LANG}. Avoid calques/anglicisms; use foreign words only for proper nouns or established technical terms. Section headers should be verb phrases, not noun stacks.
-4. Output: ONLY the summary OR 'OFFTOPIC: ...'. No preamble, no trailing commentary.`;
+3. Apply the channel directives above, if any — they shape the briefing's content, emphasis, and tone, but not the header/subtitle/section skeleton. Put any directive-driven assessment in its own short, clearly-labeled section.
+4. Language: natural ${LANG}. Avoid calques/anglicisms; use foreign words only for proper nouns or established technical terms. Section headers should be verb phrases, not noun stacks.
+5. Output: ONLY the briefing itself OR 'OFFTOPIC: ...'. No preamble, and no meta-commentary about these instructions.`;
     return await chat(prompt, {
-        system: `You are a video summarizer writing in ${LANG}. Follow the task instructions exactly. Output only the summary or 'OFFTOPIC: <reason>'. No preamble, no commentary.`,
+        system: `You write channel briefings in ${LANG}, following the task instructions and the channel's standing directives exactly. Output only the briefing or 'OFFTOPIC: <reason>' — no preamble, no meta-commentary about the instructions.`,
         model: getModel(),
     });
 }
@@ -321,7 +334,7 @@ async function expandChannel(ref) {
     const skips = [];
     for (const it of candidates) {
         if (titleSkip.has(it.videoId))
-            skips.push({ channel: it.channel, type: it.type, videoId: it.videoId });
+            skips.push({ channel: it.channel, type: it.type, videoId: it.videoId, title: it.title, reason: titleSkip.get(it.videoId) || 'title filter' });
         else
             items.push(it);
     }
@@ -418,15 +431,19 @@ async function processItem(item) {
     log(`transcript ${item.videoId} ${Date.now() - tT}ms (exit ${t.code})`);
     if (t.code === 2)
         return { kind: 'rate_limited' };
+    // exit 3 = tooling/proxy failure (503, tunnel down, missing yt-dlp) — NOT a missing-captions
+    // case. Surface it distinctly so the operator looks at the toolchain, not at YouTube/IP.
+    if (t.code === 3)
+        return { kind: 'tooling_error' };
     if (t.code !== 0)
         return { kind: 'skip', status: 'no_transcript' };
     if (!t.stdout.trim())
-        return { kind: 'skip', status: 'content_skip' };
+        return { kind: 'skip', status: 'content_skip', reason: 'empty transcript' };
     const tC = Date.now();
     const summary = await runContentFilter(item, t.stdout);
     log(`content ${item.videoId} ${Date.now() - tC}ms`);
     if (summary.startsWith('OFFTOPIC:'))
-        return { kind: 'skip', status: 'content_skip' };
+        return { kind: 'skip', status: 'content_skip', reason: summary.slice(9).trim() };
     return { kind: 'ratable', summary };
 }
 async function advance(queue) {
@@ -454,8 +471,13 @@ async function advance(queue) {
             writeFileSync(QUEUE_FILE, JSON.stringify(queue));
             emit({ status: 'rate_limited' });
         }
+        if (result.kind === 'tooling_error') {
+            writeFileSync(QUEUE_FILE, JSON.stringify(queue));
+            emit({ status: 'tooling_error' });
+        }
         if (result.kind === 'skip') {
             persistSkip(item, result.status);
+            recordSkip({ stage: 'content', channel: item.channel, videoId: item.videoId, title: item.title, reason: result.reason ?? (result.status === 'no_transcript' ? 'no transcript' : 'off-topic') });
             dropHead(queue);
             continue;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yt-briefing",
-  "version": "0.12.3",
+  "version": "0.13.1",
   "description": "A self-learning YouTube briefing engine: it sweeps the channels you follow, filters noise in two stages (title, then transcript), summarizes the rest in your language, and adapts to your ratings — one video at a time.",
   "type": "module",
   "bin": {