youtube-knowledge-mcp 2.2.0 → 3.0.1

package/dist/utils/frames.js

@@ -0,0 +1,60 @@
+import { inBatches } from './batches.js';
+import { YouTubeError } from './errors.js';
+import { FFMPEG_JPEG_QUALITY, runFfmpeg } from './ffmpeg.js';
+import { planTimestamps } from './frame-plan.js';
+import { requireFfmpeg } from './preflight.js';
+import { FFMPEG_NETWORK_INPUT_ARGS, PREVIEW_STREAM_SELECTOR, resolveStreamUrl, } from './video-stream.js';
+import { TIMEOUTS } from './ytdlp.js';
+import { getVideoInfo } from './youtube.js';
+/**
+ * Looking at a video: stills captured into memory, never onto disk.
+ *
+ * A picture returned inline is something the model can read, which is what makes
+ * "what is on screen at 3:20" answerable without a human in the loop. Nothing is
+ * written, so the same code serves a contact sheet and a handful of stills.
+ */
+/** Each frame is an inline image, so a call is bounded by what a reply can carry. */
+export const MAX_FRAMES_PER_CALL = 12;
+/** Wide enough to read slide text, small enough to be about 100 KB a frame. */
+export const DEFAULT_FRAME_WIDTH = 768;
+/** Concurrent ffmpeg processes. Each is one ranged request against the same stream. */
+const CAPTURE_CONCURRENCY = 4;
+async function captureOne(mediaUrl, timestamp, width) {
+    const failure = `ffmpeg could not capture the frame at ${String(timestamp)}s.`;
+    const nextStep = 'Try a different timestamp, or check that ffmpeg works with network input.';
+    const { stdout } = await runFfmpeg([
+        ...FFMPEG_NETWORK_INPUT_ARGS,
+        // -ss before -i seeks without decoding everything up to that point.
+        '-ss',
+        String(timestamp),
+        '-i',
+        mediaUrl,
+        '-frames:v',
+        '1',
+        '-vf',
+        `scale=min(${String(width)}\\,iw):-2`,
+        '-q:v',
+        FFMPEG_JPEG_QUALITY,
+        '-f',
+        'image2pipe',
+        '-c:v',
+        'mjpeg',
+        'pipe:1',
+    ], { failure, nextStep, timeoutMs: TIMEOUTS.transcript });
+    if (stdout.byteLength === 0)
+        throw new YouTubeError('YTDLP_FAILED', failure, { nextStep });
+    return { timestamp, bytes: stdout };
+}
+/** Capture the frames a request describes, in timestamp order. */
+export async function captureFrames(video, request, options) {
+    const { width, maxFrames, label } = options;
+    // Cheap local checks first: a missing ffmpeg or an impossible request should
+    // fail before a single network round trip.
+    await requireFfmpeg(label);
+    const info = await getVideoInfo(video);
+    const timestamps = planTimestamps(request, info.duration, maxFrames);
+    const mediaUrl = await resolveStreamUrl(info.id, PREVIEW_STREAM_SELECTOR, label);
+    const frames = await inBatches(timestamps, CAPTURE_CONCURRENCY, (timestamp) => captureOne(mediaUrl, timestamp, width));
+    return { videoId: info.id, title: info.title, duration: info.duration, frames };
+}
+//# sourceMappingURL=frames.js.map

package/dist/utils/frames.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"frames.js","sourceRoot":"","sources":["../../src/utils/frames.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAyB,MAAM,iBAAiB,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EACL,yBAAyB,EACzB,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C;;;;;;GAMG;AAEH,qFAAqF;AACrF,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AACtC,+EAA+E;AAC/E,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEvC,uFAAuF;AACvF,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAsB9B,KAAK,UAAU,UAAU,CACvB,QAAgB,EAChB,SAAiB,EACjB,KAAa;IAEb,MAAM,OAAO,GAAG,yCAAyC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC;IAC/E,MAAM,QAAQ,GAAG,2EAA2E,CAAC;IAE7F,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAChC;QACE,GAAG,yBAAyB;QAC5B,oEAAoE;QACpE,KAAK;QACL,MAAM,CAAC,SAAS,CAAC;QACjB,IAAI;QACJ,QAAQ;QACR,WAAW;QACX,GAAG;QACH,KAAK;QACL,aAAa,MAAM,CAAC,KAAK,CAAC,WAAW;QACrC,MAAM;QACN,mBAAmB;QACnB,IAAI;QACJ,YAAY;QACZ,MAAM;QACN,OAAO;QACP,QAAQ;KACT,EACD,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,UAAU,EAAE,CACtD,CAAC;IAEF,IAAI,MAAM,CAAC,UAAU,KAAK,CAAC;QAAE,MAAM,IAAI,YAAY,CAAC,cAAc,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC3F,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AACtC,CAAC;AAED,kEAAkE;AAClE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAa,EACb,OAAyB,EACzB,OAAuB;IAEvB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IAE5C,6EAA6E;IAC7E,2CAA2C;IAC3C,MAAM,aAAa,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAErE,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,uBAAuB,EAAE,KAAK,CAAC,CAAC;IACjF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,mBAAmB,EAAE,CAAC,SAAS,EAAE,EAAE,CAC5E,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CACvC,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;AAClF,CAAC"}

package/dist/utils/gif.d.ts

@@ -0,0 +1,31 @@
+/**
+ * A short animated GIF from a moment of a video.
+ *
+ * A GIF stores every frame whole, so its size grows with length, width and frame
+ * rate far faster than a video's does. The length is therefore capped: past it the
+ * file is one nobody can share, and a clip is the right tool.
+ */
+export declare const MAX_GIF_SECONDS = 15;
+export declare const DEFAULT_GIF_WIDTH = 480;
+export declare const DEFAULT_GIF_FPS = 12;
+export interface GifResult {
+    videoId: string;
+    title: string;
+    filePath: string;
+    start: number;
+    end: number;
+    duration: number;
+    width: number;
+    fps: number;
+    bytes: number;
+}
+export declare function extractGif(video: string, rangeOptions: {
+    start?: number;
+    end?: number;
+    chapter?: string;
+}, options: {
+    width: number;
+    fps: number;
+    outputDir?: string;
+}): Promise<GifResult>;
+//# sourceMappingURL=gif.d.ts.map

package/dist/utils/gif.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gif.d.ts","sourceRoot":"","sources":["../../src/utils/gif.ts"],"names":[],"mappings":"AAeA;;;;;;GAMG;AAEH,eAAO,MAAM,eAAe,KAAK,CAAC;AAClC,eAAO,MAAM,iBAAiB,MAAM,CAAC;AACrC,eAAO,MAAM,eAAe,KAAK,CAAC;AAIlC,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EACb,YAAY,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,EAChE,OAAO,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC1D,OAAO,CAAC,SAAS,CAAC,CAoEpB"}

package/dist/utils/gif.js

@@ -0,0 +1,77 @@
+import { mkdir, rm, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { resolveRange, safeStem } from './clips.js';
+import { YouTubeError } from './errors.js';
+import { runFfmpeg } from './ffmpeg.js';
+import { dataDir } from './paths.js';
+import { requireFfmpeg } from './preflight.js';
+import { resolveOutputDir } from './validate.js';
+import { FFMPEG_NETWORK_INPUT_ARGS, PREVIEW_STREAM_SELECTOR, resolveStreamUrl, } from './video-stream.js';
+import { TIMEOUTS } from './ytdlp.js';
+/**
+ * A short animated GIF from a moment of a video.
+ *
+ * A GIF stores every frame whole, so its size grows with length, width and frame
+ * rate far faster than a video's does. The length is therefore capped: past it the
+ * file is one nobody can share, and a clip is the right tool.
+ */
+export const MAX_GIF_SECONDS = 15;
+export const DEFAULT_GIF_WIDTH = 480;
+export const DEFAULT_GIF_FPS = 12;
+const GIFS_DIR = dataDir('gifs');
+export async function extractGif(video, rangeOptions, options) {
+    const { width, fps } = options;
+    // Cheap local checks first: a missing ffmpeg, an unwritable destination or an
+    // over-long range should fail now, not after a network round trip.
+    await requireFfmpeg('GIF extraction');
+    const targetDir = resolveOutputDir(options.outputDir, GIFS_DIR);
+    const { range, title, videoId } = await resolveRange(video, rangeOptions);
+    const duration = range.end - range.start;
+    if (duration > MAX_GIF_SECONDS) {
+        throw new YouTubeError('INVALID_INPUT', `A GIF can be at most ${String(MAX_GIF_SECONDS)} seconds, and this range is ${String(Math.round(duration))}.`, { nextStep: 'Narrow the range, or use extract_clip for a longer excerpt.' });
+    }
+    await mkdir(targetDir, { recursive: true });
+    const mediaUrl = await resolveStreamUrl(videoId, PREVIEW_STREAM_SELECTOR, 'extract_gif');
+    const filePath = join(targetDir, `${safeStem(title, videoId)} [${String(Math.round(range.start))}-${String(Math.round(range.end))}].gif`);
+    try {
+        await runFfmpeg([
+            ...FFMPEG_NETWORK_INPUT_ARGS,
+            // Both before -i: seek there, then read only the range.
+            '-ss',
+            String(range.start),
+            '-t',
+            String(duration),
+            '-i',
+            mediaUrl,
+            '-vf',
+            // The palette is built from this clip's own frames, which is what keeps the
+            // colours faithful within a GIF's 256.
+            `fps=${String(fps)},scale=${String(width)}:-1:flags=lanczos,split[a][b];[a]palettegen[p];[b][p]paletteuse`,
+            '-loop',
+            '0',
+            '-y',
+            filePath,
+        ], {
+            failure: 'ffmpeg could not make that GIF.',
+            nextStep: 'Try a shorter range, or check that ffmpeg works with network input.',
+            timeoutMs: TIMEOUTS.transcript,
+        });
+    }
+    catch (error) {
+        // ffmpeg writes as it goes, so a failed or cancelled run leaves half a GIF behind.
+        await rm(filePath, { force: true });
+        throw error;
+    }
+    return {
+        videoId,
+        title,
+        filePath,
+        start: range.start,
+        end: range.end,
+        duration,
+        width,
+        fps,
+        bytes: (await stat(filePath)).size,
+    };
+}
+//# sourceMappingURL=gif.js.map

package/dist/utils/gif.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gif.js","sourceRoot":"","sources":["../../src/utils/gif.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EACL,yBAAyB,EACzB,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC;;;;;;GAMG;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC;AAClC,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AACrC,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC;AAElC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AAcjC,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAa,EACb,YAAgE,EAChE,OAA2D;IAE3D,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAE/B,8EAA8E;IAC9E,mEAAmE;IACnE,MAAM,aAAa,CAAC,gBAAgB,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEhE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC;IACzC,IAAI,QAAQ,GAAG,eAAe,EAAE,CAAC;QAC/B,MAAM,IAAI,YAAY,CACpB,eAAe,EACf,wBAAwB,MAAM,CAAC,eAAe,CAAC,+BAA+B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,EAC7G,EAAE,QAAQ,EAAE,6DAA6D,EAAE,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,uBAAuB,EAAE,aAAa,CAAC,CAAC;IACzF,MAAM,QAAQ,GAAG,IAAI,CACnB,SAAS,EACT,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CACxG,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,SAAS,CACb;YACE,GAAG,yBAAyB;YAC5B,wDAAwD;YACxD,KAAK;YACL,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;YACnB,IAAI;YACJ,MAAM,CAAC,QAAQ,CAAC;YAChB,IAAI;YACJ,QAAQ;YACR,KAAK;YACL,4EAA4E;YAC5E,uCAAuC;YACvC,OAAO,MAAM,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,KAAK,CAAC,iEAAiE;YAC1G,OAAO;YACP,GAAG;YACH,IAAI;YACJ,QAAQ;SACT,EACD;YACE,OAAO,EAAE,iCAAiC;YAC1C,QAAQ,EAAE,qEAAqE;YAC/E,SAAS,EAAE,QAAQ,CAAC,UAAU;SAC/B,CACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mFAAmF;QACnF,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,MAAM,KAAK,CAAC;IACd,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,QAAQ;QACR,KAAK;QACL,GAAG;QACH,KAAK,EAAE,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;KACnC,CAAC;AACJ,CAAC"}

package/dist/utils/harvest-catalog.d.ts

@@ -0,0 +1,54 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { Coverage } from '../harvest-schemas.js';
+/**
+ * Reading a channel's whole catalogue.
+ *
+ * The listing plane is cheap and is not bot-checked: measured 1,716 videos in
+ * 31 seconds from one spawn, so a catalogue costs about a minute per channel
+ * and the old 100-video cap was self-imposed rather than forced.
+ *
+ * Every tab is walked because a channel's videos are not all under /videos —
+ * shorts, streams, releases and podcasts each have their own. The first tab
+ * that lists a video owns it; later tabs record themselves in `also_in_tabs`.
+ * Last-writer-wins would mislabel most of a music channel's catalogue.
+ */
+export declare const HARVEST_TABS: readonly ["videos", "shorts", "streams", "releases", "podcasts"];
+export type HarvestTab = (typeof HARVEST_TABS)[number];
+export declare const MAX_VIDEOS_PER_CATALOG = 10000;
+/**
+ * Whether a listing failed only because the channel has no such tab.
+ *
+ * Recognised by the error's code, which the classifier assigns from yt-dlp's own words.
+ * It used to look for those words in the message, which never carries them, so no
+ * channel lacking a tab was ever catalogued as complete.
+ */
+export declare function isAbsentTab(error: unknown): boolean;
+export interface TabResult {
+    tab: HarvestTab;
+    listed: number;
+    added: number;
+    complete: boolean;
+    error?: string;
+}
+export interface CatalogResult {
+    channelId: string;
+    name: string;
+    handle: string;
+    tabs: TabResult[];
+    videosSeen: number;
+    videosAdded: number;
+    coverage: Coverage;
+}
+/** A channel tab as yt-dlp's flat listing, one JSON object per line, newest first. */
+export declare function listTab(channelUrl: string, tab: HarvestTab, limit: number, label?: string): Promise<string>;
+/**
+ * Walks every tab of a channel into the store.
+ *
+ * A tab the channel simply does not have is not a failure — yt-dlp says so in
+ * words, and treating that as an error would make every channel without a
+ * podcasts tab look broken.
+ */
+export declare function harvestCatalog(database: DatabaseSync, channel: string, options?: {
+    maxVideos?: number;
+}): Promise<CatalogResult>;
+//# sourceMappingURL=harvest-catalog.d.ts.map

package/dist/utils/harvest-catalog.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harvest-catalog.d.ts","sourceRoot":"","sources":["../../src/utils/harvest-catalog.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAahD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEtD;;;;;;;;;;;GAWG;AAEH,eAAO,MAAM,YAAY,kEAAmE,CAAC;AAC7F,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvD,eAAO,MAAM,sBAAsB,QAAS,CAAC;AAE7C;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEnD;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,UAAU,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAsBD,sFAAsF;AACtF,wBAAsB,OAAO,CAC3B,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,MAAM,EACb,KAAK,SAA2B,GAC/B,OAAO,CAAC,MAAM,CAAC,CAiBjB;AAED;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GACnC,OAAO,CAAC,aAAa,CAAC,CAQxB"}

package/dist/utils/harvest-catalog.js

@@ -0,0 +1,173 @@
+import { runYtDlp, TIMEOUTS } from './ytdlp.js';
+import { FLAT_PRINT_TEMPLATE, flatEntrySchema, toVideoListItem } from './flat-listing.js';
+import { parseYtDlpJsonLines } from './ytdlp-parse.js';
+import { resolveListTarget } from './youtube-url.js';
+import { getChannelInfo } from './youtube-channel.js';
+import { coverageOf } from './coverage.js';
+import { saveReceipt } from './harvest-receipts.js';
+import { inTransaction } from './store.js';
+import { throwIfAborted, reportProgress } from './context.js';
+import { countRows } from './store-rows.js';
+import { withChannelHarvestLock } from './harvest-lock.js';
+import { YouTubeError } from './errors.js';
+/**
+ * Reading a channel's whole catalogue.
+ *
+ * The listing plane is cheap and is not bot-checked: measured 1,716 videos in
+ * 31 seconds from one spawn, so a catalogue costs about a minute per channel
+ * and the old 100-video cap was self-imposed rather than forced.
+ *
+ * Every tab is walked because a channel's videos are not all under /videos —
+ * shorts, streams, releases and podcasts each have their own. The first tab
+ * that lists a video owns it; later tabs record themselves in `also_in_tabs`.
+ * Last-writer-wins would mislabel most of a music channel's catalogue.
+ */
+export const HARVEST_TABS = ['videos', 'shorts', 'streams', 'releases', 'podcasts'];
+export const MAX_VIDEOS_PER_CATALOG = 10_000;
+/**
+ * Whether a listing failed only because the channel has no such tab.
+ *
+ * Recognised by the error's code, which the classifier assigns from yt-dlp's own words.
+ * It used to look for those words in the message, which never carries them, so no
+ * channel lacking a tab was ever catalogued as complete.
+ */
+export function isAbsentTab(error) {
+    return error instanceof YouTubeError && error.code === 'NOT_FOUND';
+}
+const UPSERT_VIDEO = `
+INSERT INTO video (
+  video_id, channel_id, title, duration_s, upload_date, view_count,
+  live_status, thumbnail_url, tab, also_in_tabs, catalog_rank, listed_at
+) VALUES (?,?,?,?,?,?,?,?,?,'[]',?,?)
+ON CONFLICT(video_id) DO UPDATE SET
+  title = excluded.title,
+  duration_s = excluded.duration_s,
+  view_count = excluded.view_count,
+  live_status = excluded.live_status,
+  thumbnail_url = excluded.thumbnail_url,
+  listed_at = excluded.listed_at,
+  -- The owning tab is kept; a later tab only records that it also lists this
+  -- video, so a re-walk in a different order cannot relabel the catalogue.
+  also_in_tabs = CASE
+    WHEN video.tab = excluded.tab THEN video.also_in_tabs
+    WHEN instr(video.also_in_tabs, excluded.tab) > 0 THEN video.also_in_tabs
+    ELSE json_insert(video.also_in_tabs, '$[#]', excluded.tab)
+  END`;
+/** A channel tab as yt-dlp's flat listing, one JSON object per line, newest first. */
+export async function listTab(channelUrl, tab, limit, label = `harvest_channel:${tab}`) {
+    return runYtDlp([
+        '--skip-download',
+        '--flat-playlist',
+        '--lazy-playlist',
+        '--print',
+        FLAT_PRINT_TEMPLATE,
+        '--playlist-end',
+        String(limit),
+    ], {
+        label,
+        timeoutMs: TIMEOUTS.transcript,
+        target: resolveListTarget(`${channelUrl}/${tab}`),
+    });
+}
+/**
+ * Walks every tab of a channel into the store.
+ *
+ * A tab the channel simply does not have is not a failure — yt-dlp says so in
+ * words, and treating that as an error would make every channel without a
+ * podcasts tab look broken.
+ */
+export async function harvestCatalog(database, channel, options = {}) {
+    const maxVideos = Math.min(options.maxVideos ?? 500, MAX_VIDEOS_PER_CATALOG);
+    // Resolving the channel comes before the lock because the lock is keyed on
+    // the channel id, and only YouTube can turn a handle into one.
+    const info = await getChannelInfo(channel);
+    return withChannelHarvestLock(info.channelId, () => walkTabs(database, info, maxVideos));
+}
+async function walkTabs(database, info, maxVideos) {
+    const now = Date.now();
+    database
+        .prepare(`INSERT INTO channel (channel_id, handle, name, channel_url, subscriber_count, description, first_seen_at, updated_at)
+       VALUES (?,?,?,?,?,?,?,?)
+       ON CONFLICT(channel_id) DO UPDATE SET
+         handle = excluded.handle, name = excluded.name, channel_url = excluded.channel_url,
+         subscriber_count = excluded.subscriber_count, description = excluded.description,
+         updated_at = excluded.updated_at`)
+        .run(info.channelId, info.handle, info.name, info.channelUrl, info.subscriberCount, info.description, now, now);
+    const tabs = [];
+    const seen = new Set();
+    let rank = 0;
+    // Counted from the store, not from `seen`: on a second run every video is
+    // new to the walk and none is new to the catalogue, and reporting the former
+    // as "added" would make a re-run look like a fresh harvest.
+    const countVideos = () => countRows(database.prepare('SELECT COUNT(*) FROM video WHERE channel_id = ?'), info.channelId);
+    const before = countVideos();
+    for (const tab of HARVEST_TABS) {
+        throwIfAborted();
+        reportProgress(tabs.length, HARVEST_TABS.length, `harvest_channel: ${tab}`);
+        let stdout;
+        try {
+            stdout = await listTab(info.channelUrl, tab, maxVideos);
+        }
+        catch (error) {
+            const message = error instanceof YouTubeError ? error.message : String(error);
+            const absent = isAbsentTab(error);
+            tabs.push({
+                tab,
+                listed: 0,
+                added: 0,
+                // A tab the channel does not have is a complete answer, not a gap.
+                complete: absent,
+                ...(absent ? {} : { error: message }),
+            });
+            continue;
+        }
+        const entries = parseYtDlpJsonLines(stdout, flatEntrySchema).map(toVideoListItem);
+        const insert = database.prepare(UPSERT_VIDEO);
+        const tabBefore = countVideos();
+        inTransaction(database, () => {
+            for (const entry of entries) {
+                if (seen.size >= maxVideos)
+                    break;
+                seen.add(entry.id);
+                insert.run(entry.id, info.channelId, entry.title, entry.duration, entry.uploadDate === '' ? null : entry.uploadDate, entry.viewCount ?? null, entry.liveStatus ?? null, entry.thumbnailUrl ?? null, tab, rank, now);
+                rank += 1;
+            }
+        });
+        tabs.push({
+            tab,
+            listed: entries.length,
+            added: countVideos() - tabBefore,
+            complete: entries.length < maxVideos,
+        });
+    }
+    const everyTabComplete = tabs.every((tab) => tab.complete);
+    const coverage = coverageOf({
+        scope: 'channel-catalog',
+        targetId: info.channelId,
+        have: seen.size,
+        source: `yt-dlp --flat-playlist over ${HARVEST_TABS.join(', ')}`,
+        // Only an unbroken walk of every tab proves the catalogue is whole. A tab
+        // that errored, or one that filled to the cap, leaves it unprovable.
+        ...(everyTabComplete
+            ? {
+                expected: { value: seen.size, source: 'source-exhausted' },
+                ranToExhaustion: true,
+            }
+            : {}),
+        ...(seen.size >= maxVideos ? { limitApplied: maxVideos } : {}),
+        ...(everyTabComplete ? {} : { resumeToken: `catalog:${info.channelId}` }),
+    });
+    inTransaction(database, () => {
+        saveReceipt(database, coverage, { startedAt: now });
+    });
+    return {
+        channelId: info.channelId,
+        name: info.name,
+        handle: info.handle,
+        tabs,
+        videosSeen: seen.size,
+        videosAdded: countVideos() - before,
+        coverage,
+    };
+}
+//# sourceMappingURL=harvest-catalog.js.map

package/dist/utils/harvest-catalog.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harvest-catalog.js","sourceRoot":"","sources":["../../src/utils/harvest-catalog.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC1F,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C;;;;;;;;;;;GAWG;AAEH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAU,CAAC;AAG7F,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAE7C;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,KAAc;IACxC,OAAO,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,CAAC;AACrE,CAAC;AAoBD,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;MAkBf,CAAC;AAEP,sFAAsF;AACtF,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,UAAkB,EAClB,GAAe,EACf,KAAa,EACb,KAAK,GAAG,mBAAmB,GAAG,EAAE;IAEhC,OAAO,QAAQ,CACb;QACE,iBAAiB;QACjB,iBAAiB;QACjB,iBAAiB;QACjB,SAAS;QACT,mBAAmB;QACnB,gBAAgB;QAChB,MAAM,CAAC,KAAK,CAAC;KACd,EACD;QACE,KAAK;QACL,SAAS,EAAE,QAAQ,CAAC,UAAU;QAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,UAAU,IAAI,GAAG,EAAE,CAAC;KAClD,CACF,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAsB,EACtB,OAAe,EACf,UAAkC,EAAE;IAEpC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,EAAE,sBAAsB,CAAC,CAAC;IAE7E,2EAA2E;IAC3E,+DAA+D;IAC/D,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAE3C,OAAO,sBAAsB,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,QAAsB,EACtB,IAAgD,EAChD,SAAiB;IAEjB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,QAAQ;SACL,OAAO,CACN;;;;;0CAKoC,CACrC;SACA,GAAG,CACF,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,GAAG,EACH,GAAG,CACJ,CAAC;IAEJ,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,0EAA0E;IAC1E,6EAA6E;IAC7E,4DAA4D;IAC5D,MAAM,WAAW,GAAG,GAAW,EAAE,CAC/B,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,iDAAiD,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACjG,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAC/B,cAAc,EAAE,CAAC;QACjB,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,oBAAoB,GAAG,EAAE,CAAC,CAAC;QAE5E,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QAC1D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9E,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC;gBACR,GAAG;gBACH,MAAM,EAAE,CAAC;gBACT,KAAK,EAAE,CAAC;gBACR,mEAAmE;gBACnE,QAAQ,EAAE,MAAM;gBAChB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;aACtC,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC;QAEhC,aAAa,CAAC,QAAQ,EAAE,GAAG,EAAE;YAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,IAAI,IAAI,SAAS;oBAAE,MAAM;gBAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACnB,MAAM,CAAC,GAAG,CACR,KAAK,CAAC,EAAE,EACR,IAAI,CAAC,SAAS,EACd,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,QAAQ,EACd,KAAK,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,EACjD,KAAK,CAAC,SAAS,IAAI,IAAI,EACvB,KAAK,CAAC,UAAU,IAAI,IAAI,EACxB,KAAK,CAAC,YAAY,IAAI,IAAI,EAC1B,GAAG,EACH,IAAI,EACJ,GAAG,CACJ,CAAC;gBACF,IAAI,IAAI,CAAC,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC;YACR,GAAG;YACH,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,WAAW,EAAE,GAAG,SAAS;YAChC,QAAQ,EAAE,OAAO,CAAC,MAAM,GAAG,SAAS;SACrC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,UAAU,CAAC;QAC1B,KAAK,EAAE,iBAAiB;QACxB,QAAQ,EAAE,IAAI,CAAC,SAAS;QACxB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,+BAA+B,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAChE,0EAA0E;QAC1E,qEAAqE;QACrE,GAAG,CAAC,gBAAgB;YAClB,CAAC,CAAC;gBACE,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,kBAA2B,EAAE;gBACnE,eAAe,EAAE,IAAI;aACtB;YACH,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,WAAW,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;KAC1E,CAAC,CAAC;IAEH,aAAa,CAAC,QAAQ,EAAE,GAAG,EAAE;QAC3B,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,IAAI;QACJ,UAAU,EAAE,IAAI,CAAC,IAAI;QACrB,WAAW,EAAE,WAAW,EAAE,GAAG,MAAM;QACnC,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/utils/harvest-details.d.ts

@@ -0,0 +1,30 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { Coverage } from '../harvest-schemas.js';
+import { type DetailFailure } from './detail-batch.js';
+import { type YouTubeErrorCode } from './errors.js';
+/**
+ * Reading every catalogued video of a channel, and keeping what each said.
+ *
+ * Resumable by construction: a video is read once, its row says so, and the next run
+ * takes the next unread one. What came back before a refusal is kept, because every
+ * batch commits on its own.
+ */
+export declare const DEFAULT_DETAIL_BATCH = 50;
+export declare const MAX_DETAIL_BATCH = 500;
+export interface DetailsResult {
+    channelId: string;
+    name: string;
+    attempted: number;
+    stored: number;
+    failures: DetailFailure[];
+    /** Videos still unread and still readable. */
+    remaining: number;
+    /** The error that made the run stop early, when one did. */
+    stoppedBecause?: YouTubeErrorCode;
+    coverage: Coverage;
+}
+export declare function harvestDetails(database: DatabaseSync, channel: string, options: {
+    maxVideos: number;
+    retryUnreadable?: boolean;
+}): Promise<DetailsResult>;
+//# sourceMappingURL=harvest-details.d.ts.map

package/dist/utils/harvest-details.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harvest-details.d.ts","sourceRoot":"","sources":["../../src/utils/harvest-details.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGtD,OAAO,EAAyC,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EAAgB,KAAK,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAQlE;;;;;;GAMG;AAEH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AACvC,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAQpC,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,cAAc,CAAC,EAAE,gBAAgB,CAAC;IAClC,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED,wBAAsB,cAAc,CAClC,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,eAAe,CAAC,EAAE,OAAO,CAAA;CAAE,GACxD,OAAO,CAAC,aAAa,CAAC,CAmFxB"}

package/dist/utils/harvest-details.js

@@ -0,0 +1,98 @@
+import { reportProgress, throwIfAborted } from './context.js';
+import { coverageOf } from './coverage.js';
+import { STOPPING, persist, read, stoppingCode } from './detail-batch.js';
+import { YouTubeError } from './errors.js';
+import { withChannelHarvestLock } from './harvest-lock.js';
+import { readReceipt, saveReceipt } from './harvest-receipts.js';
+import { inTransaction } from './store.js';
+import { catalogCounts, pendingVideoIds, unreadableCount } from './video-detail-store.js';
+import { getChannelInfo } from './youtube-channel.js';
+import { concurrencyState } from './ytdlp.js';
+/**
+ * Reading every catalogued video of a channel, and keeping what each said.
+ *
+ * Resumable by construction: a video is read once, its row says so, and the next run
+ * takes the next unread one. What came back before a refusal is kept, because every
+ * batch commits on its own.
+ */
+export const DEFAULT_DETAIL_BATCH = 50;
+export const MAX_DETAIL_BATCH = 500;
+/** Concurrent reads. Each is a full yt-dlp extraction, the call YouTube throttles hardest. */
+const READ_CONCURRENCY = 4;
+function plural(count, noun) {
+    return `${String(count)} ${noun}${count === 1 ? '' : 's'}`;
+}
+export async function harvestDetails(database, channel, options) {
+    const info = await getChannelInfo(channel);
+    if (catalogCounts(database, info.channelId).videos === 0) {
+        throw new YouTubeError('NOT_FOUND', `${info.name} has not been catalogued yet.`, {
+            nextStep: 'Call harvest_channel first, so there is a catalogue of videos to read.',
+        });
+    }
+    return withChannelHarvestLock(info.channelId, async () => {
+        const startedAt = Date.now();
+        const { channelId } = info;
+        const queue = pendingVideoIds(database, channelId, options.maxVideos, {
+            includeUnreadable: options.retryUnreadable === true,
+        });
+        const width = Math.max(1, Math.min(concurrencyState().limit, READ_CONCURRENCY));
+        const failures = [];
+        let stored = 0;
+        let attempted = 0;
+        let stoppedBecause;
+        for (let start = 0; start < queue.length && stoppedBecause === undefined; start += width) {
+            throwIfAborted();
+            reportProgress(start, queue.length, 'harvest_details');
+            const batch = await Promise.all(queue.slice(start, start + width).map(read));
+            attempted += batch.length;
+            const saved = persist(database, batch);
+            stored += saved.stored;
+            failures.push(...saved.failures);
+            stoppedBecause = stoppingCode(batch);
+        }
+        const counts = catalogCounts(database, channelId);
+        const remaining = pendingVideoIds(database, channelId, counts.videos).length;
+        const unreadable = unreadableCount(database, channelId);
+        const capBound = attempted >= options.maxVideos && remaining > 0;
+        const catalog = readReceipt(database, 'channel-catalog', channelId);
+        const stopReason = stoppedBecause === undefined ? undefined : STOPPING[stoppedBecause];
+        const coverage = coverageOf({
+            scope: 'video-details',
+            targetId: channelId,
+            have: counts.detailed,
+            source: 'yt-dlp -j per catalogued video',
+            // The catalogue's size is a fair total only if the catalogue itself is whole.
+            ...(catalog?.complete === true
+                ? { expected: { value: counts.videos, source: 'store:catalog' } }
+                : {}),
+            ...(capBound ? { limitApplied: options.maxVideos } : {}),
+            ...(stopReason !== undefined
+                ? { reason: stopReason }
+                : remaining === 0 && (failures.length > 0 || unreadable > 0)
+                    ? { reason: 'PARTIAL_ERROR' }
+                    : {}),
+            ...(remaining > 0 || stoppedBecause !== undefined
+                ? { resumeToken: `details:${channelId}` }
+                : {}),
+            ...(unreadable > 0
+                ? {
+                    note: `${plural(unreadable, 'video')} cannot be read (private, members-only, age-gated, removed or failing), so this stays incomplete.`,
+                }
+                : {}),
+        });
+        inTransaction(database, () => {
+            saveReceipt(database, coverage, { startedAt });
+        });
+        return {
+            channelId,
+            name: info.name,
+            attempted,
+            stored,
+            failures,
+            remaining,
+            ...(stoppedBecause === undefined ? {} : { stoppedBecause }),
+            coverage,
+        };
+    });
+}
+//# sourceMappingURL=harvest-details.js.map

package/dist/utils/harvest-details.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harvest-details.js","sourceRoot":"","sources":["../../src/utils/harvest-details.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAsB,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EAAE,YAAY,EAAyB,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1F,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C;;;;;;GAMG;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,EAAE,CAAC;AACvC,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AACpC,8FAA8F;AAC9F,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAE3B,SAAS,MAAM,CAAC,KAAa,EAAE,IAAY;IACzC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;AAC7D,CAAC;AAeD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAsB,EACtB,OAAe,EACf,OAAyD;IAEzD,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAE3C,IAAI,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,YAAY,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,IAAI,+BAA+B,EAAE;YAC/E,QAAQ,EAAE,wEAAwE;SACnF,CAAC,CAAC;IACL,CAAC;IAED,OAAO,sBAAsB,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAC3B,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE;YACpE,iBAAiB,EAAE,OAAO,CAAC,eAAe,KAAK,IAAI;SACpD,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAEhF,MAAM,QAAQ,GAAoB,EAAE,CAAC;QACrC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,cAA4C,CAAC;QAEjD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,EAAE,KAAK,IAAI,KAAK,EAAE,CAAC;YACzF,cAAc,EAAE,CAAC;YACjB,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;YAEvD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;YAC7E,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;YAE1B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACvC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACvB,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;YACjC,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAC7E,MAAM,UAAU,GAAG,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,SAAS,IAAI,OAAO,CAAC,SAAS,IAAI,SAAS,GAAG,CAAC,CAAC;QAEjE,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,EAAE,iBAAiB,EAAE,SAAS,CAAC,CAAC;QACpE,MAAM,UAAU,GAAG,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QAEvF,MAAM,QAAQ,GAAG,UAAU,CAAC;YAC1B,KAAK,EAAE,eAAe;YACtB,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,MAAM,CAAC,QAAQ;YACrB,MAAM,EAAE,gCAAgC;YACxC,8EAA8E;YAC9E,GAAG,CAAC,OAAO,EAAE,QAAQ,KAAK,IAAI;gBAC5B,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,eAAwB,EAAE,EAAE;gBAC1E,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,UAAU,KAAK,SAAS;gBAC1B,CAAC,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE;gBACxB,CAAC,CAAC,SAAS,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,CAAC;oBAC1D,CAAC,CAAC,EAAE,MAAM,EAAE,eAAwB,EAAE;oBACtC,CAAC,CAAC,EAAE,CAAC;YACT,GAAG,CAAC,SAAS,GAAG,CAAC,IAAI,cAAc,KAAK,SAAS;gBAC/C,CAAC,CAAC,EAAE,WAAW,EAAE,WAAW,SAAS,EAAE,EAAE;gBACzC,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,UAAU,GAAG,CAAC;gBAChB,CAAC,CAAC;oBACE,IAAI,EAAE,GAAG,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,mGAAmG;iBACxI;gBACH,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;QAEH,aAAa,CAAC,QAAQ,EAAE,GAAG,EAAE;YAC3B,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,SAAS;YACT,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS;YACT,MAAM;YACN,QAAQ;YACR,SAAS;YACT,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;YAC3D,QAAQ;SACT,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/utils/harvest-lock.d.ts

@@ -0,0 +1,3 @@
+export declare function withChannelHarvestLock<T>(channelId: string, work: () => Promise<T>): Promise<T>;
+export declare function withVideoHarvestLock<T>(videoId: string, work: () => Promise<T>): Promise<T>;
+//# sourceMappingURL=harvest-lock.d.ts.map

package/dist/utils/harvest-lock.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harvest-lock.d.ts","sourceRoot":"","sources":["../../src/utils/harvest-lock.ts"],"names":[],"mappings":"AAsBA,wBAAsB,sBAAsB,CAAC,CAAC,EAC5C,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACrB,OAAO,CAAC,CAAC,CAAC,CAKZ;AAED,wBAAsB,oBAAoB,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAKjG"}

package/dist/utils/harvest-lock.js

@@ -0,0 +1,26 @@
+import { YouTubeError } from './errors.js';
+import { ensurePrivateDir } from './paths.js';
+import { withFileLock } from './file-lock.js';
+import { channelHarvestLockPath, storeDir, videoHarvestLockPath } from './store-paths.js';
+/**
+ * Exclusive access to one harvest target.
+ *
+ * WAL already keeps concurrent writes consistent, so this is not about the
+ * rows. It is about the requests: two harvests of the same channel would each
+ * spawn yt-dlp and double the rate at the one resource that throttles us.
+ */
+function alreadyRunning(what, id, existing) {
+    const owner = existing === undefined ? '' : `Started at ${existing.startedAt} by process ${existing.pid}. `;
+    return new YouTubeError('INVALID_INPUT', `A harvest of ${what} ${id} is already running.`, {
+        nextStep: `${owner}Wait for it to finish, then call the tool again to continue where it stopped.`,
+    });
+}
+export async function withChannelHarvestLock(channelId, work) {
+    await ensurePrivateDir(storeDir());
+    return withFileLock(channelHarvestLockPath(channelId), work, (existing) => alreadyRunning('channel', channelId, existing));
+}
+export async function withVideoHarvestLock(videoId, work) {
+    await ensurePrivateDir(storeDir());
+    return withFileLock(videoHarvestLockPath(videoId), work, (existing) => alreadyRunning('video', videoId, existing));
+}
+//# sourceMappingURL=harvest-lock.js.map

package/dist/utils/harvest-lock.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harvest-lock.js","sourceRoot":"","sources":["../../src/utils/harvest-lock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAmB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,sBAAsB,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE1F;;;;;;GAMG;AAEH,SAAS,cAAc,CAAC,IAAY,EAAE,EAAU,EAAE,QAAgC;IAChF,MAAM,KAAK,GACT,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,QAAQ,CAAC,SAAS,eAAe,QAAQ,CAAC,GAAG,IAAI,CAAC;IAEhG,OAAO,IAAI,YAAY,CAAC,eAAe,EAAE,gBAAgB,IAAI,IAAI,EAAE,sBAAsB,EAAE;QACzF,QAAQ,EAAE,GAAG,KAAK,+EAA+E;KAClG,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,SAAiB,EACjB,IAAsB;IAEtB,MAAM,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnC,OAAO,YAAY,CAAC,sBAAsB,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,EAAE,CACxE,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,CAAC,CAC/C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAI,OAAe,EAAE,IAAsB;IACnF,MAAM,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnC,OAAO,YAAY,CAAC,oBAAoB,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,EAAE,CACpE,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAC3C,CAAC;AACJ,CAAC"}

package/dist/utils/harvest-receipts.d.ts

@@ -0,0 +1,14 @@
+import type { DatabaseSync } from 'node:sqlite';
+import { type Coverage } from '../harvest-schemas.js';
+export declare function saveReceipt(database: DatabaseSync, coverage: Coverage, options?: {
+    startedAt?: number;
+    errorCode?: string;
+}): void;
+export declare function readReceipt(database: DatabaseSync, scope: Coverage['scope'], targetId: string): Coverage | undefined;
+export declare function listReceipts(database: DatabaseSync, options?: {
+    scope?: Coverage['scope'];
+    limit?: number;
+    offset?: number;
+}): Coverage[];
+export declare function countReceipts(database: DatabaseSync, scope?: Coverage['scope']): number;
+//# sourceMappingURL=harvest-receipts.d.ts.map

package/dist/utils/harvest-receipts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harvest-receipts.d.ts","sourceRoot":"","sources":["../../src/utils/harvest-receipts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAsC,KAAK,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAgE1F,wBAAgB,WAAW,CACzB,QAAQ,EAAE,YAAY,EACtB,QAAQ,EAAE,QAAQ,EAClB,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GACvD,IAAI,CAsCN;AAuBD,wBAAgB,WAAW,CACzB,QAAQ,EAAE,YAAY,EACtB,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,EACxB,QAAQ,EAAE,MAAM,GACf,QAAQ,GAAG,SAAS,CAQtB;AAED,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAC3E,QAAQ,EAAE,CAwBZ;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,MAAM,CAavF"}