youtube-knowledge-mcp 2.0.2 → 2.2.0

package/dist/tools/search-videos.d.ts

@@ -20,24 +20,27 @@ export declare const searchVideosOutputSchema: {
         channel: z.ZodOptional<z.ZodString>;
         viewCount: z.ZodOptional<z.ZodNumber>;
         uploadDate: z.ZodOptional<z.ZodString>;
+        thumbnailUrl: z.ZodOptional<z.ZodString>;
     }, "strip", z.ZodTypeAny, {
-        url: string;
-        title: string;
         id: string;
-        durationFormatted: string;
+        title: string;
         durationSeconds: number;
+        durationFormatted: string;
+        url: string;
         channel?: string | undefined;
-        uploadDate?: string | undefined;
         viewCount?: number | undefined;
+        uploadDate?: string | undefined;
+        thumbnailUrl?: string | undefined;
     }, {
-        url: string;
-        title: string;
         id: string;
-        durationFormatted: string;
+        title: string;
         durationSeconds: number;
+        durationFormatted: string;
+        url: string;
         channel?: string | undefined;
-        uploadDate?: string | undefined;
         viewCount?: number | undefined;
+        uploadDate?: string | undefined;
+        thumbnailUrl?: string | undefined;
     }>, "many">;
 };
 export declare function searchVideosHandler({ query, limit, }: {

package/dist/tools/search-videos.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search-videos.d.ts","sourceRoot":"","sources":["../../src/tools/search-videos.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAEzE,eAAO,MAAM,kBAAkB;;;CAU9B,CAAC;AAEF,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAIpC,CAAC;AAEF,wBAAsB,mBAAmB,CAAC,EACxC,KAAK,EACL,KAAK,GACN,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,cAAc,CAAC,CA6B1B"}
+{"version":3,"file":"search-videos.d.ts","sourceRoot":"","sources":["../../src/tools/search-videos.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAEzE,eAAO,MAAM,kBAAkB;;;CAU9B,CAAC;AAEF,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAIpC,CAAC;AAEF,wBAAsB,mBAAmB,CAAC,EACxC,KAAK,EACL,KAAK,GACN,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,cAAc,CAAC,CA6B1B"}

package/dist/utils/batches.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Run `work` over the items, `size` at a time.
+ *
+ * Deliberately not a rolling window: a batch boundary is a natural place for a
+ * checkpoint and a cancellation check, and the limiter inside yt-dlp is what
+ * actually paces the network. Results come back in item order, whichever
+ * finished first.
+ *
+ * A batch always settles before a failure propagates. `Promise.all` would
+ * reject the moment one item did — on a cancelled request, before the items
+ * already in flight had finished — and a checkpoint taken in that instant
+ * would record none of them.
+ */
+export declare function inBatches<T, R>(items: T[], size: number, work: (item: T, index: number) => Promise<R>): Promise<R[]>;
+//# sourceMappingURL=batches.d.ts.map

package/dist/utils/batches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batches.d.ts","sourceRoot":"","sources":["../../src/utils/batches.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,wBAAsB,SAAS,CAAC,CAAC,EAAE,CAAC,EAClC,KAAK,EAAE,CAAC,EAAE,EACV,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GAC3C,OAAO,CAAC,CAAC,EAAE,CAAC,CAiBd"}

package/dist/utils/batches.js

@@ -0,0 +1,31 @@
+/**
+ * Run `work` over the items, `size` at a time.
+ *
+ * Deliberately not a rolling window: a batch boundary is a natural place for a
+ * checkpoint and a cancellation check, and the limiter inside yt-dlp is what
+ * actually paces the network. Results come back in item order, whichever
+ * finished first.
+ *
+ * A batch always settles before a failure propagates. `Promise.all` would
+ * reject the moment one item did — on a cancelled request, before the items
+ * already in flight had finished — and a checkpoint taken in that instant
+ * would record none of them.
+ */
+export async function inBatches(items, size, work) {
+    const results = [];
+    const width = Math.max(1, size);
+    for (let start = 0; start < items.length; start += width) {
+        const slice = items.slice(start, start + width);
+        const settled = await Promise.allSettled(slice.map((item, offset) => work(item, start + offset)));
+        for (const outcome of settled) {
+            if (outcome.status === 'rejected')
+                throw asError(outcome.reason);
+            results.push(outcome.value);
+        }
+    }
+    return results;
+}
+function asError(reason) {
+    return reason instanceof Error ? reason : new Error(String(reason));
+}
+//# sourceMappingURL=batches.js.map

package/dist/utils/batches.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batches.js","sourceRoot":"","sources":["../../src/utils/batches.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAU,EACV,IAAY,EACZ,IAA4C;IAE5C,MAAM,OAAO,GAAQ,EAAE,CAAC;IACxB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAEhC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,KAAK,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC,CACxD,CAAC;QAEF,KAAK,MAAM,OAAO,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU;gBAAE,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,OAAO,CAAC,MAAe;IAC9B,OAAO,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;AACtE,CAAC"}

package/dist/utils/brain-build.d.ts

@@ -0,0 +1,40 @@
+import type { BrainManifest } from '../brain-schemas.js';
+import { type VideoFilters } from './brain-ingest.js';
+import type { ChannelInfo, VideoListItem } from './youtube.js';
+/**
+ * Reading a channel into a brain.
+ *
+ * The shape of this is dictated by how long it takes. Several hundred videos is
+ * several hundred yt-dlp invocations, so a build is something that gets
+ * interrupted: cancelled by the client, throttled by YouTube, or killed along
+ * with the editor it was started from. Every choice below follows from that —
+ * per-video state rather than one "built" flag, checkpoints rather than a
+ * single write at the end, and a failed video costing only itself.
+ *
+ * Which is also why there is no separate refresh: a second build sees what it
+ * already has, skips it, and reads whatever is new.
+ */
+/** Frequent enough that little work is lost, rare enough not to dominate. */
+export declare const CHECKPOINT_EVERY_VIDEOS = 10;
+/** Past this many throttled videos in a row, YouTube means it. */
+export declare const RATE_LIMIT_TOLERANCE = 3;
+export interface BuildBrainOptions extends VideoFilters {
+    channel: ChannelInfo;
+    videos: VideoListItem[];
+    existing: BrainManifest | undefined;
+    /** Caption language to read. A brain is built from one language at a time. */
+    language: string;
+}
+export interface BuildBrainResult {
+    manifest: BrainManifest;
+    /** Videos the filters kept, whether or not this call had to read them. */
+    considered: number;
+    /** Videos read during this call, as opposed to carried over from an earlier one. */
+    processed: number;
+    skipped: number;
+    excluded: number;
+    stoppedEarly: boolean;
+    stopReason?: string;
+}
+export declare function buildBrain(options: BuildBrainOptions): Promise<BuildBrainResult>;
+//# sourceMappingURL=brain-build.d.ts.map

package/dist/utils/brain-build.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"brain-build.d.ts","sourceRoot":"","sources":["../../src/utils/brain-build.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,aAAa,EAAmB,MAAM,qBAAqB,CAAC;AAEtF,OAAO,EAKL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAM3B,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE/D;;;;;;;;;;;;GAYG;AAEH,6EAA6E;AAC7E,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAc1C,kEAAkE;AAClE,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAEtC,MAAM,WAAW,iBAAkB,SAAQ,YAAY;IACrD,OAAO,EAAE,WAAW,CAAC;IACrB,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,QAAQ,EAAE,aAAa,GAAG,SAAS,CAAC;IACpC,8EAA8E;IAC9E,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,aAAa,CAAC;IACxB,0EAA0E;IAC1E,UAAU,EAAE,MAAM,CAAC;IACnB,oFAAoF;IACpF,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA4EtF"}

package/dist/utils/brain-build.js

@@ -0,0 +1,200 @@
+import { readChunks, writeChunks } from './brain-index.js';
+import { excluded, ingestVideo, isExcluded, pendingState, } from './brain-ingest.js';
+import { computeStats } from './brain-stats.js';
+import { BRAIN_MANIFEST_VERSION, writeManifest } from './brain-storage.js';
+import { reportProgress, throwIfAborted } from './context.js';
+import { concurrencyState } from './ytdlp.js';
+import { inBatches } from './batches.js';
+/**
+ * Reading a channel into a brain.
+ *
+ * The shape of this is dictated by how long it takes. Several hundred videos is
+ * several hundred yt-dlp invocations, so a build is something that gets
+ * interrupted: cancelled by the client, throttled by YouTube, or killed along
+ * with the editor it was started from. Every choice below follows from that —
+ * per-video state rather than one "built" flag, checkpoints rather than a
+ * single write at the end, and a failed video costing only itself.
+ *
+ * Which is also why there is no separate refresh: a second build sees what it
+ * already has, skips it, and reads whatever is new.
+ */
+/** Frequent enough that little work is lost, rare enough not to dominate. */
+export const CHECKPOINT_EVERY_VIDEOS = 10;
+/**
+ * How many videos to read at once: whatever the yt-dlp limiter is actually set
+ * to, which `YOUTUBE_MCP_MAX_CONCURRENCY` can change.
+ *
+ * Read rather than copied. A second number that merely happened to match would
+ * stop matching the moment someone tuned the first, and would then either queue
+ * work behind a limit it could not see or batch below one it was allowed.
+ */
+function buildConcurrency() {
+    return concurrencyState().limit;
+}
+/** Past this many throttled videos in a row, YouTube means it. */
+export const RATE_LIMIT_TOLERANCE = 3;
+export async function buildBrain(options) {
+    const { channel, videos, existing, language, since, minDurationSeconds } = options;
+    const filters = { minDurationSeconds, ...(since === undefined ? {} : { since }) };
+    const createdAt = existing?.createdAt ?? new Date().toISOString();
+    const chunksByVideo = groupByVideo(await readChunks(channel.channelId));
+    const states = reconciled(existing, videos, chunksByVideo, filters);
+    const outstanding = videos.filter((video) => isOutstanding(states.get(video.id)));
+    /**
+     * `phrases` is the difference between a checkpoint and the end of a build:
+     * the phrase pass reads the whole corpus and takes seconds at the sizes a
+     * brain can reach, so it runs once, when there is a finished corpus to run it
+     * over. A checkpoint records what has been read, which is all it is for.
+     */
+    const save = async (phrases) => {
+        const passages = flatten(states, chunksByVideo);
+        const manifest = toManifest(channel, states, passages, createdAt, language, phrases);
+        await writeChunks(channel.channelId, passages);
+        await writeManifest(manifest);
+        return manifest;
+    };
+    let attempted = 0;
+    let processed = 0;
+    let consecutiveThrottles = 0;
+    let stopReason;
+    try {
+        await inBatches(outstanding, buildConcurrency(), async (video) => {
+            if (stopReason !== undefined)
+                return;
+            throwIfAborted();
+            const { state, chunks } = await ingestVideo(video, countChunks(chunksByVideo), {
+                language,
+                ...filters,
+            });
+            states.set(video.id, state);
+            if (chunks.length > 0)
+                chunksByVideo.set(video.id, chunks);
+            else
+                chunksByVideo.delete(video.id);
+            attempted++;
+            // A video the filters ruled out was looked at, not read.
+            if (state.state !== 'excluded')
+                processed++;
+            consecutiveThrottles = state.error === 'RATE_LIMITED' ? consecutiveThrottles + 1 : 0;
+            if (consecutiveThrottles >= RATE_LIMIT_TOLERANCE) {
+                stopReason = 'YouTube is rate limiting this client.';
+            }
+            reportProgress(attempted, outstanding.length, `Read ${attempted} of ${outstanding.length}`);
+            if (attempted % CHECKPOINT_EVERY_VIDEOS === 0)
+                await save(false);
+        });
+    }
+    catch (error) {
+        // A cancelled build must not throw away the videos read since the last
+        // checkpoint. Saving first is what makes "interrupt it and call it again"
+        // true rather than nearly true — and it is a checkpoint, not a finished
+        // corpus, so it does not pay for the phrase pass either.
+        await save(false);
+        throw error;
+    }
+    const manifest = await save(true);
+    const excludedCount = videos.filter((video) => states.get(video.id)?.state === 'excluded').length;
+    return {
+        manifest,
+        considered: videos.length - excludedCount,
+        processed,
+        skipped: videos.length - excludedCount - processed,
+        excluded: excludedCount,
+        stoppedEarly: stopReason !== undefined,
+        ...(stopReason === undefined ? {} : { stopReason }),
+    };
+}
+/**
+ * Per-video state, reconciled against the passages on disk and re-tested
+ * against the filters in force right now.
+ *
+ * Two corrections happen here, both for the same reason: what the manifest says
+ * was true when it was written, and neither the corpus nor the caller's filters
+ * are obliged to have stayed the same.
+ *
+ * The manifest and the passage file are two documents, and only one can be
+ * written first. A crash between them, a half-restored backup, or a
+ * `chunks.json` truncated by a full disk all end the same way — the manifest
+ * says a video was read and there is nothing to show for it. Left alone that
+ * brain is stranded for good: every build skips the video as already done while
+ * every search returns nothing. So the passages win, and a video the corpus
+ * cannot account for goes back to pending.
+ *
+ * A filter is re-applied rather than remembered. Narrowing one excludes videos
+ * already read; widening one brings excluded videos back. Both are decided from
+ * the dates and lengths already recorded, so changing your mind costs nothing
+ * until there is something new to fetch.
+ */
+function reconciled(existing, videos, chunksByVideo, filters) {
+    const states = new Map();
+    for (const [videoId, recorded] of Object.entries(existing?.videos ?? {})) {
+        const lost = recorded.state === 'indexed' && (chunksByVideo.get(videoId)?.length ?? 0) === 0;
+        const state = lost
+            ? { ...recorded, state: 'pending', chunkCount: 0, wordCount: 0 }
+            : recorded;
+        states.set(videoId, refiltered(state, filters));
+    }
+    for (const video of videos) {
+        // The listing's own length is real data, so a filter that can be decided
+        // from it is decided here — before spending a request to learn what the
+        // listing already said.
+        if (!states.has(video.id))
+            states.set(video.id, refiltered(pendingState(video), filters));
+    }
+    return states;
+}
+/**
+ * `undefined` — the filters cannot be decided from what is recorded — leaves the
+ * state alone, so the video is read and the question answered from the metadata
+ * that read fetches.
+ */
+function refiltered(state, filters) {
+    const ruledOut = isExcluded(state, filters);
+    if (ruledOut === true)
+        return excluded(state);
+    if (ruledOut === false && state.state === 'excluded') {
+        return { ...state, state: 'pending' };
+    }
+    return state;
+}
+/** Anything not already read: never attempted, or attempted and failed. */
+function isOutstanding(state) {
+    return state === undefined || state.state === 'pending' || state.state === 'failed';
+}
+function groupByVideo(chunks) {
+    const grouped = new Map();
+    for (const chunk of chunks) {
+        grouped.set(chunk.videoId, [...(grouped.get(chunk.videoId) ?? []), chunk]);
+    }
+    return grouped;
+}
+function countChunks(chunksByVideo) {
+    let total = 0;
+    for (const chunks of chunksByVideo.values())
+        total += chunks.length;
+    return total;
+}
+/**
+ * The corpus: every passage of every video the brain holds, in manifest order so
+ * the file changes as little as possible between builds.
+ *
+ * A video the filters exclude contributes nothing, which is what makes the
+ * filters mean something. Leaving its passages in would let `ask_brain` quote a
+ * video that `get_brain_info` says is not part of this brain, and no wording
+ * makes that defensible.
+ */
+function flatten(states, chunksByVideo) {
+    return [...states.entries()].flatMap(([videoId, state]) => state.state === 'excluded' ? [] : (chunksByVideo.get(videoId) ?? []));
+}
+function toManifest(channel, states, passages, createdAt, language, phrases) {
+    return {
+        version: BRAIN_MANIFEST_VERSION,
+        channel,
+        language,
+        createdAt,
+        updatedAt: new Date().toISOString(),
+        videos: Object.fromEntries(states),
+        stats: computeStats([...states.values()], passages, { phrases }),
+    };
+}
+//# sourceMappingURL=brain-build.js.map

package/dist/utils/brain-build.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"brain-build.js","sourceRoot":"","sources":["../../src/utils/brain-build.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EACL,QAAQ,EACR,WAAW,EACX,UAAU,EACV,YAAY,GAEb,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC;;;;;;;;;;;;GAYG;AAEH,6EAA6E;AAC7E,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAE1C;;;;;;;GAOG;AACH,SAAS,gBAAgB;IACvB,OAAO,gBAAgB,EAAE,CAAC,KAAK,CAAC;AAClC,CAAC;AAED,kEAAkE;AAClE,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAsBtC,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B;IACzD,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC;IACnF,MAAM,OAAO,GAAiB,EAAE,kBAAkB,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAChG,MAAM,SAAS,GAAG,QAAQ,EAAE,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAElE,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAElF;;;;;OAKG;IACH,MAAM,IAAI,GAAG,KAAK,EAAE,OAAgB,EAA0B,EAAE;QAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErF,MAAM,WAAW,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC/C,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC9B,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,UAA8B,CAAC;IAEnC,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,WAAW,EAAE,gBAAgB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC/D,IAAI,UAAU,KAAK,SAAS;gBAAE,OAAO;YACrC,cAAc,EAAE,CAAC;YAEjB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,aAAa,CAAC,EAAE;gBAC7E,QAAQ;gBACR,GAAG,OAAO;aACX,CAAC,CAAC;YAEH,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YAC5B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBAAE,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;;gBACtD,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAEpC,SAAS,EAAE,CAAC;YACZ,yDAAyD;YACzD,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU;gBAAE,SAAS,EAAE,CAAC;YAE5C,oBAAoB,GAAG,KAAK,CAAC,KAAK,KAAK,cAAc,CAAC,CAAC,CAAC,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrF,IAAI,oBAAoB,IAAI,oBAAoB,EAAE,CAAC;gBACjD,UAAU,GAAG,uCAAuC,CAAC;YACvD,CAAC;YAED,cAAc,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,QAAQ,SAAS,OAAO,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5F,IAAI,SAAS,GAAG,uBAAuB,KAAK,CAAC;gBAAE,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,uEAAuE;QACvE,0EAA0E;QAC1E,wEAAwE;QACxE,yDAAyD;QACzD,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,KAAK,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;IAElG,OAAO;QACL,QAAQ;QACR,UAAU,EAAE,MAAM,CAAC,MAAM,GAAG,aAAa;QACzC,SAAS;QACT,OAAO,EAAE,MAAM,CAAC,MAAM,GAAG,aAAa,GAAG,SAAS;QAClD,QAAQ,EAAE,aAAa;QACvB,YAAY,EAAE,UAAU,KAAK,SAAS;QACtC,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;KACpD,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,SAAS,UAAU,CACjB,QAAmC,EACnC,MAAuB,EACvB,aAAwC,EACxC,OAAqB;IAErB,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2B,CAAC;IAElD,KAAK,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC;QAC7F,MAAM,KAAK,GAAG,IAAI;YAChB,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,SAAkB,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;YACzE,CAAC,CAAC,QAAQ,CAAC;QAEb,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,yEAAyE;QACzE,wEAAwE;QACxE,wBAAwB;QACxB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAC5F,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,KAAsB,EAAE,OAAqB;IAC/D,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE5C,IAAI,QAAQ,KAAK,IAAI;QAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,QAAQ,KAAK,KAAK,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;QACrD,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,2EAA2E;AAC3E,SAAS,aAAa,CAAC,KAAkC;IACvD,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC;AACtF,CAAC;AAED,SAAS,YAAY,CAAC,MAAoB;IACxC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAEhD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,aAAwC;IAC3D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,MAAM,IAAI,aAAa,CAAC,MAAM,EAAE;QAAE,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC;IACpE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,OAAO,CACd,MAAoC,EACpC,aAAwC;IAExC,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,CACxD,KAAK,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CACrE,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CACjB,OAAoB,EACpB,MAAoC,EACpC,QAAsB,EACtB,SAAiB,EACjB,QAAgB,EAChB,OAAgB;IAEhB,OAAO;QACL,OAAO,EAAE,sBAAsB;QAC/B,OAAO;QACP,QAAQ;QACR,SAAS;QACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC;QAClC,KAAK,EAAE,YAAY,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC;KACjE,CAAC;AACJ,CAAC"}

package/dist/utils/brain-chunks.d.ts

@@ -0,0 +1,33 @@
+import type { BrainChunk } from '../brain-schemas.js';
+import { type TranscriptSegment } from './transcript.js';
+import type { Chapter } from './youtube.js';
+/**
+ * Transcripts cut into retrievable passages.
+ *
+ * A brain is searched, not read: a whole transcript is too long to rank
+ * usefully and far too long to return, while a single caption cue is too short
+ * to carry a thought. A passage of a few sentences is what BM25 scores well and
+ * what a reader can check against the video in one click.
+ *
+ * Chunks do not overlap. Overlap is worth its cost with embeddings, where a
+ * concept split across a boundary can be missed entirely; BM25 matches terms,
+ * and a duplicated passage would only mean two results saying the same thing.
+ */
+/** Where a passage stops being short enough to keep growing. */
+export declare const CHUNK_TARGET_CHARS = 800;
+/**
+ * Where it stops regardless. Speech without a pause for this long is rare, and
+ * an unbroken passage is worse than one broken in an awkward place.
+ */
+export declare const CHUNK_MAX_CHARS = 1200;
+/** A pause this long is a sentence or a thought ending. */
+export declare const CHUNK_BREAK_GAP_SECONDS = 2;
+export interface ChunkTranscriptOptions {
+    videoId: string;
+    title: string;
+    segments: TranscriptSegment[];
+    /** Chapter starts are always passage boundaries: a new chapter is a new subject. */
+    chapters?: Chapter[];
+}
+export declare function chunkTranscript(options: ChunkTranscriptOptions): BrainChunk[];
+//# sourceMappingURL=brain-chunks.d.ts.map

package/dist/utils/brain-chunks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"brain-chunks.d.ts","sourceRoot":"","sources":["../../src/utils/brain-chunks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAkB,KAAK,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE5C;;;;;;;;;;;GAWG;AAEH,gEAAgE;AAChE,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAEtC;;;GAGG;AACH,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,2DAA2D;AAC3D,eAAO,MAAM,uBAAuB,IAAI,CAAC;AAEzC,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,oFAAoF;IACpF,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;CACtB;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,UAAU,EAAE,CAwC7E"}

package/dist/utils/brain-chunks.js

@@ -0,0 +1,87 @@
+import { segmentsToText } from './transcript.js';
+/**
+ * Transcripts cut into retrievable passages.
+ *
+ * A brain is searched, not read: a whole transcript is too long to rank
+ * usefully and far too long to return, while a single caption cue is too short
+ * to carry a thought. A passage of a few sentences is what BM25 scores well and
+ * what a reader can check against the video in one click.
+ *
+ * Chunks do not overlap. Overlap is worth its cost with embeddings, where a
+ * concept split across a boundary can be missed entirely; BM25 matches terms,
+ * and a duplicated passage would only mean two results saying the same thing.
+ */
+/** Where a passage stops being short enough to keep growing. */
+export const CHUNK_TARGET_CHARS = 800;
+/**
+ * Where it stops regardless. Speech without a pause for this long is rare, and
+ * an unbroken passage is worse than one broken in an awkward place.
+ */
+export const CHUNK_MAX_CHARS = 1200;
+/** A pause this long is a sentence or a thought ending. */
+export const CHUNK_BREAK_GAP_SECONDS = 2;
+export function chunkTranscript(options) {
+    const { videoId, title, segments, chapters = [] } = options;
+    const chapterStarts = chapters.map((chapter) => chapter.startTime).sort((a, b) => a - b);
+    const chunks = [];
+    let current = [];
+    let currentChars = 0;
+    let nextChapter = 0;
+    const flush = () => {
+        const chunk = toChunk(videoId, title, current, chunks.length);
+        if (chunk !== undefined)
+            chunks.push(chunk);
+        current = [];
+        currentChars = 0;
+    };
+    for (const segment of segments) {
+        // Consume every chapter that starts at or before this segment. More than
+        // one can land in the same gap when a chapter is shorter than a cue.
+        let startsChapter = false;
+        while (nextChapter < chapterStarts.length) {
+            const start = chapterStarts[nextChapter];
+            if (start === undefined || start > segment.start)
+                break;
+            nextChapter++;
+            startsChapter = true;
+        }
+        const previous = current[current.length - 1];
+        if (previous !== undefined && shouldBreak(previous, segment, currentChars, startsChapter)) {
+            flush();
+        }
+        current.push(segment);
+        // Plus one for the space this segment will be joined with.
+        currentChars += segment.text.length + 1;
+    }
+    flush();
+    return chunks;
+}
+function shouldBreak(previous, segment, currentChars, startsChapter) {
+    if (startsChapter)
+        return true;
+    if (currentChars >= CHUNK_MAX_CHARS)
+        return true;
+    return (currentChars >= CHUNK_TARGET_CHARS && segment.start - previous.end >= CHUNK_BREAK_GAP_SECONDS);
+}
+/** Undefined for an empty run, and for one that turned out to be only whitespace. */
+function toChunk(videoId, title, segments, ordinal) {
+    const first = segments[0];
+    const last = segments[segments.length - 1];
+    if (first === undefined || last === undefined)
+        return undefined;
+    const text = segmentsToText(segments);
+    if (text === '')
+        return undefined;
+    return {
+        // Ordinal rather than start time: two passages can begin in the same second
+        // when cues are short, and a colliding id would silently drop one of them
+        // from the index. The start time is a field of its own.
+        id: `${videoId}:${ordinal}`,
+        videoId,
+        title,
+        startSeconds: first.start,
+        endSeconds: last.end,
+        text,
+    };
+}
+//# sourceMappingURL=brain-chunks.js.map

package/dist/utils/brain-chunks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"brain-chunks.js","sourceRoot":"","sources":["../../src/utils/brain-chunks.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAA0B,MAAM,iBAAiB,CAAC;AAGzE;;;;;;;;;;;GAWG;AAEH,gEAAgE;AAChE,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAEtC;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAEpC,2DAA2D;AAC3D,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAUzC,MAAM,UAAU,eAAe,CAAC,OAA+B;IAC7D,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;IAE5D,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACzF,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,IAAI,OAAO,GAAwB,EAAE,CAAC;IACtC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,MAAM,KAAK,GAAG,GAAS,EAAE;QACvB,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9D,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5C,OAAO,GAAG,EAAE,CAAC;QACb,YAAY,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,yEAAyE;QACzE,qEAAqE;QACrE,IAAI,aAAa,GAAG,KAAK,CAAC;QAC1B,OAAO,WAAW,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC;YACzC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK;gBAAE,MAAM;YACxD,WAAW,EAAE,CAAC;YACd,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS,IAAI,WAAW,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,aAAa,CAAC,EAAE,CAAC;YAC1F,KAAK,EAAE,CAAC;QACV,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,2DAA2D;QAC3D,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,EAAE,CAAC;IACR,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAClB,QAA2B,EAC3B,OAA0B,EAC1B,YAAoB,EACpB,aAAsB;IAEtB,IAAI,aAAa;QAAE,OAAO,IAAI,CAAC;IAC/B,IAAI,YAAY,IAAI,eAAe;QAAE,OAAO,IAAI,CAAC;IAEjD,OAAO,CACL,YAAY,IAAI,kBAAkB,IAAI,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG,IAAI,uBAAuB,CAC9F,CAAC;AACJ,CAAC;AAED,qFAAqF;AACrF,SAAS,OAAO,CACd,OAAe,EACf,KAAa,EACb,QAA6B,EAC7B,OAAe;IAEf,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3C,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhE,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IAElC,OAAO;QACL,4EAA4E;QAC5E,0EAA0E;QAC1E,wDAAwD;QACxD,EAAE,EAAE,GAAG,OAAO,IAAI,OAAO,EAAE;QAC3B,OAAO;QACP,KAAK;QACL,YAAY,EAAE,KAAK,CAAC,KAAK;QACzB,UAAU,EAAE,IAAI,CAAC,GAAG;QACpB,IAAI;KACL,CAAC;AACJ,CAAC"}

package/dist/utils/brain-index.d.ts

@@ -0,0 +1,27 @@
+import { type BrainChunk, type BrainPassage } from '../brain-schemas.js';
+/**
+ * The searchable half of a brain: the passages, and a BM25 index over them.
+ *
+ * The passages are the only thing persisted. An index file alongside them would
+ * hold a second copy of every word — the serialised index carries the text it
+ * scored — and would then be free to disagree with them. It is built in memory
+ * on first use instead and kept until the passages change: measured at 1.5s for
+ * a corpus at this brain's 100,000-passage ceiling, once, against a class of
+ * bugs forever. A search over that corpus then costs 73ms.
+ *
+ * Retrieval is the only way a corpus reaches a model. That ceiling is 81MB of
+ * transcript, so there is no "give me everything" path here: `readChunks`
+ * exists for statistics and rebuilding, not for answering.
+ */
+export declare const BRAIN_CHUNK_FILE_VERSION = 1;
+export declare function forgetBrainCorpus(channelId: string): void;
+export declare function readChunks(channelId: string): Promise<BrainChunk[]>;
+export declare function writeChunks(channelId: string, chunks: BrainChunk[]): Promise<void>;
+export interface BrainSearchResults {
+    passages: BrainPassage[];
+    /** Passages matching the query, not passages returned. */
+    total: number;
+}
+/** Passages ranked against the query, each with the link that opens it. */
+export declare function searchBrain(channelId: string, query: string, limit: number, offset?: number): Promise<BrainSearchResults>;
+//# sourceMappingURL=brain-index.d.ts.map

package/dist/utils/brain-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"brain-index.d.ts","sourceRoot":"","sources":["../../src/utils/brain-index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAwB,KAAK,UAAU,EAAE,KAAK,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAO/F;;;;;;;;;;;;;GAaG;AAEH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAoB1C,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAEzD;AAED,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAGzE;AAED,wBAAsB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CASxF;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,0DAA0D;IAC1D,KAAK,EAAE,MAAM,CAAC;CACf;AAED,2EAA2E;AAC3E,wBAAsB,WAAW,CAC/B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,MAAM,SAAI,GACT,OAAO,CAAC,kBAAkB,CAAC,CAuB7B"}

package/dist/utils/brain-index.js

@@ -0,0 +1,101 @@
+import { existsSync } from 'node:fs';
+import { stat } from 'node:fs/promises';
+import { brainChunkFileSchema } from '../brain-schemas.js';
+import { brainDir, chunksPath } from './brain-paths.js';
+import { ensurePrivateDir } from './paths.js';
+import { readJsonFile, writeJsonAtomic } from './json-file.js';
+import { SearchIndex } from './search-index.js';
+import { deepLink, formatTimestamp } from './transcript.js';
+/**
+ * The searchable half of a brain: the passages, and a BM25 index over them.
+ *
+ * The passages are the only thing persisted. An index file alongside them would
+ * hold a second copy of every word — the serialised index carries the text it
+ * scored — and would then be free to disagree with them. It is built in memory
+ * on first use instead and kept until the passages change: measured at 1.5s for
+ * a corpus at this brain's 100,000-passage ceiling, once, against a class of
+ * bugs forever. A search over that corpus then costs 73ms.
+ *
+ * Retrieval is the only way a corpus reaches a model. That ceiling is 81MB of
+ * transcript, so there is no "give me everything" path here: `readChunks`
+ * exists for statistics and rebuilding, not for answering.
+ */
+export const BRAIN_CHUNK_FILE_VERSION = 1;
+/**
+ * Rebuilding the index on every query would re-read and re-tokenise the whole
+ * corpus each time. The passage file's modification time decides when the
+ * cached copy is wrong, so a build running in another process is picked up
+ * without any coordination between the two.
+ */
+const cache = new Map();
+export function forgetBrainCorpus(channelId) {
+    cache.delete(channelId);
+}
+export async function readChunks(channelId) {
+    const file = await readJsonFile(chunksPath(channelId), brainChunkFileSchema);
+    return file?.chunks ?? [];
+}
+export async function writeChunks(channelId, chunks) {
+    await ensurePrivateDir(brainDir(channelId));
+    // Not pretty-printed: indentation would be a fifth of a file nobody reads.
+    await writeJsonAtomic(chunksPath(channelId), { version: BRAIN_CHUNK_FILE_VERSION, chunks }, { pretty: false });
+    forgetBrainCorpus(channelId);
+}
+/** Passages ranked against the query, each with the link that opens it. */
+export async function searchBrain(channelId, query, limit, offset = 0) {
+    const { index, byId } = await corpusOf(channelId);
+    const { hits, total } = index.search(query, limit, offset);
+    const passages = hits.flatMap((hit) => {
+        const chunk = byId.get(hit.id);
+        if (chunk === undefined)
+            return [];
+        return [
+            {
+                videoId: chunk.videoId,
+                title: chunk.title,
+                startSeconds: chunk.startSeconds,
+                startFormatted: formatTimestamp(chunk.startSeconds),
+                endSeconds: chunk.endSeconds,
+                score: hit.score,
+                text: chunk.text,
+                url: deepLink(chunk.videoId, chunk.startSeconds),
+            },
+        ];
+    });
+    return { passages, total };
+}
+async function corpusOf(channelId) {
+    const path = chunksPath(channelId);
+    if (!existsSync(path))
+        return emptyCorpus();
+    const { mtimeMs } = await stat(path);
+    const cached = cache.get(channelId);
+    if (cached?.mtimeMs === mtimeMs)
+        return cached.corpus;
+    const corpus = buildCorpus(await readChunks(channelId));
+    cache.set(channelId, { corpus, mtimeMs });
+    return corpus;
+}
+function emptyCorpus() {
+    return { index: new SearchIndex(), byId: new Map() };
+}
+/**
+ * The shared index is used unchanged: a passage already has an id, a video, a
+ * title and text, which is everything BM25 scores on. `kind` distinguishes
+ * these rows from the library's summaries and skills.
+ */
+function buildCorpus(chunks) {
+    const corpus = emptyCorpus();
+    for (const chunk of chunks) {
+        corpus.index.add({
+            id: chunk.id,
+            videoId: chunk.videoId,
+            title: chunk.title,
+            kind: 'chunk',
+            text: chunk.text,
+        });
+        corpus.byId.set(chunk.id, chunk);
+    }
+    return corpus;
+}
+//# sourceMappingURL=brain-index.js.map

package/dist/utils/brain-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"brain-index.js","sourceRoot":"","sources":["../../src/utils/brain-index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,oBAAoB,EAAsC,MAAM,qBAAqB,CAAC;AAC/F,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAE5D;;;;;;;;;;;;;GAaG;AAEH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AAY1C;;;;;GAKG;AACH,MAAM,KAAK,GAAG,IAAI,GAAG,EAAwB,CAAC;AAE9C,MAAM,UAAU,iBAAiB,CAAC,SAAiB;IACjD,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,SAAiB;IAChD,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAC7E,OAAO,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,SAAiB,EAAE,MAAoB;IACvE,MAAM,gBAAgB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;IAC5C,2EAA2E;IAC3E,MAAM,eAAe,CACnB,UAAU,CAAC,SAAS,CAAC,EACrB,EAAE,OAAO,EAAE,wBAAwB,EAAE,MAAM,EAAE,EAC7C,EAAE,MAAM,EAAE,KAAK,EAAE,CAClB,CAAC;IACF,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAC/B,CAAC;AAQD,2EAA2E;AAC3E,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAiB,EACjB,KAAa,EACb,KAAa,EACb,MAAM,GAAG,CAAC;IAEV,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAE3D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/B,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QAEnC,OAAO;YACL;gBACE,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,cAAc,EAAE,eAAe,CAAC,KAAK,CAAC,YAAY,CAAC;gBACnD,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC;aACjD;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,SAAiB;IACvC,MAAM,IAAI,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IACnC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,EAAE,CAAC;IAE5C,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,MAAM,EAAE,OAAO,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC;IAEtD,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;IACxD,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW;IAClB,OAAO,EAAE,KAAK,EAAE,IAAI,WAAW,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,MAAoB;IACvC,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAE7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;YACf,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,KAAK,CAAC,IAAI;SACjB,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}