youtube-knowledge-mcp 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/README.md +110 -12
  2. package/dist/brain-schemas.d.ts +502 -0
  3. package/dist/brain-schemas.d.ts.map +1 -0
  4. package/dist/brain-schemas.js +113 -0
  5. package/dist/brain-schemas.js.map +1 -0
  6. package/dist/http.d.ts.map +1 -1
  7. package/dist/http.js +20 -5
  8. package/dist/http.js.map +1 -1
  9. package/dist/index.d.ts.map +1 -1
  10. package/dist/index.js +86 -0
  11. package/dist/index.js.map +1 -1
  12. package/dist/prompts.d.ts.map +1 -1
  13. package/dist/prompts.js +42 -0
  14. package/dist/prompts.js.map +1 -1
  15. package/dist/resources.d.ts.map +1 -1
  16. package/dist/resources.js +66 -0
  17. package/dist/resources.js.map +1 -1
  18. package/dist/schemas.d.ts +67 -56
  19. package/dist/schemas.d.ts.map +1 -1
  20. package/dist/schemas.js +25 -4
  21. package/dist/schemas.js.map +1 -1
  22. package/dist/tools/ask-brain.d.ts +62 -0
  23. package/dist/tools/ask-brain.d.ts.map +1 -0
  24. package/dist/tools/ask-brain.js +70 -0
  25. package/dist/tools/ask-brain.js.map +1 -0
  26. package/dist/tools/batch.d.ts +10 -10
  27. package/dist/tools/batch.js +2 -2
  28. package/dist/tools/batch.js.map +1 -1
  29. package/dist/tools/brain-info.d.ts +180 -0
  30. package/dist/tools/brain-info.d.ts.map +1 -0
  31. package/dist/tools/brain-info.js +108 -0
  32. package/dist/tools/brain-info.js.map +1 -0
  33. package/dist/tools/build-brain.d.ts +120 -0
  34. package/dist/tools/build-brain.d.ts.map +1 -0
  35. package/dist/tools/build-brain.js +123 -0
  36. package/dist/tools/build-brain.js.map +1 -0
  37. package/dist/tools/clips.d.ts +4 -4
  38. package/dist/tools/fetch-videos.d.ts +8 -8
  39. package/dist/tools/get-chapters.d.ts +2 -2
  40. package/dist/tools/get-comments.d.ts +2 -2
  41. package/dist/tools/get-transcript.d.ts +2 -2
  42. package/dist/tools/library.d.ts +9 -7
  43. package/dist/tools/library.d.ts.map +1 -1
  44. package/dist/tools/library.js +10 -4
  45. package/dist/tools/library.js.map +1 -1
  46. package/dist/tools/list-library.d.ts +4 -4
  47. package/dist/tools/manage-brain.d.ts +39 -0
  48. package/dist/tools/manage-brain.d.ts.map +1 -0
  49. package/dist/tools/manage-brain.js +53 -0
  50. package/dist/tools/manage-brain.js.map +1 -0
  51. package/dist/tools/search-channels.d.ts +4 -4
  52. package/dist/tools/search-transcript.d.ts +2 -2
  53. package/dist/tools/search-videos.d.ts +8 -8
  54. package/dist/utils/brain-build.d.ts +40 -0
  55. package/dist/utils/brain-build.d.ts.map +1 -0
  56. package/dist/utils/brain-build.js +211 -0
  57. package/dist/utils/brain-build.js.map +1 -0
  58. package/dist/utils/brain-chunks.d.ts +33 -0
  59. package/dist/utils/brain-chunks.d.ts.map +1 -0
  60. package/dist/utils/brain-chunks.js +87 -0
  61. package/dist/utils/brain-chunks.js.map +1 -0
  62. package/dist/utils/brain-index.d.ts +27 -0
  63. package/dist/utils/brain-index.d.ts.map +1 -0
  64. package/dist/utils/brain-index.js +100 -0
  65. package/dist/utils/brain-index.js.map +1 -0
  66. package/dist/utils/brain-ingest.d.ts +57 -0
  67. package/dist/utils/brain-ingest.d.ts.map +1 -0
  68. package/dist/utils/brain-ingest.js +127 -0
  69. package/dist/utils/brain-ingest.js.map +1 -0
  70. package/dist/utils/brain-lock.d.ts +20 -0
  71. package/dist/utils/brain-lock.d.ts.map +1 -0
  72. package/dist/utils/brain-lock.js +93 -0
  73. package/dist/utils/brain-lock.js.map +1 -0
  74. package/dist/utils/brain-lookup.d.ts +3 -0
  75. package/dist/utils/brain-lookup.d.ts.map +1 -0
  76. package/dist/utils/brain-lookup.js +62 -0
  77. package/dist/utils/brain-lookup.js.map +1 -0
  78. package/dist/utils/brain-paths.d.ts +19 -0
  79. package/dist/utils/brain-paths.d.ts.map +1 -0
  80. package/dist/utils/brain-paths.js +34 -0
  81. package/dist/utils/brain-paths.js.map +1 -0
  82. package/dist/utils/brain-phrases.d.ts +31 -0
  83. package/dist/utils/brain-phrases.d.ts.map +1 -0
  84. package/dist/utils/brain-phrases.js +116 -0
  85. package/dist/utils/brain-phrases.js.map +1 -0
  86. package/dist/utils/brain-stats.d.ts +25 -0
  87. package/dist/utils/brain-stats.d.ts.map +1 -0
  88. package/dist/utils/brain-stats.js +64 -0
  89. package/dist/utils/brain-stats.js.map +1 -0
  90. package/dist/utils/brain-storage.d.ts +28 -0
  91. package/dist/utils/brain-storage.d.ts.map +1 -0
  92. package/dist/utils/brain-storage.js +87 -0
  93. package/dist/utils/brain-storage.js.map +1 -0
  94. package/dist/utils/clips.js +4 -4
  95. package/dist/utils/clips.js.map +1 -1
  96. package/dist/utils/json-file.d.ts +36 -0
  97. package/dist/utils/json-file.d.ts.map +1 -0
  98. package/dist/utils/json-file.js +45 -0
  99. package/dist/utils/json-file.js.map +1 -0
  100. package/dist/utils/paths.d.ts +9 -0
  101. package/dist/utils/paths.d.ts.map +1 -0
  102. package/dist/utils/paths.js +13 -0
  103. package/dist/utils/paths.js.map +1 -0
  104. package/dist/utils/preflight.d.ts.map +1 -1
  105. package/dist/utils/preflight.js +14 -2
  106. package/dist/utils/preflight.js.map +1 -1
  107. package/dist/utils/search-index.d.ts +22 -7
  108. package/dist/utils/search-index.d.ts.map +1 -1
  109. package/dist/utils/search-index.js +17 -4
  110. package/dist/utils/search-index.js.map +1 -1
  111. package/dist/utils/storage.d.ts +14 -44
  112. package/dist/utils/storage.d.ts.map +1 -1
  113. package/dist/utils/storage.js +37 -80
  114. package/dist/utils/storage.js.map +1 -1
  115. package/dist/utils/transcript.d.ts +5 -4
  116. package/dist/utils/transcript.d.ts.map +1 -1
  117. package/dist/utils/transcript.js +3 -0
  118. package/dist/utils/transcript.js.map +1 -1
  119. package/dist/utils/validate.d.ts +1 -0
  120. package/dist/utils/validate.d.ts.map +1 -1
  121. package/dist/utils/validate.js +18 -0
  122. package/dist/utils/validate.js.map +1 -1
  123. package/dist/utils/youtube.d.ts +14 -0
  124. package/dist/utils/youtube.d.ts.map +1 -1
  125. package/dist/utils/youtube.js +19 -13
  126. package/dist/utils/youtube.js.map +1 -1
  127. package/dist/utils/ytdlp.d.ts.map +1 -1
  128. package/dist/utils/ytdlp.js +56 -6
  129. package/dist/utils/ytdlp.js.map +1 -1
  130. package/package.json +2 -2
@@ -0,0 +1,87 @@
1
+ import { segmentsToText } from './transcript.js';
2
+ /**
3
+ * Transcripts cut into retrievable passages.
4
+ *
5
+ * A brain is searched, not read: a whole transcript is too long to rank
6
+ * usefully and far too long to return, while a single caption cue is too short
7
+ * to carry a thought. A passage of a few sentences is what BM25 scores well and
8
+ * what a reader can check against the video in one click.
9
+ *
10
+ * Chunks do not overlap. Overlap is worth its cost with embeddings, where a
11
+ * concept split across a boundary can be missed entirely; BM25 matches terms,
12
+ * and a duplicated passage would only mean two results saying the same thing.
13
+ */
14
+ /** Where a passage stops being short enough to keep growing. */
15
+ export const CHUNK_TARGET_CHARS = 800;
16
+ /**
17
+ * Where it stops regardless. Speech without a pause for this long is rare, and
18
+ * an unbroken passage is worse than one broken in an awkward place.
19
+ */
20
+ export const CHUNK_MAX_CHARS = 1200;
21
+ /** A pause this long is a sentence or a thought ending. */
22
+ export const CHUNK_BREAK_GAP_SECONDS = 2;
23
+ export function chunkTranscript(options) {
24
+ const { videoId, title, segments, chapters = [] } = options;
25
+ const chapterStarts = chapters.map((chapter) => chapter.startTime).sort((a, b) => a - b);
26
+ const chunks = [];
27
+ let current = [];
28
+ let currentChars = 0;
29
+ let nextChapter = 0;
30
+ const flush = () => {
31
+ const chunk = toChunk(videoId, title, current, chunks.length);
32
+ if (chunk !== undefined)
33
+ chunks.push(chunk);
34
+ current = [];
35
+ currentChars = 0;
36
+ };
37
+ for (const segment of segments) {
38
+ // Consume every chapter that starts at or before this segment. More than
39
+ // one can land in the same gap when a chapter is shorter than a cue.
40
+ let startsChapter = false;
41
+ while (nextChapter < chapterStarts.length) {
42
+ const start = chapterStarts[nextChapter];
43
+ if (start === undefined || start > segment.start)
44
+ break;
45
+ nextChapter++;
46
+ startsChapter = true;
47
+ }
48
+ const previous = current[current.length - 1];
49
+ if (previous !== undefined && shouldBreak(previous, segment, currentChars, startsChapter)) {
50
+ flush();
51
+ }
52
+ current.push(segment);
53
+ // Plus one for the space this segment will be joined with.
54
+ currentChars += segment.text.length + 1;
55
+ }
56
+ flush();
57
+ return chunks;
58
+ }
59
+ function shouldBreak(previous, segment, currentChars, startsChapter) {
60
+ if (startsChapter)
61
+ return true;
62
+ if (currentChars >= CHUNK_MAX_CHARS)
63
+ return true;
64
+ return (currentChars >= CHUNK_TARGET_CHARS && segment.start - previous.end >= CHUNK_BREAK_GAP_SECONDS);
65
+ }
66
+ /** Undefined for an empty run, and for one that turned out to be only whitespace. */
67
+ function toChunk(videoId, title, segments, ordinal) {
68
+ const first = segments[0];
69
+ const last = segments[segments.length - 1];
70
+ if (first === undefined || last === undefined)
71
+ return undefined;
72
+ const text = segmentsToText(segments);
73
+ if (text === '')
74
+ return undefined;
75
+ return {
76
+ // Ordinal rather than start time: two passages can begin in the same second
77
+ // when cues are short, and a colliding id would silently drop one of them
78
+ // from the index. The start time is a field of its own.
79
+ id: `${videoId}:${ordinal}`,
80
+ videoId,
81
+ title,
82
+ startSeconds: first.start,
83
+ endSeconds: last.end,
84
+ text,
85
+ };
86
+ }
87
+ //# sourceMappingURL=brain-chunks.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brain-chunks.js","sourceRoot":"","sources":["../../src/utils/brain-chunks.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAA0B,MAAM,iBAAiB,CAAC;AAGzE;;;;;;;;;;;GAWG;AAEH,gEAAgE;AAChE,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAEtC;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAEpC,2DAA2D;AAC3D,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAUzC,MAAM,UAAU,eAAe,CAAC,OAA+B;IAC7D,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;IAE5D,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACzF,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,IAAI,OAAO,GAAwB,EAAE,CAAC;IACtC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,MAAM,KAAK,GAAG,GAAS,EAAE;QACvB,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9D,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5C,OAAO,GAAG,EAAE,CAAC;QACb,YAAY,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,yEAAyE;QACzE,qEAAqE;QACrE,IAAI,aAAa,GAAG,KAAK,CAAC;QAC1B,OAAO,WAAW,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC;YACzC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK;gBAAE,MAAM;YACxD,WAAW,EAAE,CAAC;YACd,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS,IAAI,WAAW,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,aAAa,CAAC,EAAE,CAAC;YAC1F,KAAK,EAAE,CAAC;QACV,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,2DAA2D;QAC3D,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,EAAE,CAAC;IACR,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAClB,QAA2B,EAC3B,OAA0B,EAC1B,YAAoB,EACpB,aAAsB;IAEtB,IAAI,aAAa;QAAE,OAAO,IAAI,CAAC;IAC/B,IAAI,YAAY,IAAI,eAAe;QAAE,OAAO,IAAI,CAAC;IAEjD,OAAO,CACL,YAAY,IAAI,kBAAkB,IAAI,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG,IAAI,uBAAuB,CAC9F,CAAC;AACJ,CAAC;AAED,qFAAqF;AACrF,SAAS,OAAO,CACd,OAAe,EACf,KAAa,EACb,QAA6B,EAC7B,OAAe;IAEf,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3C,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhE,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IAElC,OAAO;QACL,4EAA4E;QAC5E,0EAA0E;QAC1E,wDAAwD;QACxD,EAAE,EAAE,GAAG,OAAO,IAAI,OAAO,EAAE;QAC3B,OAAO;QACP,KAAK;QACL,YAAY,EAAE,KAAK,CAAC,KAAK;QACzB,UAAU,EAAE,IAAI,CAAC,GAAG;QACpB,IAAI;KACL,CAAC;AACJ,CAAC"}
@@ -0,0 +1,27 @@
1
+ import { type BrainChunk, type BrainPassage } from '../brain-schemas.js';
2
+ /**
3
+ * The searchable half of a brain: the passages, and a BM25 index over them.
4
+ *
5
+ * The passages are the only thing persisted. An index file alongside them would
6
+ * hold a second copy of every word — the serialised index carries the text it
7
+ * scored — and would then be free to disagree with them. It is built in memory
8
+ * on first use instead and kept until the passages change: measured at 1.5s for
9
+ * a corpus at this brain's 100,000-passage ceiling, once, against a class of
10
+ * bugs forever. A search over that corpus then costs 73ms.
11
+ *
12
+ * Retrieval is the only way a corpus reaches a model. That ceiling is 81MB of
13
+ * transcript, so there is no "give me everything" path here: `readChunks`
14
+ * exists for statistics and rebuilding, not for answering.
15
+ */
16
+ export declare const BRAIN_CHUNK_FILE_VERSION = 1;
17
+ export declare function forgetBrainCorpus(channelId: string): void;
18
+ export declare function readChunks(channelId: string): Promise<BrainChunk[]>;
19
+ export declare function writeChunks(channelId: string, chunks: BrainChunk[]): Promise<void>;
20
+ export interface BrainSearchResults {
21
+ passages: BrainPassage[];
22
+ /** Passages matching the query, not passages returned. */
23
+ total: number;
24
+ }
25
+ /** Passages ranked against the query, each with the link that opens it. */
26
+ export declare function searchBrain(channelId: string, query: string, limit: number, offset?: number): Promise<BrainSearchResults>;
27
+ //# sourceMappingURL=brain-index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brain-index.d.ts","sourceRoot":"","sources":["../../src/utils/brain-index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAwB,KAAK,UAAU,EAAE,KAAK,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAM/F;;;;;;;;;;;;;GAaG;AAEH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAoB1C,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAEzD;AAED,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAGzE;AAED,wBAAsB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CASxF;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,0DAA0D;IAC1D,KAAK,EAAE,MAAM,CAAC;CACf;AAED,2EAA2E;AAC3E,wBAAsB,WAAW,CAC/B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,MAAM,SAAI,GACT,OAAO,CAAC,kBAAkB,CAAC,CAuB7B"}
@@ -0,0 +1,100 @@
1
+ import { existsSync } from 'node:fs';
2
+ import { mkdir, stat } from 'node:fs/promises';
3
+ import { brainChunkFileSchema } from '../brain-schemas.js';
4
+ import { brainDir, chunksPath } from './brain-paths.js';
5
+ import { readJsonFile, writeJsonAtomic } from './json-file.js';
6
+ import { SearchIndex } from './search-index.js';
7
+ import { deepLink, formatTimestamp } from './transcript.js';
8
+ /**
9
+ * The searchable half of a brain: the passages, and a BM25 index over them.
10
+ *
11
+ * The passages are the only thing persisted. An index file alongside them would
12
+ * hold a second copy of every word — the serialised index carries the text it
13
+ * scored — and would then be free to disagree with them. It is built in memory
14
+ * on first use instead and kept until the passages change: measured at 1.5s for
15
+ * a corpus at this brain's 100,000-passage ceiling, once, against a class of
16
+ * bugs forever. A search over that corpus then costs 73ms.
17
+ *
18
+ * Retrieval is the only way a corpus reaches a model. That ceiling is 81MB of
19
+ * transcript, so there is no "give me everything" path here: `readChunks`
20
+ * exists for statistics and rebuilding, not for answering.
21
+ */
22
+ export const BRAIN_CHUNK_FILE_VERSION = 1;
23
+ /**
24
+ * Rebuilding the index on every query would re-read and re-tokenise the whole
25
+ * corpus each time. The passage file's modification time decides when the
26
+ * cached copy is wrong, so a build running in another process is picked up
27
+ * without any coordination between the two.
28
+ */
29
+ const cache = new Map();
30
+ export function forgetBrainCorpus(channelId) {
31
+ cache.delete(channelId);
32
+ }
33
+ export async function readChunks(channelId) {
34
+ const file = await readJsonFile(chunksPath(channelId), brainChunkFileSchema);
35
+ return file?.chunks ?? [];
36
+ }
37
+ export async function writeChunks(channelId, chunks) {
38
+ await mkdir(brainDir(channelId), { recursive: true });
39
+ // Not pretty-printed: indentation would be a fifth of a file nobody reads.
40
+ await writeJsonAtomic(chunksPath(channelId), { version: BRAIN_CHUNK_FILE_VERSION, chunks }, { pretty: false });
41
+ forgetBrainCorpus(channelId);
42
+ }
43
+ /** Passages ranked against the query, each with the link that opens it. */
44
+ export async function searchBrain(channelId, query, limit, offset = 0) {
45
+ const { index, byId } = await corpusOf(channelId);
46
+ const { hits, total } = index.search(query, limit, offset);
47
+ const passages = hits.flatMap((hit) => {
48
+ const chunk = byId.get(hit.id);
49
+ if (chunk === undefined)
50
+ return [];
51
+ return [
52
+ {
53
+ videoId: chunk.videoId,
54
+ title: chunk.title,
55
+ startSeconds: chunk.startSeconds,
56
+ startFormatted: formatTimestamp(chunk.startSeconds),
57
+ endSeconds: chunk.endSeconds,
58
+ score: hit.score,
59
+ text: chunk.text,
60
+ url: deepLink(chunk.videoId, chunk.startSeconds),
61
+ },
62
+ ];
63
+ });
64
+ return { passages, total };
65
+ }
66
+ async function corpusOf(channelId) {
67
+ const path = chunksPath(channelId);
68
+ if (!existsSync(path))
69
+ return emptyCorpus();
70
+ const { mtimeMs } = await stat(path);
71
+ const cached = cache.get(channelId);
72
+ if (cached?.mtimeMs === mtimeMs)
73
+ return cached.corpus;
74
+ const corpus = buildCorpus(await readChunks(channelId));
75
+ cache.set(channelId, { corpus, mtimeMs });
76
+ return corpus;
77
+ }
78
+ function emptyCorpus() {
79
+ return { index: new SearchIndex(), byId: new Map() };
80
+ }
81
+ /**
82
+ * The shared index is used unchanged: a passage already has an id, a video, a
83
+ * title and text, which is everything BM25 scores on. `kind` distinguishes
84
+ * these rows from the library's summaries and skills.
85
+ */
86
+ function buildCorpus(chunks) {
87
+ const corpus = emptyCorpus();
88
+ for (const chunk of chunks) {
89
+ corpus.index.add({
90
+ id: chunk.id,
91
+ videoId: chunk.videoId,
92
+ title: chunk.title,
93
+ kind: 'chunk',
94
+ text: chunk.text,
95
+ });
96
+ corpus.byId.set(chunk.id, chunk);
97
+ }
98
+ return corpus;
99
+ }
100
+ //# sourceMappingURL=brain-index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brain-index.js","sourceRoot":"","sources":["../../src/utils/brain-index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAsC,MAAM,qBAAqB,CAAC;AAC/F,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAE5D;;;;;;;;;;;;;GAaG;AAEH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AAY1C;;;;;GAKG;AACH,MAAM,KAAK,GAAG,IAAI,GAAG,EAAwB,CAAC;AAE9C,MAAM,UAAU,iBAAiB,CAAC,SAAiB;IACjD,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,SAAiB;IAChD,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAC7E,OAAO,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,SAAiB,EAAE,MAAoB;IACvE,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,2EAA2E;IAC3E,MAAM,eAAe,CACnB,UAAU,CAAC,SAAS,CAAC,EACrB,EAAE,OAAO,EAAE,wBAAwB,EAAE,MAAM,EAAE,EAC7C,EAAE,MAAM,EAAE,KAAK,EAAE,CAClB,CAAC;IACF,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAC/B,CAAC;AAQD,2EAA2E;AAC3E,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAiB,EACjB,KAAa,EACb,KAAa,EACb,MAAM,GAAG,CAAC;IAEV,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAE3D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/B,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QAEnC,OAAO;YACL;gBACE,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,cAAc,EAAE,eAAe,CAAC,KAAK,CAAC,YAAY,CAAC;gBACnD,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC;aACjD;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,SAAiB;IACvC,MAAM,IAAI,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IACnC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,EAAE,CAAC;IAE5C,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,MAAM,EAAE,OAAO,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC;IAEtD,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;IACxD,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW;IAClB,OAAO,EAAE,KAAK,EAAE,IAAI,WAAW,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,MAAoB;IACvC,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAE7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;YACf,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,KAAK,CAAC,IAAI;SACjB,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -0,0 +1,57 @@
1
+ import type { BrainChunk, BrainVideoState } from '../brain-schemas.js';
2
+ import { type VideoListItem } from './youtube.js';
3
+ /**
4
+ * Reading one video into passages.
5
+ *
6
+ * Nothing here throws for anything to do with the video itself. A channel with
7
+ * a members-only upload, a live stream still processing, or an episode nobody
8
+ * captioned is still a channel worth having a brain for; which videos those
9
+ * were belongs in the manifest, not in an exception that loses the other two
10
+ * hundred.
11
+ *
12
+ * Metadata is read before the transcript, in that order deliberately: it is the
13
+ * cheaper request, it carries the date and length the filters need, and a video
14
+ * ruled out by them then costs one request instead of two.
15
+ */
16
+ /**
17
+ * A transcript that chunks past this is not a transcript. Recording the video
18
+ * as failed keeps the brain buildable; letting it through does not.
19
+ *
20
+ * The brain-wide ceiling is what bounds memory, and it was chosen by measuring
21
+ * rather than by feel: a corpus of 100,000 passages is 81MB on disk, and the
22
+ * heaviest thing done to it — the phrase pass — peaked at 767MB of heap. Raising
23
+ * it means measuring again.
24
+ */
25
+ export declare const MAX_CHUNKS_PER_VIDEO = 2000;
26
+ export declare const MAX_CHUNKS_PER_BRAIN = 100000;
27
+ /** Recorded against a video the size guards refused, alongside yt-dlp's own codes. */
28
+ export declare const TOO_LARGE = "TOO_LARGE";
29
+ export declare const BRAIN_FULL = "BRAIN_FULL";
30
+ export interface VideoFilters {
31
+ /** Ignore anything published before this date (YYYY-MM-DD). */
32
+ since?: string;
33
+ /** Ignore anything shorter, to leave out shorts and clips. */
34
+ minDurationSeconds: number;
35
+ }
36
+ export interface IngestOptions extends VideoFilters {
37
+ language: string;
38
+ }
39
+ export interface IngestResult {
40
+ state: BrainVideoState;
41
+ chunks: BrainChunk[];
42
+ }
43
+ export declare function ingestVideo(video: VideoListItem, chunksSoFar: number, options: IngestOptions): Promise<IngestResult>;
44
+ /**
45
+ * Whether the filters rule this video out, or `undefined` when its recorded
46
+ * state cannot say.
47
+ *
48
+ * A build asks this of the manifest before deciding what to read, which is what
49
+ * lets a changed filter take effect without re-fetching anything: once a video
50
+ * has been looked at, its real date and length are on disk and the question is
51
+ * answerable from there. `undefined` is the honest answer for a video only ever
52
+ * seen in a flat listing — not a licence to assume it qualifies.
53
+ */
54
+ export declare function isExcluded(state: BrainVideoState, filters: VideoFilters): boolean | undefined;
55
+ export declare function pendingState(video: VideoListItem): BrainVideoState;
56
+ export declare function excluded(state: BrainVideoState): BrainVideoState;
57
+ //# sourceMappingURL=brain-ingest.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brain-ingest.d.ts","sourceRoot":"","sources":["../../src/utils/brain-ingest.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAKvE,OAAO,EAAkC,KAAK,aAAa,EAAE,MAAM,cAAc,CAAC;AAElF;;;;;;;;;;;;GAYG;AAEH;;;;;;;;GAQG;AACH,eAAO,MAAM,oBAAoB,OAAO,CAAC;AACzC,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,sFAAsF;AACtF,eAAO,MAAM,SAAS,cAAc,CAAC;AACrC,eAAO,MAAM,UAAU,eAAe,CAAC;AAEvC,MAAM,WAAW,YAAY;IAC3B,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,eAAe,CAAC;IACvB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB;AAED,wBAAsB,WAAW,CAC/B,KAAK,EAAE,aAAa,EACpB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,YAAY,CAAC,CAoDvB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,GAAG,SAAS,CAe7F;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,aAAa,GAAG,eAAe,CAWlE;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,eAAe,GAAG,eAAe,CAEhE"}
@@ -0,0 +1,127 @@
1
+ import { chunkTranscript } from './brain-chunks.js';
2
+ import { throwIfAborted } from './context.js';
3
+ import { asYouTubeError } from './errors.js';
4
+ import { countWords } from './transcript.js';
5
+ import { getTranscript, getVideoDetails } from './youtube.js';
6
+ /**
7
+ * Reading one video into passages.
8
+ *
9
+ * Nothing here throws for anything to do with the video itself. A channel with
10
+ * a members-only upload, a live stream still processing, or an episode nobody
11
+ * captioned is still a channel worth having a brain for; which videos those
12
+ * were belongs in the manifest, not in an exception that loses the other two
13
+ * hundred.
14
+ *
15
+ * Metadata is read before the transcript, in that order deliberately: it is the
16
+ * cheaper request, it carries the date and length the filters need, and a video
17
+ * ruled out by them then costs one request instead of two.
18
+ */
19
+ /**
20
+ * A transcript that chunks past this is not a transcript. Recording the video
21
+ * as failed keeps the brain buildable; letting it through does not.
22
+ *
23
+ * The brain-wide ceiling is what bounds memory, and it was chosen by measuring
24
+ * rather than by feel: a corpus of 100,000 passages is 81MB on disk, and the
25
+ * heaviest thing done to it — the phrase pass — peaked at 767MB of heap. Raising
26
+ * it means measuring again.
27
+ */
28
+ export const MAX_CHUNKS_PER_VIDEO = 2000;
29
+ export const MAX_CHUNKS_PER_BRAIN = 100_000;
30
+ /** Recorded against a video the size guards refused, alongside yt-dlp's own codes. */
31
+ export const TOO_LARGE = 'TOO_LARGE';
32
+ export const BRAIN_FULL = 'BRAIN_FULL';
33
+ export async function ingestVideo(video, chunksSoFar, options) {
34
+ const listed = pendingState(video);
35
+ try {
36
+ // The listing is flat, which is what makes enumerating a thousand videos one
37
+ // request rather than a thousand — and flat entries carry no date. These are
38
+ // the real values, and the chapters come back in the same request.
39
+ const details = await getVideoDetails(video.id);
40
+ const known = {
41
+ ...listed,
42
+ uploadDate: details.uploadDate,
43
+ durationSeconds: details.durationSeconds,
44
+ };
45
+ if (isExcluded(known, options) === true)
46
+ return { state: excluded(known), chunks: [] };
47
+ const transcript = await getTranscript(video.id, { language: options.language });
48
+ const chunks = chunkTranscript({
49
+ videoId: video.id,
50
+ title: video.title,
51
+ segments: transcript.segments,
52
+ chapters: details.chapters,
53
+ });
54
+ if (chunks.length > MAX_CHUNKS_PER_VIDEO)
55
+ return refused(known, TOO_LARGE);
56
+ if (chunksSoFar + chunks.length > MAX_CHUNKS_PER_BRAIN)
57
+ return refused(known, BRAIN_FULL);
58
+ return {
59
+ state: {
60
+ ...known,
61
+ state: 'indexed',
62
+ chunkCount: chunks.length,
63
+ wordCount: chunks.reduce((total, chunk) => total + countWords(chunk.text), 0),
64
+ },
65
+ chunks,
66
+ };
67
+ }
68
+ catch (error) {
69
+ // Cancellation is the client's decision, not this video's problem. Asking
70
+ // the signal rather than classifying the error catches it however it was
71
+ // raised — an aborted spawn, a rejected read, or the check between videos.
72
+ throwIfAborted();
73
+ const failure = asYouTubeError(error);
74
+ return {
75
+ state: failure.code === 'NO_CAPTIONS'
76
+ ? { ...listed, state: 'no-captions' }
77
+ : { ...listed, state: 'failed', error: failure.code },
78
+ chunks: [],
79
+ };
80
+ }
81
+ }
82
+ /**
83
+ * Whether the filters rule this video out, or `undefined` when its recorded
84
+ * state cannot say.
85
+ *
86
+ * A build asks this of the manifest before deciding what to read, which is what
87
+ * lets a changed filter take effect without re-fetching anything: once a video
88
+ * has been looked at, its real date and length are on disk and the question is
89
+ * answerable from there. `undefined` is the honest answer for a video only ever
90
+ * seen in a flat listing — not a licence to assume it qualifies.
91
+ */
92
+ export function isExcluded(state, filters) {
93
+ const { since, minDurationSeconds } = filters;
94
+ let undecided = false;
95
+ if (minDurationSeconds > 0) {
96
+ if (state.durationSeconds <= 0)
97
+ undecided = true;
98
+ else if (state.durationSeconds < minDurationSeconds)
99
+ return true;
100
+ }
101
+ if (since !== undefined) {
102
+ if (state.uploadDate === '')
103
+ undecided = true;
104
+ else if (state.uploadDate < since)
105
+ return true;
106
+ }
107
+ return undecided ? undefined : false;
108
+ }
109
+ export function pendingState(video) {
110
+ return {
111
+ videoId: video.id,
112
+ title: video.title,
113
+ url: video.url,
114
+ uploadDate: video.uploadDate,
115
+ durationSeconds: video.duration,
116
+ state: 'pending',
117
+ chunkCount: 0,
118
+ wordCount: 0,
119
+ };
120
+ }
121
+ export function excluded(state) {
122
+ return { ...state, state: 'excluded', chunkCount: 0, wordCount: 0 };
123
+ }
124
+ function refused(state, reason) {
125
+ return { state: { ...state, state: 'failed', error: reason }, chunks: [] };
126
+ }
127
+ //# sourceMappingURL=brain-ingest.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brain-ingest.js","sourceRoot":"","sources":["../../src/utils/brain-ingest.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAsB,MAAM,cAAc,CAAC;AAElF;;;;;;;;;;;;GAYG;AAEH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,CAAC;AACzC,MAAM,CAAC,MAAM,oBAAoB,GAAG,OAAO,CAAC;AAE5C,sFAAsF;AACtF,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAC;AACrC,MAAM,CAAC,MAAM,UAAU,GAAG,YAAY,CAAC;AAkBvC,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAoB,EACpB,WAAmB,EACnB,OAAsB;IAEtB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAEnC,IAAI,CAAC;QACH,6EAA6E;QAC7E,6EAA6E;QAC7E,mEAAmE;QACnE,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChD,MAAM,KAAK,GAAoB;YAC7B,GAAG,MAAM;YACT,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,eAAe,EAAE,OAAO,CAAC,eAAe;SACzC,CAAC;QAEF,IAAI,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI;YAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAEvF,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjF,MAAM,MAAM,GAAG,eAAe,CAAC;YAC7B,OAAO,EAAE,KAAK,CAAC,EAAE;YACjB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,QAAQ,EAAE,OAAO,CAAC,QAAQ;SAC3B,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,MAAM,GAAG,oBAAoB;YAAE,OAAO,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAC3E,IAAI,WAAW,GAAG,MAAM,CAAC,MAAM,GAAG,oBAAoB;YAAE,OAAO,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAE1F,OAAO;YACL,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,KAAK,EAAE,SAAS;gBAChB,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aAC9E;YACD,MAAM;SACP,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,0EAA0E;QAC1E,yEAAyE;QACzE,2EAA2E;QAC3E,cAAc,EAAE,CAAC;QAEjB,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QAEtC,OAAO;YACL,KAAK,EACH,OAAO,CAAC,IAAI,KAAK,aAAa;gBAC5B,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE;gBACrC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE;YACzD,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,UAAU,CAAC,KAAsB,EAAE,OAAqB;IACtE,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC;IAC9C,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,IAAI,kBAAkB,GAAG,CAAC,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC;aAC5C,IAAI,KAAK,CAAC,eAAe,GAAG,kBAAkB;YAAE,OAAO,IAAI,CAAC;IACnE,CAAC;IAED,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,IAAI,KAAK,CAAC,UAAU,KAAK,EAAE;YAAE,SAAS,GAAG,IAAI,CAAC;aACzC,IAAI,KAAK,CAAC,UAAU,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC;IACjD,CAAC;IAED,OAAO,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAoB;IAC/C,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,eAAe,EAAE,KAAK,CAAC,QAAQ;QAC/B,KAAK,EAAE,SAAS;QAChB,UAAU,EAAE,CAAC;QACb,SAAS,EAAE,CAAC;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAsB;IAC7C,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AACtE,CAAC;AAED,SAAS,OAAO,CAAC,KAAsB,EAAE,MAAc;IACrD,OAAO,EAAE,KAAK,EAAE,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;AAC7E,CAAC"}
@@ -0,0 +1,20 @@
1
+ /**
2
+ * Exclusive access to a brain while it is being built.
3
+ *
4
+ * Two builds of one channel would interleave their manifest checkpoints, and
5
+ * the later write would silently drop whatever the other had recorded since it
6
+ * last read. A lock file rather than an in-process mutex because a second
7
+ * server process — a second editor window, a stray `npm start` — is the case
8
+ * that actually happens.
9
+ */
10
+ /** Long enough that a slow build is never mistaken for a dead one. */
11
+ export declare const LOCK_STALE_MS: number;
12
+ /**
13
+ * Run `build` holding the lock.
14
+ *
15
+ * It is released on any exit, including cancellation. A process killed outright
16
+ * leaves one behind, which is why a lock is also broken when its owner is gone
17
+ * or when it is simply too old to be real.
18
+ */
19
+ export declare function withBuildLock<T>(channelId: string, build: () => Promise<T>): Promise<T>;
20
+ //# sourceMappingURL=brain-lock.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brain-lock.d.ts","sourceRoot":"","sources":["../../src/utils/brain-lock.ts"],"names":[],"mappings":"AAQA;;;;;;;;GAQG;AAEH,sEAAsE;AACtE,eAAO,MAAM,aAAa,QAAiB,CAAC;AAE5C;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAyB7F"}
@@ -0,0 +1,93 @@
1
+ import { rm, writeFile } from 'node:fs/promises';
2
+ import { brainLockSchema } from '../brain-schemas.js';
3
+ import { ensureBrainDir } from './brain-storage.js';
4
+ import { lockPath } from './brain-paths.js';
5
+ import { YouTubeError } from './errors.js';
6
+ import { readJsonFile } from './json-file.js';
7
+ import { isRecord } from './ytdlp.js';
8
+ /**
9
+ * Exclusive access to a brain while it is being built.
10
+ *
11
+ * Two builds of one channel would interleave their manifest checkpoints, and
12
+ * the later write would silently drop whatever the other had recorded since it
13
+ * last read. A lock file rather than an in-process mutex because a second
14
+ * server process — a second editor window, a stray `npm start` — is the case
15
+ * that actually happens.
16
+ */
17
+ /** Long enough that a slow build is never mistaken for a dead one. */
18
+ export const LOCK_STALE_MS = 30 * 60 * 1000;
19
+ /**
20
+ * Run `build` holding the lock.
21
+ *
22
+ * It is released on any exit, including cancellation. A process killed outright
23
+ * leaves one behind, which is why a lock is also broken when its owner is gone
24
+ * or when it is simply too old to be real.
25
+ */
26
+ export async function withBuildLock(channelId, build) {
27
+ await ensureBrainDir(channelId);
28
+ const path = lockPath(channelId);
29
+ if (!(await acquire(path))) {
30
+ const existing = await readJsonFile(path, brainLockSchema);
31
+ // An unreadable lock is one being written this instant, which is as held as
32
+ // it gets. Only a lock whose owner is demonstrably gone gets broken.
33
+ if (existing === undefined || !isStale(existing.pid, existing.startedAt)) {
34
+ throw alreadyRunning(channelId, existing);
35
+ }
36
+ await rm(path, { force: true });
37
+ // Losing this second race means another caller broke the same stale lock
38
+ // first, and is now the legitimate owner.
39
+ if (!(await acquire(path)))
40
+ throw alreadyRunning(channelId, undefined);
41
+ }
42
+ try {
43
+ return await build();
44
+ }
45
+ finally {
46
+ await rm(path, { force: true });
47
+ }
48
+ }
49
+ function alreadyRunning(channelId, existing) {
50
+ const owner = existing === undefined ? '' : `Started at ${existing.startedAt} by process ${existing.pid}. `;
51
+ return new YouTubeError('INVALID_INPUT', `A build for ${channelId} is already running.`, {
52
+ nextStep: `${owner}Wait for it to finish, then call build_brain again to continue where it stopped.`,
53
+ });
54
+ }
55
+ /**
56
+ * Create the lock file, or report that someone else already has.
57
+ *
58
+ * `wx` fails when the file exists, and that check and the creation are one
59
+ * operation in the kernel. Reading first and writing after is not a lock: two
60
+ * builds starting together both see nothing and both proceed.
61
+ */
62
+ async function acquire(path) {
63
+ const lock = { pid: process.pid, startedAt: new Date().toISOString() };
64
+ try {
65
+ await writeFile(path, JSON.stringify(lock), { encoding: 'utf-8', flag: 'wx' });
66
+ return true;
67
+ }
68
+ catch (error) {
69
+ if (isRecord(error) && error.code === 'EEXIST')
70
+ return false;
71
+ throw error;
72
+ }
73
+ }
74
+ function isStale(pid, startedAt) {
75
+ const started = Date.parse(startedAt);
76
+ if (Number.isNaN(started) || Date.now() - started > LOCK_STALE_MS)
77
+ return true;
78
+ return !isProcessAlive(pid);
79
+ }
80
+ function isProcessAlive(pid) {
81
+ try {
82
+ // Signal 0 performs the permission and existence checks without delivering
83
+ // anything.
84
+ process.kill(pid, 0);
85
+ return true;
86
+ }
87
+ catch (error) {
88
+ // EPERM means the process exists but belongs to another user, which for
89
+ // this purpose is very much alive.
90
+ return isRecord(error) && error.code === 'EPERM';
91
+ }
92
+ }
93
+ //# sourceMappingURL=brain-lock.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brain-lock.js","sourceRoot":"","sources":["../../src/utils/brain-lock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC;;;;;;;;GAQG;AAEH,sEAAsE;AACtE,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE5C;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAI,SAAiB,EAAE,KAAuB;IAC/E,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;IAEjC,IAAI,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;QAE3D,4EAA4E;QAC5E,qEAAqE;QACrE,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACzE,MAAM,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAEhC,yEAAyE;QACzE,0CAA0C;QAC1C,IAAI,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;YAAE,MAAM,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,CAAC;QACH,OAAO,MAAM,KAAK,EAAE,CAAC;IACvB,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACrB,SAAiB,EACjB,QAAwD;IAExD,MAAM,KAAK,GACT,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,QAAQ,CAAC,SAAS,eAAe,QAAQ,CAAC,GAAG,IAAI,CAAC;IAEhG,OAAO,IAAI,YAAY,CAAC,eAAe,EAAE,eAAe,SAAS,sBAAsB,EAAE;QACvF,QAAQ,EAAE,GAAG,KAAK,kFAAkF;KACrG,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,OAAO,CAAC,IAAY;IACjC,MAAM,IAAI,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IAEvE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/E,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC7D,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,GAAW,EAAE,SAAiB;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,GAAG,aAAa;QAAE,OAAO,IAAI,CAAC;IAE/E,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,cAAc,CAAC,GAAW;IACjC,IAAI,CAAC;QACH,2EAA2E;QAC3E,YAAY;QACZ,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wEAAwE;QACxE,mCAAmC;QACnC,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;IACnD,CAAC;AACH,CAAC"}
@@ -0,0 +1,3 @@
1
+ import type { BrainManifest } from '../brain-schemas.js';
2
+ export declare function resolveBrain(channel: string): Promise<BrainManifest>;
3
+ //# sourceMappingURL=brain-lookup.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brain-lookup.d.ts","sourceRoot":"","sources":["../../src/utils/brain-lookup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAmBzD,wBAAsB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAY1E"}
@@ -0,0 +1,62 @@
1
+ import { listManifests } from './brain-storage.js';
2
+ import { YouTubeError } from './errors.js';
3
+ /**
4
+ * Find a brain from whatever the caller called the channel.
5
+ *
6
+ * Every brain tool except `build_brain` answers from disk, so none of them
7
+ * should need the network to work out which brain is meant — asking YouTube to
8
+ * resolve a handle before every search would put a round trip in front of a
9
+ * lookup that is otherwise instant, and would make a brain unusable offline.
10
+ *
11
+ * The manifest already records the channel's id, handle, name and URL, which is
12
+ * everything a caller is likely to type.
13
+ */
14
+ const CHANNEL_ID_IN_TEXT = /UC[A-Za-z0-9_-]{22}/;
15
+ const HANDLE_IN_TEXT = /@[\w.-]+/;
16
+ export async function resolveBrain(channel) {
17
+ const manifests = await listManifests();
18
+ const wanted = candidates(channel);
19
+ const matches = manifests.filter((manifest) => keysOf(manifest).some((key) => wanted.has(key)));
20
+ const [first, second] = matches;
21
+ if (first === undefined)
22
+ throw noBrain(channel, manifests);
23
+ if (second !== undefined)
24
+ throw ambiguous(channel, matches);
25
+ return first;
26
+ }
27
+ function candidates(channel) {
28
+ const keys = new Set([normalize(channel)]);
29
+ const id = CHANNEL_ID_IN_TEXT.exec(channel)?.[0];
30
+ if (id !== undefined)
31
+ keys.add(normalize(id));
32
+ const handle = HANDLE_IN_TEXT.exec(channel)?.[0];
33
+ if (handle !== undefined) {
34
+ keys.add(normalize(handle));
35
+ keys.add(normalize(handle.slice(1)));
36
+ }
37
+ return keys;
38
+ }
39
+ function keysOf(manifest) {
40
+ const { channelId, handle, name, channelUrl } = manifest.channel;
41
+ return [channelId, handle, handle.replace(/^@/, ''), name, channelUrl]
42
+ .map(normalize)
43
+ .filter((key) => key !== '');
44
+ }
45
+ function normalize(value) {
46
+ return value.trim().toLowerCase().replace(/\/+$/, '');
47
+ }
48
+ function noBrain(channel, manifests) {
49
+ const built = manifests.map((manifest) => manifest.channel.handle || manifest.channel.name);
50
+ return new YouTubeError('NOT_FOUND', `No brain has been built for "${channel}".`, {
51
+ nextStep: built.length === 0
52
+ ? 'Call build_brain with the channel URL or handle to build one.'
53
+ : `Call build_brain to build it. Brains that exist: ${built.join(', ')}.`,
54
+ });
55
+ }
56
+ function ambiguous(channel, matches) {
57
+ const names = matches.map((manifest) => manifest.channel.channelId);
58
+ return new YouTubeError('INVALID_INPUT', `"${channel}" matches more than one brain.`, {
59
+ nextStep: `Use the channel id instead: ${names.join(', ')}.`,
60
+ });
61
+ }
62
+ //# sourceMappingURL=brain-lookup.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"brain-lookup.js","sourceRoot":"","sources":["../../src/utils/brain-lookup.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C;;;;;;;;;;GAUG;AAEH,MAAM,kBAAkB,GAAG,qBAAqB,CAAC;AACjD,MAAM,cAAc,GAAG,UAAU,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAe;IAChD,MAAM,SAAS,GAAG,MAAM,aAAa,EAAE,CAAC;IACxC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAEhG,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC;IAEhC,IAAI,KAAK,KAAK,SAAS;QAAE,MAAM,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAC3D,IAAI,MAAM,KAAK,SAAS;QAAE,MAAM,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAE5D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,OAAe;IACjC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAEnD,MAAM,EAAE,GAAG,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjD,IAAI,EAAE,KAAK,SAAS;QAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;IAE9C,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,MAAM,CAAC,QAAuB;IACrC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC;IAEjE,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC;SACnE,GAAG,CAAC,SAAS,CAAC;SACd,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,OAAO,CAAC,OAAe,EAAE,SAA0B;IAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5F,OAAO,IAAI,YAAY,CAAC,WAAW,EAAE,gCAAgC,OAAO,IAAI,EAAE;QAChF,QAAQ,EACN,KAAK,CAAC,MAAM,KAAK,CAAC;YAChB,CAAC,CAAC,+DAA+D;YACjE,CAAC,CAAC,oDAAoD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;KAC9E,CAAC,CAAC;AACL,CAAC;AAED,SAAS,SAAS,CAAC,OAAe,EAAE,OAAwB;IAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAEpE,OAAO,IAAI,YAAY,CAAC,eAAe,EAAE,IAAI,OAAO,gCAAgC,EAAE;QACpF,QAAQ,EAAE,+BAA+B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;KAC7D,CAAC,CAAC;AACL,CAAC"}