youtube-knowledge-mcp 2.0.2 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -4
- package/dist/brain-schemas.d.ts +502 -0
- package/dist/brain-schemas.d.ts.map +1 -0
- package/dist/brain-schemas.js +113 -0
- package/dist/brain-schemas.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +86 -0
- package/dist/index.js.map +1 -1
- package/dist/prompts.d.ts.map +1 -1
- package/dist/prompts.js +42 -0
- package/dist/prompts.js.map +1 -1
- package/dist/resources.d.ts.map +1 -1
- package/dist/resources.js +66 -0
- package/dist/resources.js.map +1 -1
- package/dist/schemas.d.ts +67 -56
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +25 -4
- package/dist/schemas.js.map +1 -1
- package/dist/tools/ask-brain.d.ts +62 -0
- package/dist/tools/ask-brain.d.ts.map +1 -0
- package/dist/tools/ask-brain.js +70 -0
- package/dist/tools/ask-brain.js.map +1 -0
- package/dist/tools/batch.d.ts +10 -10
- package/dist/tools/batch.js +2 -2
- package/dist/tools/batch.js.map +1 -1
- package/dist/tools/brain-info.d.ts +180 -0
- package/dist/tools/brain-info.d.ts.map +1 -0
- package/dist/tools/brain-info.js +108 -0
- package/dist/tools/brain-info.js.map +1 -0
- package/dist/tools/build-brain.d.ts +120 -0
- package/dist/tools/build-brain.d.ts.map +1 -0
- package/dist/tools/build-brain.js +123 -0
- package/dist/tools/build-brain.js.map +1 -0
- package/dist/tools/clips.d.ts +4 -4
- package/dist/tools/fetch-videos.d.ts +8 -8
- package/dist/tools/get-chapters.d.ts +2 -2
- package/dist/tools/get-comments.d.ts +2 -2
- package/dist/tools/get-transcript.d.ts +2 -2
- package/dist/tools/library.d.ts +9 -7
- package/dist/tools/library.d.ts.map +1 -1
- package/dist/tools/library.js +10 -4
- package/dist/tools/library.js.map +1 -1
- package/dist/tools/list-library.d.ts +4 -4
- package/dist/tools/manage-brain.d.ts +39 -0
- package/dist/tools/manage-brain.d.ts.map +1 -0
- package/dist/tools/manage-brain.js +53 -0
- package/dist/tools/manage-brain.js.map +1 -0
- package/dist/tools/search-channels.d.ts +4 -4
- package/dist/tools/search-transcript.d.ts +2 -2
- package/dist/tools/search-videos.d.ts +8 -8
- package/dist/utils/brain-build.d.ts +40 -0
- package/dist/utils/brain-build.d.ts.map +1 -0
- package/dist/utils/brain-build.js +211 -0
- package/dist/utils/brain-build.js.map +1 -0
- package/dist/utils/brain-chunks.d.ts +33 -0
- package/dist/utils/brain-chunks.d.ts.map +1 -0
- package/dist/utils/brain-chunks.js +87 -0
- package/dist/utils/brain-chunks.js.map +1 -0
- package/dist/utils/brain-index.d.ts +27 -0
- package/dist/utils/brain-index.d.ts.map +1 -0
- package/dist/utils/brain-index.js +100 -0
- package/dist/utils/brain-index.js.map +1 -0
- package/dist/utils/brain-ingest.d.ts +57 -0
- package/dist/utils/brain-ingest.d.ts.map +1 -0
- package/dist/utils/brain-ingest.js +127 -0
- package/dist/utils/brain-ingest.js.map +1 -0
- package/dist/utils/brain-lock.d.ts +20 -0
- package/dist/utils/brain-lock.d.ts.map +1 -0
- package/dist/utils/brain-lock.js +93 -0
- package/dist/utils/brain-lock.js.map +1 -0
- package/dist/utils/brain-lookup.d.ts +3 -0
- package/dist/utils/brain-lookup.d.ts.map +1 -0
- package/dist/utils/brain-lookup.js +62 -0
- package/dist/utils/brain-lookup.js.map +1 -0
- package/dist/utils/brain-paths.d.ts +19 -0
- package/dist/utils/brain-paths.d.ts.map +1 -0
- package/dist/utils/brain-paths.js +34 -0
- package/dist/utils/brain-paths.js.map +1 -0
- package/dist/utils/brain-phrases.d.ts +31 -0
- package/dist/utils/brain-phrases.d.ts.map +1 -0
- package/dist/utils/brain-phrases.js +116 -0
- package/dist/utils/brain-phrases.js.map +1 -0
- package/dist/utils/brain-stats.d.ts +25 -0
- package/dist/utils/brain-stats.d.ts.map +1 -0
- package/dist/utils/brain-stats.js +64 -0
- package/dist/utils/brain-stats.js.map +1 -0
- package/dist/utils/brain-storage.d.ts +28 -0
- package/dist/utils/brain-storage.d.ts.map +1 -0
- package/dist/utils/brain-storage.js +87 -0
- package/dist/utils/brain-storage.js.map +1 -0
- package/dist/utils/clips.js +4 -4
- package/dist/utils/clips.js.map +1 -1
- package/dist/utils/json-file.d.ts +36 -0
- package/dist/utils/json-file.d.ts.map +1 -0
- package/dist/utils/json-file.js +45 -0
- package/dist/utils/json-file.js.map +1 -0
- package/dist/utils/paths.d.ts +9 -0
- package/dist/utils/paths.d.ts.map +1 -0
- package/dist/utils/paths.js +13 -0
- package/dist/utils/paths.js.map +1 -0
- package/dist/utils/search-index.d.ts +22 -7
- package/dist/utils/search-index.d.ts.map +1 -1
- package/dist/utils/search-index.js +17 -4
- package/dist/utils/search-index.js.map +1 -1
- package/dist/utils/storage.d.ts +14 -44
- package/dist/utils/storage.d.ts.map +1 -1
- package/dist/utils/storage.js +37 -80
- package/dist/utils/storage.js.map +1 -1
- package/dist/utils/transcript.d.ts +5 -4
- package/dist/utils/transcript.d.ts.map +1 -1
- package/dist/utils/transcript.js +3 -0
- package/dist/utils/transcript.js.map +1 -1
- package/dist/utils/validate.d.ts +1 -0
- package/dist/utils/validate.d.ts.map +1 -1
- package/dist/utils/validate.js +18 -0
- package/dist/utils/validate.js.map +1 -1
- package/dist/utils/youtube.d.ts +14 -0
- package/dist/utils/youtube.d.ts.map +1 -1
- package/dist/utils/youtube.js +19 -13
- package/dist/utils/youtube.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Every file a brain is made of.
|
|
3
|
+
*
|
|
4
|
+
* Separate from the modules that read and write them so that the manifest layer
|
|
5
|
+
* and the index layer can each know where things live without importing each
|
|
6
|
+
* other.
|
|
7
|
+
*
|
|
8
|
+
* A brain is keyed on the channel id rather than the handle: handles get
|
|
9
|
+
* renamed, and a renamed handle would orphan the brain built under the old one.
|
|
10
|
+
* The id is also what reaches the filesystem, so it is validated here — once,
|
|
11
|
+
* at the only place that turns one into a path.
|
|
12
|
+
*/
|
|
13
|
+
export declare function brainsDir(): string;
|
|
14
|
+
export declare function brainDir(channelId: string): string;
|
|
15
|
+
export declare function manifestPath(channelId: string): string;
|
|
16
|
+
export declare function chunksPath(channelId: string): string;
|
|
17
|
+
export declare function profilePath(channelId: string): string;
|
|
18
|
+
export declare function lockPath(channelId: string): string;
|
|
19
|
+
//# sourceMappingURL=brain-paths.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brain-paths.d.ts","sourceRoot":"","sources":["../../src/utils/brain-paths.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;GAWG;AAEH,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAElD"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { dataDir } from './paths.js';
|
|
3
|
+
import { assertChannelId } from './validate.js';
|
|
4
|
+
/**
|
|
5
|
+
* Every file a brain is made of.
|
|
6
|
+
*
|
|
7
|
+
* Separate from the modules that read and write them so that the manifest layer
|
|
8
|
+
* and the index layer can each know where things live without importing each
|
|
9
|
+
* other.
|
|
10
|
+
*
|
|
11
|
+
* A brain is keyed on the channel id rather than the handle: handles get
|
|
12
|
+
* renamed, and a renamed handle would orphan the brain built under the old one.
|
|
13
|
+
* The id is also what reaches the filesystem, so it is validated here — once,
|
|
14
|
+
* at the only place that turns one into a path.
|
|
15
|
+
*/
|
|
16
|
+
export function brainsDir() {
|
|
17
|
+
return dataDir('brains');
|
|
18
|
+
}
|
|
19
|
+
export function brainDir(channelId) {
|
|
20
|
+
return join(brainsDir(), assertChannelId(channelId));
|
|
21
|
+
}
|
|
22
|
+
export function manifestPath(channelId) {
|
|
23
|
+
return join(brainDir(channelId), 'manifest.json');
|
|
24
|
+
}
|
|
25
|
+
export function chunksPath(channelId) {
|
|
26
|
+
return join(brainDir(channelId), 'chunks.json');
|
|
27
|
+
}
|
|
28
|
+
export function profilePath(channelId) {
|
|
29
|
+
return join(brainDir(channelId), 'profile.md');
|
|
30
|
+
}
|
|
31
|
+
export function lockPath(channelId) {
|
|
32
|
+
return join(brainDir(channelId), 'build.lock');
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=brain-paths.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brain-paths.js","sourceRoot":"","sources":["../../src/utils/brain-paths.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD;;;;;;;;;;;GAWG;AAEH,MAAM,UAAU,SAAS;IACvB,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,SAAiB;IACxC,OAAO,IAAI,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC5C,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,aAAa,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,SAAiB;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,YAAY,CAAC,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,SAAiB;IACxC,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,YAAY,CAAC,CAAC;AACjD,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { BrainChunk } from '../brain-schemas.js';
|
|
2
|
+
/**
|
|
3
|
+
* Phrases a creator repeats across videos.
|
|
4
|
+
*
|
|
5
|
+
* This is the one statistic here that is not a plain count, so it is worth
|
|
6
|
+
* saying what it does and does not claim. A phrase appearing in several
|
|
7
|
+
* different videos is repeated; that is a fact about the corpus. Whether it is
|
|
8
|
+
* a catchphrase, a verbal tic or a segment title is a judgement, and belongs to
|
|
9
|
+
* whoever reads the list.
|
|
10
|
+
*
|
|
11
|
+
* Counting every 3-to-6-word window at once would hold millions of keys in
|
|
12
|
+
* memory. Instead each length is counted only where the length below it
|
|
13
|
+
* survived — a phrase cannot recur more often than its own prefix does — so all
|
|
14
|
+
* but the first pass work on a small candidate set.
|
|
15
|
+
*
|
|
16
|
+
* It is still the most expensive thing done to a corpus: 12.6s over 100,000
|
|
17
|
+
* passages, which is why a build runs it once at the end rather than at every
|
|
18
|
+
* checkpoint.
|
|
19
|
+
*/
|
|
20
|
+
export declare const PHRASE_MIN_WORDS = 3;
|
|
21
|
+
export declare const PHRASE_MAX_WORDS = 6;
|
|
22
|
+
/** Repetition inside one video is a habit of that video, not of the channel. */
|
|
23
|
+
export declare const PHRASE_MIN_VIDEOS = 3;
|
|
24
|
+
export declare const PHRASE_LIMIT = 20;
|
|
25
|
+
export interface RecurringPhrase {
|
|
26
|
+
phrase: string;
|
|
27
|
+
videoCount: number;
|
|
28
|
+
occurrences: number;
|
|
29
|
+
}
|
|
30
|
+
export declare function findRecurringPhrases(chunks: BrainChunk[]): RecurringPhrase[];
|
|
31
|
+
//# sourceMappingURL=brain-phrases.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brain-phrases.d.ts","sourceRoot":"","sources":["../../src/utils/brain-phrases.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAGtD;;;;;;;;;;;;;;;;;GAiBG;AAEH,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAClC,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAClC,gFAAgF;AAChF,eAAO,MAAM,iBAAiB,IAAI,CAAC;AACnC,eAAO,MAAM,YAAY,KAAK,CAAC;AAE/B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAaD,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,eAAe,EAAE,CAuB5E"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { isStopWord } from './search-index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Phrases a creator repeats across videos.
|
|
4
|
+
*
|
|
5
|
+
* This is the one statistic here that is not a plain count, so it is worth
|
|
6
|
+
* saying what it does and does not claim. A phrase appearing in several
|
|
7
|
+
* different videos is repeated; that is a fact about the corpus. Whether it is
|
|
8
|
+
* a catchphrase, a verbal tic or a segment title is a judgement, and belongs to
|
|
9
|
+
* whoever reads the list.
|
|
10
|
+
*
|
|
11
|
+
* Counting every 3-to-6-word window at once would hold millions of keys in
|
|
12
|
+
* memory. Instead each length is counted only where the length below it
|
|
13
|
+
* survived — a phrase cannot recur more often than its own prefix does — so all
|
|
14
|
+
* but the first pass work on a small candidate set.
|
|
15
|
+
*
|
|
16
|
+
* It is still the most expensive thing done to a corpus: 12.6s over 100,000
|
|
17
|
+
* passages, which is why a build runs it once at the end rather than at every
|
|
18
|
+
* checkpoint.
|
|
19
|
+
*/
|
|
20
|
+
export const PHRASE_MIN_WORDS = 3;
|
|
21
|
+
export const PHRASE_MAX_WORDS = 6;
|
|
22
|
+
/** Repetition inside one video is a habit of that video, not of the channel. */
|
|
23
|
+
export const PHRASE_MIN_VIDEOS = 3;
|
|
24
|
+
export const PHRASE_LIMIT = 20;
|
|
25
|
+
export function findRecurringPhrases(chunks) {
|
|
26
|
+
const corpus = wordsByVideo(chunks);
|
|
27
|
+
const kept = [];
|
|
28
|
+
let candidates = new Set();
|
|
29
|
+
for (let length = PHRASE_MIN_WORDS; length <= PHRASE_MAX_WORDS; length++) {
|
|
30
|
+
const tallies = countPhrases(corpus, length, length === PHRASE_MIN_WORDS ? undefined : candidates);
|
|
31
|
+
candidates = new Set();
|
|
32
|
+
for (const [phrase, tally] of tallies) {
|
|
33
|
+
if (tally.videoCount < PHRASE_MIN_VIDEOS)
|
|
34
|
+
continue;
|
|
35
|
+
candidates.add(phrase);
|
|
36
|
+
kept.push({ phrase, videoCount: tally.videoCount, occurrences: tally.occurrences });
|
|
37
|
+
}
|
|
38
|
+
if (candidates.size === 0)
|
|
39
|
+
break;
|
|
40
|
+
}
|
|
41
|
+
return rank(kept);
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* The corpus as one word list per video, in a stable order.
|
|
45
|
+
*
|
|
46
|
+
* Grouping is what lets distinct-video counts be kept as a single field per
|
|
47
|
+
* phrase rather than a set of video ids per phrase, which for a large channel
|
|
48
|
+
* is the difference between tens of megabytes and hundreds.
|
|
49
|
+
*/
|
|
50
|
+
function wordsByVideo(chunks) {
|
|
51
|
+
const byVideo = new Map();
|
|
52
|
+
for (const chunk of chunks) {
|
|
53
|
+
const words = byVideo.get(chunk.videoId) ?? [];
|
|
54
|
+
words.push(...phraseWords(chunk.text));
|
|
55
|
+
byVideo.set(chunk.videoId, words);
|
|
56
|
+
}
|
|
57
|
+
return [...byVideo.values()];
|
|
58
|
+
}
|
|
59
|
+
/** Lowercased words, apostrophes kept because "don't" is one word. */
|
|
60
|
+
function phraseWords(text) {
|
|
61
|
+
return text
|
|
62
|
+
.toLowerCase()
|
|
63
|
+
.split(/[^\p{L}\p{N}']+/u)
|
|
64
|
+
.filter(Boolean);
|
|
65
|
+
}
|
|
66
|
+
function countPhrases(corpus, length, candidatePrefixes) {
|
|
67
|
+
const tallies = new Map();
|
|
68
|
+
for (const [videoIndex, words] of corpus.entries()) {
|
|
69
|
+
for (let start = 0; start + length <= words.length; start++) {
|
|
70
|
+
const window = words.slice(start, start + length);
|
|
71
|
+
// A longer phrase only recurs where its prefix already did. A phrase
|
|
72
|
+
// whose opening words are all filler is therefore never reached, which is
|
|
73
|
+
// the intended trade: it costs "and then i went on" to drop "and then i".
|
|
74
|
+
if (candidatePrefixes !== undefined) {
|
|
75
|
+
if (!candidatePrefixes.has(window.slice(0, length - 1).join(' ')))
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
else if (window.every(isStopWord)) {
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
const phrase = window.join(' ');
|
|
82
|
+
const tally = tallies.get(phrase);
|
|
83
|
+
if (tally === undefined) {
|
|
84
|
+
tallies.set(phrase, { occurrences: 1, videoCount: 1, lastVideoIndex: videoIndex });
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
tally.occurrences++;
|
|
88
|
+
if (tally.lastVideoIndex !== videoIndex) {
|
|
89
|
+
tally.videoCount++;
|
|
90
|
+
tally.lastVideoIndex = videoIndex;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return tallies;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Drop the truncations, then order by reach.
|
|
98
|
+
*
|
|
99
|
+
* A phrase contained in a longer kept phrase that is just as widespread says
|
|
100
|
+
* nothing the longer one does not — otherwise every long catchphrase drags in
|
|
101
|
+
* three truncations of itself. Subsumption has to consider the longest first,
|
|
102
|
+
* which is not the order worth reporting, so the two happen in separate passes.
|
|
103
|
+
*/
|
|
104
|
+
function rank(phrases) {
|
|
105
|
+
const longestFirst = [...phrases].sort((a, b) => b.phrase.split(' ').length - a.phrase.split(' ').length || b.videoCount - a.videoCount);
|
|
106
|
+
const distinct = [];
|
|
107
|
+
for (const candidate of longestFirst) {
|
|
108
|
+
const subsumed = distinct.some((existing) => existing.phrase.includes(candidate.phrase) && existing.videoCount >= candidate.videoCount);
|
|
109
|
+
if (!subsumed)
|
|
110
|
+
distinct.push(candidate);
|
|
111
|
+
}
|
|
112
|
+
return distinct
|
|
113
|
+
.sort((a, b) => b.videoCount - a.videoCount || b.occurrences - a.occurrences)
|
|
114
|
+
.slice(0, PHRASE_LIMIT);
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=brain-phrases.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brain-phrases.js","sourceRoot":"","sources":["../../src/utils/brain-phrases.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C;;;;;;;;;;;;;;;;;GAiBG;AAEH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAClC,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAClC,gFAAgF;AAChF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AACnC,MAAM,CAAC,MAAM,YAAY,GAAG,EAAE,CAAC;AAmB/B,MAAM,UAAU,oBAAoB,CAAC,MAAoB;IACvD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,IAAI,GAAsB,EAAE,CAAC;IACnC,IAAI,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,KAAK,IAAI,MAAM,GAAG,gBAAgB,EAAE,MAAM,IAAI,gBAAgB,EAAE,MAAM,EAAE,EAAE,CAAC;QACzE,MAAM,OAAO,GAAG,YAAY,CAC1B,MAAM,EACN,MAAM,EACN,MAAM,KAAK,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CACrD,CAAC;QAEF,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YACtC,IAAI,KAAK,CAAC,UAAU,GAAG,iBAAiB;gBAAE,SAAS;YACnD,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC;YAAE,MAAM;IACnC,CAAC;IAED,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,YAAY,CAAC,MAAoB;IACxC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE5C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/C,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/B,CAAC;AAED,sEAAsE;AACtE,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,IAAI;SACR,WAAW,EAAE;SACb,KAAK,CAAC,kBAAkB,CAAC;SACzB,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC;AAED,SAAS,YAAY,CACnB,MAAkB,EAClB,MAAc,EACd,iBAA0C;IAE1C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IAEzC,KAAK,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACnD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YAC5D,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC;YAElD,qEAAqE;YACrE,0EAA0E;YAC1E,0EAA0E;YAC1E,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;gBACpC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAAE,SAAS;YAC9E,CAAC;iBAAM,IAAI,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpC,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAElC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC,CAAC;gBACnF,SAAS;YACX,CAAC;YAED,KAAK,CAAC,WAAW,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;gBACxC,KAAK,CAAC,UAAU,EAAE,CAAC;gBACnB,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC;YACpC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,IAAI,CAAC,OAA0B;IACtC,MAAM,YAAY,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CACpC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CACjG,CAAC;IAEF,MAAM,QAAQ,GAAsB,EAAE,CAAC;IAEvC,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAC5B,CAAC,QAAQ,EAAE,EAAE,CACX,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,CAC5F,CAAC;QACF,IAAI,CAAC,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,QAAQ;SACZ,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC;SAC5E,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;AAC5B,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { BrainChunk, BrainStats, BrainVideoState } from '../brain-schemas.js';
|
|
2
|
+
/**
|
|
3
|
+
* What the server can say about a channel without inferring anything.
|
|
4
|
+
*
|
|
5
|
+
* Everything here is counted or measured from the corpus. Deliberately absent:
|
|
6
|
+
* named entities, because auto-generated captions arrive lowercase and
|
|
7
|
+
* unpunctuated so capitalisation heuristics produce noise; and distinctive
|
|
8
|
+
* vocabulary by TF-IDF, because there is no honest background corpus to compare
|
|
9
|
+
* a channel against and inventing one would produce confident nonsense.
|
|
10
|
+
*
|
|
11
|
+
* Anything the numbers cannot support belongs in the profile, which a model
|
|
12
|
+
* writes from passages it can cite.
|
|
13
|
+
*/
|
|
14
|
+
export interface StatsOptions {
|
|
15
|
+
/**
|
|
16
|
+
* Whether to run the phrase pass, which reads every word of the corpus. It is
|
|
17
|
+
* measured in seconds at the sizes a brain is allowed to reach — 12.6s over
|
|
18
|
+
* 100,000 passages on the machine this was written on — so a build that
|
|
19
|
+
* checkpoints every ten videos must not pay for it fifty times over. Omitted
|
|
20
|
+
* rather than guessed at when it has not been run.
|
|
21
|
+
*/
|
|
22
|
+
phrases: boolean;
|
|
23
|
+
}
|
|
24
|
+
export declare function computeStats(all: BrainVideoState[], chunks: BrainChunk[], options: StatsOptions): BrainStats;
|
|
25
|
+
//# sourceMappingURL=brain-stats.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brain-stats.d.ts","sourceRoot":"","sources":["../../src/utils/brain-stats.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGnF;;;;;;;;;;;GAWG;AAEH,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wBAAgB,YAAY,CAC1B,GAAG,EAAE,eAAe,EAAE,EACtB,MAAM,EAAE,UAAU,EAAE,EACpB,OAAO,EAAE,YAAY,GACpB,UAAU,CAyBZ"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { findRecurringPhrases } from './brain-phrases.js';
|
|
2
|
+
export function computeStats(all, chunks, options) {
|
|
3
|
+
// A video the filters ruled out is not part of this brain, so it does not get
|
|
4
|
+
// to move the brain's numbers — its date must not stretch the upload range,
|
|
5
|
+
// and its length must not enter the speaking rate. It is reported as its own
|
|
6
|
+
// count instead, because "we chose not to read 40 videos" is worth knowing.
|
|
7
|
+
const videos = all.filter((video) => video.state !== 'excluded');
|
|
8
|
+
const uploadDates = videos
|
|
9
|
+
.map((video) => video.uploadDate)
|
|
10
|
+
.filter(Boolean)
|
|
11
|
+
.sort();
|
|
12
|
+
return {
|
|
13
|
+
videoCount: videos.length,
|
|
14
|
+
excludedCount: all.length - videos.length,
|
|
15
|
+
indexedCount: countState(videos, 'indexed'),
|
|
16
|
+
noCaptionsCount: countState(videos, 'no-captions'),
|
|
17
|
+
failedCount: countState(videos, 'failed'),
|
|
18
|
+
pendingCount: countState(videos, 'pending'),
|
|
19
|
+
chunkCount: chunks.length,
|
|
20
|
+
totalWords: videos.reduce((total, video) => total + video.wordCount, 0),
|
|
21
|
+
medianWordsPerMinute: medianWordsPerMinute(videos),
|
|
22
|
+
...firstAndLast(uploadDates),
|
|
23
|
+
uploadsPerMonth: uploadsPerMonth(uploadDates),
|
|
24
|
+
...(options.phrases ? { recurringPhrases: findRecurringPhrases(chunks) } : {}),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
function countState(videos, state) {
|
|
28
|
+
return videos.filter((video) => video.state === state).length;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The median rather than the mean: one mis-timed transcript, or a ten-hour
|
|
32
|
+
* ambience upload with forty words in it, would drag an average anywhere.
|
|
33
|
+
*/
|
|
34
|
+
function medianWordsPerMinute(videos) {
|
|
35
|
+
const rates = videos
|
|
36
|
+
.filter((video) => video.durationSeconds > 0 && video.wordCount > 0)
|
|
37
|
+
.map((video) => (video.wordCount / video.durationSeconds) * 60)
|
|
38
|
+
.sort((a, b) => a - b);
|
|
39
|
+
if (rates.length === 0)
|
|
40
|
+
return 0;
|
|
41
|
+
const middle = Math.floor(rates.length / 2);
|
|
42
|
+
const median = rates.length % 2 === 0
|
|
43
|
+
? ((rates[middle - 1] ?? 0) + (rates[middle] ?? 0)) / 2
|
|
44
|
+
: (rates[middle] ?? 0);
|
|
45
|
+
return Math.round(median);
|
|
46
|
+
}
|
|
47
|
+
function firstAndLast(sortedDates) {
|
|
48
|
+
const first = sortedDates[0];
|
|
49
|
+
const last = sortedDates[sortedDates.length - 1];
|
|
50
|
+
return {
|
|
51
|
+
...(first === undefined ? {} : { firstUpload: first }),
|
|
52
|
+
...(last === undefined ? {} : { lastUpload: last }),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
/** Months with no uploads are absent rather than zero: this is a tally, not a series. */
|
|
56
|
+
function uploadsPerMonth(sortedDates) {
|
|
57
|
+
const months = new Map();
|
|
58
|
+
for (const date of sortedDates) {
|
|
59
|
+
const month = date.slice(0, 7);
|
|
60
|
+
months.set(month, (months.get(month) ?? 0) + 1);
|
|
61
|
+
}
|
|
62
|
+
return [...months.entries()].map(([month, videos]) => ({ month, videos }));
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=brain-stats.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brain-stats.js","sourceRoot":"","sources":["../../src/utils/brain-stats.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AA0B1D,MAAM,UAAU,YAAY,CAC1B,GAAsB,EACtB,MAAoB,EACpB,OAAqB;IAErB,8EAA8E;IAC9E,4EAA4E;IAC5E,6EAA6E;IAC7E,4EAA4E;IAC5E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC;IACjE,MAAM,WAAW,GAAG,MAAM;SACvB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC;SAChC,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,EAAE,CAAC;IAEV,OAAO;QACL,UAAU,EAAE,MAAM,CAAC,MAAM;QACzB,aAAa,EAAE,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;QACzC,YAAY,EAAE,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC;QAC3C,eAAe,EAAE,UAAU,CAAC,MAAM,EAAE,aAAa,CAAC;QAClD,WAAW,EAAE,UAAU,CAAC,MAAM,EAAE,QAAQ,CAAC;QACzC,YAAY,EAAE,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC;QAC3C,UAAU,EAAE,MAAM,CAAC,MAAM;QACzB,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QACvE,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,CAAC;QAClD,GAAG,YAAY,CAAC,WAAW,CAAC;QAC5B,eAAe,EAAE,eAAe,CAAC,WAAW,CAAC;QAC7C,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/E,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,MAAyB,EAAE,KAA+B;IAC5E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;AAChE,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,MAAyB;IACrD,MAAM,KAAK,GAAG,MAAM;SACjB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,GAAG,CAAC,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;SACnE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;SAC9D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5C,MAAM,MAAM,GACV,KAAK,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC;QACpB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;QACvD,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAE3B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,YAAY,CAAC,WAAqB;IACzC,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEjD,OAAO;QACL,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QACtD,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;KACpD,CAAC;AACJ,CAAC;AAED,yFAAyF;AACzF,SAAS,eAAe,CAAC,WAAqB;IAC5C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEzC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;AAC7E,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { type BrainManifest } from '../brain-schemas.js';
|
|
2
|
+
/**
|
|
3
|
+
* What a brain records about itself, and who is allowed to be writing it.
|
|
4
|
+
*
|
|
5
|
+
* The manifest is the brain's account of its own construction: which videos it
|
|
6
|
+
* covers, which it could not read, and where a build stopped. Everything that
|
|
7
|
+
* makes a build resumable is here rather than inferred from what happens to be
|
|
8
|
+
* on disk.
|
|
9
|
+
*/
|
|
10
|
+
export declare const BRAIN_MANIFEST_VERSION = 1;
|
|
11
|
+
export declare function ensureBrainDir(channelId: string): Promise<string>;
|
|
12
|
+
export declare function readManifest(channelId: string): Promise<BrainManifest | undefined>;
|
|
13
|
+
/**
|
|
14
|
+
* The manifest, or a typed failure naming the tool that would create it.
|
|
15
|
+
*
|
|
16
|
+
* Most callers want this rather than `readManifest`: "there is no brain yet" is
|
|
17
|
+
* the single most likely reason a brain tool cannot do its job, and the model
|
|
18
|
+
* needs to be told what to call, not handed an undefined.
|
|
19
|
+
*/
|
|
20
|
+
export declare function requireManifest(channelId: string): Promise<BrainManifest>;
|
|
21
|
+
export declare function writeManifest(manifest: BrainManifest): Promise<void>;
|
|
22
|
+
/** Every brain on disk, newest first. Unreadable directories are skipped. */
|
|
23
|
+
export declare function listManifests(): Promise<BrainManifest[]>;
|
|
24
|
+
export declare function deleteBrain(channelId: string): Promise<boolean>;
|
|
25
|
+
export declare function readProfile(channelId: string): Promise<string | undefined>;
|
|
26
|
+
export declare function writeProfile(channelId: string, content: string): Promise<string>;
|
|
27
|
+
export declare function hasProfile(channelId: string): boolean;
|
|
28
|
+
//# sourceMappingURL=brain-storage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brain-storage.d.ts","sourceRoot":"","sources":["../../src/utils/brain-storage.ts"],"names":[],"mappings":"AAGA,OAAO,EAAuB,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAM9E;;;;;;;GAOG;AAEH,eAAO,MAAM,sBAAsB,IAAI,CAAC;AAExC,wBAAsB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAIvE;AAID,wBAAsB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAExF;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAW/E;AAED,wBAAsB,aAAa,CAAC,QAAQ,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAG1E;AAED,6EAA6E;AAC7E,wBAAsB,aAAa,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC,CAmB9D;AAED,wBAAsB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAQrE;AAID,wBAAsB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAIhF;AAED,wBAAsB,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAKtF;AAED,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAErD"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { brainManifestSchema } from '../brain-schemas.js';
|
|
5
|
+
import { YouTubeError } from './errors.js';
|
|
6
|
+
import { readJsonFile, writeJsonAtomic } from './json-file.js';
|
|
7
|
+
import { brainDir, brainsDir, manifestPath, profilePath } from './brain-paths.js';
|
|
8
|
+
import { forgetBrainCorpus } from './brain-index.js';
|
|
9
|
+
/**
|
|
10
|
+
* What a brain records about itself, and who is allowed to be writing it.
|
|
11
|
+
*
|
|
12
|
+
* The manifest is the brain's account of its own construction: which videos it
|
|
13
|
+
* covers, which it could not read, and where a build stopped. Everything that
|
|
14
|
+
* makes a build resumable is here rather than inferred from what happens to be
|
|
15
|
+
* on disk.
|
|
16
|
+
*/
|
|
17
|
+
export const BRAIN_MANIFEST_VERSION = 1;
|
|
18
|
+
export async function ensureBrainDir(channelId) {
|
|
19
|
+
const directory = brainDir(channelId);
|
|
20
|
+
await mkdir(directory, { recursive: true });
|
|
21
|
+
return directory;
|
|
22
|
+
}
|
|
23
|
+
// -- Manifest ------------------------------------------------------------
|
|
24
|
+
export async function readManifest(channelId) {
|
|
25
|
+
return readJsonFile(manifestPath(channelId), brainManifestSchema);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* The manifest, or a typed failure naming the tool that would create it.
|
|
29
|
+
*
|
|
30
|
+
* Most callers want this rather than `readManifest`: "there is no brain yet" is
|
|
31
|
+
* the single most likely reason a brain tool cannot do its job, and the model
|
|
32
|
+
* needs to be told what to call, not handed an undefined.
|
|
33
|
+
*/
|
|
34
|
+
export async function requireManifest(channelId) {
|
|
35
|
+
const manifest = await readManifest(channelId);
|
|
36
|
+
if (manifest === undefined) {
|
|
37
|
+
throw new YouTubeError('NOT_FOUND', `No brain has been built for channel ${channelId}.`, {
|
|
38
|
+
nextStep: 'Call build_brain with the channel URL or handle, or list_brains to see what exists.',
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
return manifest;
|
|
42
|
+
}
|
|
43
|
+
export async function writeManifest(manifest) {
|
|
44
|
+
await ensureBrainDir(manifest.channel.channelId);
|
|
45
|
+
await writeJsonAtomic(manifestPath(manifest.channel.channelId), manifest);
|
|
46
|
+
}
|
|
47
|
+
/** Every brain on disk, newest first. Unreadable directories are skipped. */
|
|
48
|
+
export async function listManifests() {
|
|
49
|
+
if (!existsSync(brainsDir()))
|
|
50
|
+
return [];
|
|
51
|
+
const entries = await readdir(brainsDir(), { withFileTypes: true });
|
|
52
|
+
const manifests = [];
|
|
53
|
+
for (const entry of entries) {
|
|
54
|
+
if (!entry.isDirectory())
|
|
55
|
+
continue;
|
|
56
|
+
// A directory that is not a channel id was not written by us, and passing
|
|
57
|
+
// it to `brainDir` would throw rather than skip it.
|
|
58
|
+
const manifest = await readJsonFile(join(brainsDir(), entry.name, 'manifest.json'), brainManifestSchema);
|
|
59
|
+
if (manifest !== undefined)
|
|
60
|
+
manifests.push(manifest);
|
|
61
|
+
}
|
|
62
|
+
return manifests.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
|
63
|
+
}
|
|
64
|
+
export async function deleteBrain(channelId) {
|
|
65
|
+
const directory = brainDir(channelId);
|
|
66
|
+
const existed = existsSync(directory);
|
|
67
|
+
await rm(directory, { recursive: true, force: true });
|
|
68
|
+
forgetBrainCorpus(channelId);
|
|
69
|
+
return existed;
|
|
70
|
+
}
|
|
71
|
+
// -- Profile -------------------------------------------------------------
|
|
72
|
+
export async function readProfile(channelId) {
|
|
73
|
+
const path = profilePath(channelId);
|
|
74
|
+
if (!existsSync(path))
|
|
75
|
+
return undefined;
|
|
76
|
+
return readFile(path, 'utf-8');
|
|
77
|
+
}
|
|
78
|
+
export async function writeProfile(channelId, content) {
|
|
79
|
+
await ensureBrainDir(channelId);
|
|
80
|
+
const path = profilePath(channelId);
|
|
81
|
+
await writeFile(path, content, 'utf-8');
|
|
82
|
+
return path;
|
|
83
|
+
}
|
|
84
|
+
export function hasProfile(channelId) {
|
|
85
|
+
return existsSync(profilePath(channelId));
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=brain-storage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brain-storage.js","sourceRoot":"","sources":["../../src/utils/brain-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,mBAAmB,EAAsB,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD;;;;;;;GAOG;AAEH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,SAAiB;IACpD,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;IACtC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,2EAA2E;AAE3E,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,SAAiB;IAClD,OAAO,YAAY,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,mBAAmB,CAAC,CAAC;AACpE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,SAAiB;IACrD,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;IAE/C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,WAAW,EAAE,uCAAuC,SAAS,GAAG,EAAE;YACvF,QAAQ,EACN,qFAAqF;SACxF,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAuB;IACzD,MAAM,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACjD,MAAM,eAAe,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,CAAC;AAC5E,CAAC;AAED,6EAA6E;AAC7E,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAAE,OAAO,EAAE,CAAC;IAExC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACpE,MAAM,SAAS,GAAoB,EAAE,CAAC;IAEtC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;YAAE,SAAS;QAEnC,0EAA0E;QAC1E,oDAAoD;QACpD,MAAM,QAAQ,GAAG,MAAM,YAAY,CACjC,IAAI,CAAC,SAAS,EAAE,EAAE,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,EAC9C,mBAAmB,CACpB,CAAC;QACF,IAAI,QAAQ,KAAK,SAAS;YAAE,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,SAAiB;IACjD,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IAEtC,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,2EAA2E;AAE3E,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,SAAiB;IACjD,MAAM,IAAI,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,OAAO,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,SAAiB,EAAE,OAAe;IACnE,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,OAAO,UAAU,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC;AAC5C,CAAC"}
|
package/dist/utils/clips.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { homedir } from 'node:os';
|
|
2
1
|
import { join } from 'node:path';
|
|
3
2
|
import { mkdir } from 'node:fs/promises';
|
|
4
3
|
import { execa } from 'execa';
|
|
@@ -8,6 +7,7 @@ import { resolveOutputDir } from './validate.js';
|
|
|
8
7
|
import { TIMEOUTS, runYtDlp } from './ytdlp.js';
|
|
9
8
|
import { formatSrtTimestamp } from './transcript.js';
|
|
10
9
|
import { getChapters, getVideoInfo } from './youtube.js';
|
|
10
|
+
import { dataDir } from './paths.js';
|
|
11
11
|
/**
|
|
12
12
|
* Pulling usable excerpts out of a video for editing.
|
|
13
13
|
*
|
|
@@ -16,9 +16,9 @@ import { getChapters, getVideoInfo } from './youtube.js';
|
|
|
16
16
|
* range that covers the requested window, which is both far faster and far
|
|
17
17
|
* kinder to YouTube than pulling a three-hour file to keep 20 seconds of it.
|
|
18
18
|
*/
|
|
19
|
-
const CLIPS_DIR =
|
|
20
|
-
const FRAMES_DIR =
|
|
21
|
-
const SUBTITLES_DIR =
|
|
19
|
+
const CLIPS_DIR = dataDir('clips');
|
|
20
|
+
const FRAMES_DIR = dataDir('frames');
|
|
21
|
+
const SUBTITLES_DIR = dataDir('subtitles');
|
|
22
22
|
/**
|
|
23
23
|
* Filesystem-safe stem derived from the video title.
|
|
24
24
|
*
|
package/dist/utils/clips.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"clips.js","sourceRoot":"","sources":["../../src/utils/clips.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"clips.js","sourceRoot":"","sources":["../../src/utils/clips.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC;;;;;;;GAOG;AAEH,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AACnC,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACrC,MAAM,aAAa,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;AAgB3C;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAa,EAAE,QAAgB;IACtD,MAAM,OAAO,GAAG,KAAK;QACnB,yEAAyE;QACzE,qEAAqE;SACpE,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;QACrB,sEAAsE;QACtE,uEAAuE;SACtE,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;SACtB,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC;QAC9B,kFAAkF;SACjF,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;SACnB,IAAI,EAAE;SACN,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACZ,IAAI,EAAE,CAAC;IAEV,OAAO,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;AAC7C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAa,EACb,OAA2D;IAE3D,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IAEvC,IAAI,KAAgB,CAAC;IAErB,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,YAAY,CAAC,WAAW,EAAE,6BAA6B,EAAE;gBACjE,QAAQ,EAAE,6BAA6B;aACxC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,KAAK,GACT,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;YAClE,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAE3E,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,YAAY,CAAC,WAAW,EAAE,uBAAuB,OAAO,CAAC,OAAO,IAAI,EAAE;gBAC9E,QAAQ,EAAE,uBAAuB,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aACvF,CAAC,CAAC;QACL,CAAC;QACD,KAAK,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;IACzD,CAAC;SAAM,CAAC;QACN,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC7D,MAAM,IAAI,YAAY,CAAC,eAAe,EAAE,gDAAgD,EAAE;gBACxF,QAAQ,EAAE,6EAA6E;aACxF,CAAC,CAAC;QACL,CAAC;QACD,KAAK,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;IACrD,CAAC;IAED,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,YAAY,CAAC,eAAe,EAAE,+BAA+B,CAAC,CAAC;IAC3E,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACtD,MAAM,IAAI,YAAY,CACpB,eAAe,EACf,UAAU,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,8BAA8B,IAAI,CAAC,QAAQ,UAAU,EACtF,EAAE,QAAQ,EAAE,qCAAqC,EAAE,CACpD,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,+BAA+B;IAC/B,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnD,KAAK,GAAG,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;IAC3C,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;AACjF,CAAC;AAED,SAAS,UAAU,CAAC,KAAgB;IAClC,OAAO,IAAI,kBAAkB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;AACpH,CAAC;AAYD,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAa,EACb,YAAgE,EAChE,OAA2B;IAE3B,0EAA0E;IAC1E,2DAA2D;IAC3D,MAAM,aAAa,CAAC,iBAAiB,CAAC,CAAC;IACvC,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAEjE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;IAC1E,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,MAAM,IAAI,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;IACjG,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,UAAU,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAG,mCAAmC,OAAO,EAAE,CAAC;IAEzD,MAAM,IAAI,GAAG;QACX,IAAI;QACJ,OAAO,CAAC,cAAc;QACtB,qBAAqB;QACrB,UAAU,CAAC,KAAK,CAAC;QACjB,IAAI;QACJ,cAAc;QACd,eAAe;KAChB,CAAC;IAEF,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,4EAA4E;QAC5E,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,gBAAgB,EAAE,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC,CAAC;IAC/E,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEf,MAAM,QAAQ,CAAC,IAAI,EAAE;QACnB,KAAK,EAAE,cAAc;QACrB,SAAS,EAAE,QAAQ,CAAC,QAAQ;QAC5B,KAAK,EAAE,KAAK;KACb,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,MAAM,QAAQ,CACnC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,iBAAiB,EAAE,GAAG,CAAC,EACrE,EAAE,KAAK,EAAE,yBAAyB,EAAE,CACrC,CAAC;IAEF,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IACzF,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC;IACvC,MAAM,QAAQ,GACZ,QAAQ,KAAK,EAAE;QACb,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;QACzC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAE5C,OAAO;QACL,OAAO;QACP,KAAK;QACL,QAAQ;QACR,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,QAAQ,EAAE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK;KAClC,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY,EAAE,SAAiB;IACvD,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,SAAS,EAAE,CAAC,CAAC;AACtD,CAAC;AASD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAa,EACb,SAAiB,EACjB,OAAsD;IAEtD,MAAM,aAAa,CAAC,kBAAkB,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IAEvC,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpD,MAAM,IAAI,YAAY,CACpB,eAAe,EACf,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,6BAA6B,IAAI,CAAC,QAAQ,UAAU,EAC5E,EAAE,QAAQ,EAAE,oCAAoC,EAAE,CACnD,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,MAAM,GAAG,GAAG,mCAAmC,IAAI,CAAC,EAAE,EAAE,CAAC;IACzD,MAAM,QAAQ,GAAG,CACf,MAAM,QAAQ,CAAC,CAAC,IAAI,EAAE,8BAA8B,EAAE,WAAW,EAAE,eAAe,EAAE,GAAG,CAAC,EAAE;QACxF,KAAK,EAAE,yBAAyB;KACjC,CAAC,CACH;SACE,IAAI,EAAE;SACN,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAElB,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QAC9C,MAAM,IAAI,YAAY,CAAC,cAAc,EAAE,kDAAkD,EAAE;YACzF,QAAQ,EAAE,oDAAoD;SAC/D,CAAC,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CACnB,SAAS,EACT,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,OAAO,CAAC,MAAM,EAAE,CACjF,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,KAAK,CACT,QAAQ,EACR;YACE,oEAAoE;YACpE,KAAK;YACL,MAAM,CAAC,SAAS,CAAC;YACjB,IAAI;YACJ,QAAQ;YACR,WAAW;YACX,GAAG;YACH,MAAM;YACN,GAAG;YACH,IAAI;YACJ,QAAQ;SACT,EACD,EAAE,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,CACjC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,YAAY,CAAC,cAAc,EAAE,sCAAsC,EAAE;YAC7E,QAAQ,EAAE,2EAA2E;YACrF,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,aAAa,CAAC;AACvB,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Reading and writing the JSON documents this server keeps on disk.
|
|
4
|
+
*
|
|
5
|
+
* Both halves exist because the library and the brains make the same two
|
|
6
|
+
* mistakes otherwise. Writing straight to the destination means an interrupted
|
|
7
|
+
* process leaves a half-written file where a valid one used to be — survivable
|
|
8
|
+
* for a cache, fatal for a brain manifest that is checkpointed every few videos
|
|
9
|
+
* across a build lasting tens of minutes. Reading without validation means a
|
|
10
|
+
* file edited by hand, or truncated by that same crash, propagates `undefined`
|
|
11
|
+
* into code that assumed a shape.
|
|
12
|
+
*/
|
|
13
|
+
export interface WriteJsonOptions {
|
|
14
|
+
/**
|
|
15
|
+
* Indent the output. Worth it for files a human may open; not for a search
|
|
16
|
+
* index, where indentation is most of the bytes.
|
|
17
|
+
*/
|
|
18
|
+
pretty?: boolean;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Write JSON through a temporary file and rename it into place.
|
|
22
|
+
*
|
|
23
|
+
* `rename` within a directory is atomic on POSIX and on Windows via
|
|
24
|
+
* `MoveFileEx`, so a reader sees either the previous document or the new one,
|
|
25
|
+
* never a partial write.
|
|
26
|
+
*/
|
|
27
|
+
export declare function writeJsonAtomic(path: string, value: unknown, options?: WriteJsonOptions): Promise<void>;
|
|
28
|
+
/**
|
|
29
|
+
* Read and validate a JSON document, or `undefined` if it is missing,
|
|
30
|
+
* unparseable or does not match.
|
|
31
|
+
*
|
|
32
|
+
* Callers decide what a missing document means — an empty library, a brain that
|
|
33
|
+
* has not been built — which is information this function does not have.
|
|
34
|
+
*/
|
|
35
|
+
export declare function readJsonFile<T>(path: string, schema: z.ZodType<T, z.ZodTypeDef, unknown>): Promise<T | undefined>;
|
|
36
|
+
//# sourceMappingURL=json-file.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-file.d.ts","sourceRoot":"","sources":["../../src/utils/json-file.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,EACd,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAgBf;AAID;;;;;;GAMG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAClC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,CAAC,GAC1C,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CASxB"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
/**
|
|
4
|
+
* Write JSON through a temporary file and rename it into place.
|
|
5
|
+
*
|
|
6
|
+
* `rename` within a directory is atomic on POSIX and on Windows via
|
|
7
|
+
* `MoveFileEx`, so a reader sees either the previous document or the new one,
|
|
8
|
+
* never a partial write.
|
|
9
|
+
*/
|
|
10
|
+
export async function writeJsonAtomic(path, value, options = {}) {
|
|
11
|
+
const { pretty = true } = options;
|
|
12
|
+
const serialised = JSON.stringify(value, null, pretty ? 2 : undefined);
|
|
13
|
+
// A shared temporary name would let two writers of the same document destroy
|
|
14
|
+
// each other's staging file, and the loser's rename would fail on a path that
|
|
15
|
+
// the winner had already moved away.
|
|
16
|
+
const temporaryPath = `${path}.${process.pid}.${writeSequence++}.tmp`;
|
|
17
|
+
try {
|
|
18
|
+
await writeFile(temporaryPath, serialised, 'utf-8');
|
|
19
|
+
await rename(temporaryPath, path);
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
await rm(temporaryPath, { force: true });
|
|
23
|
+
throw error;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
let writeSequence = 0;
|
|
27
|
+
/**
|
|
28
|
+
* Read and validate a JSON document, or `undefined` if it is missing,
|
|
29
|
+
* unparseable or does not match.
|
|
30
|
+
*
|
|
31
|
+
* Callers decide what a missing document means — an empty library, a brain that
|
|
32
|
+
* has not been built — which is information this function does not have.
|
|
33
|
+
*/
|
|
34
|
+
export async function readJsonFile(path, schema) {
|
|
35
|
+
if (!existsSync(path))
|
|
36
|
+
return undefined;
|
|
37
|
+
try {
|
|
38
|
+
const parsed = schema.safeParse(JSON.parse(await readFile(path, 'utf-8')));
|
|
39
|
+
return parsed.success ? parsed.data : undefined;
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=json-file.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-file.js","sourceRoot":"","sources":["../../src/utils/json-file.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAuBnE;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAY,EACZ,KAAc,EACd,UAA4B,EAAE;IAE9B,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAEvE,6EAA6E;IAC7E,8EAA8E;IAC9E,qCAAqC;IACrC,MAAM,aAAa,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,IAAI,aAAa,EAAE,MAAM,CAAC;IAEtE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,aAAa,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,MAAM,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,IAAI,aAAa,GAAG,CAAC,CAAC;AAEtB;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAY,EACZ,MAA2C;IAE3C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IAExC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QAC3E,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Everything this server writes lives under one directory in the user's home:
|
|
3
|
+
* transcripts, the library, clips, and the channel brains.
|
|
4
|
+
*
|
|
5
|
+
* Spelled out in six places before this existed, which is five chances for a
|
|
6
|
+
* rename to leave a stray directory behind.
|
|
7
|
+
*/
|
|
8
|
+
export declare function dataDir(...segments: string[]): string;
|
|
9
|
+
//# sourceMappingURL=paths.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paths.d.ts","sourceRoot":"","sources":["../../src/utils/paths.ts"],"names":[],"mappings":"AAGA;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,GAAG,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAErD"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { homedir } from 'node:os';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
/**
|
|
4
|
+
* Everything this server writes lives under one directory in the user's home:
|
|
5
|
+
* transcripts, the library, clips, and the channel brains.
|
|
6
|
+
*
|
|
7
|
+
* Spelled out in six places before this existed, which is five chances for a
|
|
8
|
+
* rename to leave a stray directory behind.
|
|
9
|
+
*/
|
|
10
|
+
export function dataDir(...segments) {
|
|
11
|
+
return join(homedir(), '.youtube-knowledge', ...segments);
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=paths.js.map
|