youtube-knowledge-mcp 2.2.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/README.md +62 -18
  2. package/dist/harvest-schemas.d.ts +75 -0
  3. package/dist/harvest-schemas.d.ts.map +1 -0
  4. package/dist/harvest-schemas.js +106 -0
  5. package/dist/harvest-schemas.js.map +1 -0
  6. package/dist/registry/harvest-tools.d.ts +10 -0
  7. package/dist/registry/harvest-tools.d.ts.map +1 -0
  8. package/dist/registry/harvest-tools.js +109 -0
  9. package/dist/registry/harvest-tools.js.map +1 -0
  10. package/dist/registry/health-tools.d.ts.map +1 -1
  11. package/dist/registry/health-tools.js +18 -0
  12. package/dist/registry/health-tools.js.map +1 -1
  13. package/dist/registry/index.d.ts.map +1 -1
  14. package/dist/registry/index.js +2 -0
  15. package/dist/registry/index.js.map +1 -1
  16. package/dist/registry/video-tools.js +1 -1
  17. package/dist/registry/video-tools.js.map +1 -1
  18. package/dist/schemas.d.ts +239 -0
  19. package/dist/schemas.d.ts.map +1 -1
  20. package/dist/schemas.js +56 -0
  21. package/dist/schemas.js.map +1 -1
  22. package/dist/thumbnail-schemas.d.ts +28 -28
  23. package/dist/tools/ask-brain.js +1 -1
  24. package/dist/tools/ask-brain.js.map +1 -1
  25. package/dist/tools/brain-info.js +1 -1
  26. package/dist/tools/brain-info.js.map +1 -1
  27. package/dist/tools/check-health.d.ts +59 -0
  28. package/dist/tools/check-health.d.ts.map +1 -1
  29. package/dist/tools/check-health.js +85 -6
  30. package/dist/tools/check-health.js.map +1 -1
  31. package/dist/tools/fetch-channel-thumbnails.d.ts +8 -8
  32. package/dist/tools/fetch-videos.d.ts +15 -4
  33. package/dist/tools/fetch-videos.d.ts.map +1 -1
  34. package/dist/tools/fetch-videos.js +63 -22
  35. package/dist/tools/fetch-videos.js.map +1 -1
  36. package/dist/tools/get-comments.d.ts +238 -6
  37. package/dist/tools/get-comments.d.ts.map +1 -1
  38. package/dist/tools/get-comments.js +85 -21
  39. package/dist/tools/get-comments.js.map +1 -1
  40. package/dist/tools/get-coverage.d.ts +122 -0
  41. package/dist/tools/get-coverage.d.ts.map +1 -0
  42. package/dist/tools/get-coverage.js +105 -0
  43. package/dist/tools/get-coverage.js.map +1 -0
  44. package/dist/tools/harvest-channel.d.ts +92 -0
  45. package/dist/tools/harvest-channel.d.ts.map +1 -0
  46. package/dist/tools/harvest-channel.js +62 -0
  47. package/dist/tools/harvest-channel.js.map +1 -0
  48. package/dist/tools/harvest-comments.d.ts +81 -0
  49. package/dist/tools/harvest-comments.d.ts.map +1 -0
  50. package/dist/tools/harvest-comments.js +115 -0
  51. package/dist/tools/harvest-comments.js.map +1 -0
  52. package/dist/tools/library.js +1 -1
  53. package/dist/tools/library.js.map +1 -1
  54. package/dist/tools/list-channel-thumbnails.d.ts +10 -10
  55. package/dist/tools/list-channel-thumbnails.js +1 -1
  56. package/dist/tools/list-channel-thumbnails.js.map +1 -1
  57. package/dist/tools/list-library.js +1 -1
  58. package/dist/tools/list-library.js.map +1 -1
  59. package/dist/tools/prune-harvest.d.ts +59 -0
  60. package/dist/tools/prune-harvest.d.ts.map +1 -0
  61. package/dist/tools/prune-harvest.js +108 -0
  62. package/dist/tools/prune-harvest.js.map +1 -0
  63. package/dist/tools/query-comments.d.ts +125 -0
  64. package/dist/tools/query-comments.d.ts.map +1 -0
  65. package/dist/tools/query-comments.js +106 -0
  66. package/dist/tools/query-comments.js.map +1 -0
  67. package/dist/tools/query-videos.d.ts +118 -0
  68. package/dist/tools/query-videos.d.ts.map +1 -0
  69. package/dist/tools/query-videos.js +97 -0
  70. package/dist/tools/query-videos.js.map +1 -0
  71. package/dist/tools/repair-store.d.ts +34 -0
  72. package/dist/tools/repair-store.d.ts.map +1 -0
  73. package/dist/tools/repair-store.js +58 -0
  74. package/dist/tools/repair-store.js.map +1 -0
  75. package/dist/tools/search-channels.d.ts.map +1 -1
  76. package/dist/tools/search-channels.js +5 -1
  77. package/dist/tools/search-channels.js.map +1 -1
  78. package/dist/tools/search-videos.js +1 -1
  79. package/dist/tools/search-videos.js.map +1 -1
  80. package/dist/utils/comment-store.d.ts +71 -0
  81. package/dist/utils/comment-store.d.ts.map +1 -0
  82. package/dist/utils/comment-store.js +129 -0
  83. package/dist/utils/comment-store.js.map +1 -0
  84. package/dist/utils/comment-threads.d.ts +51 -0
  85. package/dist/utils/comment-threads.d.ts.map +1 -0
  86. package/dist/utils/comment-threads.js +76 -0
  87. package/dist/utils/comment-threads.js.map +1 -0
  88. package/dist/utils/coverage-text.d.ts +3 -0
  89. package/dist/utils/coverage-text.d.ts.map +1 -0
  90. package/dist/utils/coverage-text.js +62 -0
  91. package/dist/utils/coverage-text.js.map +1 -0
  92. package/dist/utils/coverage.d.ts +57 -0
  93. package/dist/utils/coverage.d.ts.map +1 -0
  94. package/dist/utils/coverage.js +142 -0
  95. package/dist/utils/coverage.js.map +1 -0
  96. package/dist/utils/errors.d.ts +1 -1
  97. package/dist/utils/errors.d.ts.map +1 -1
  98. package/dist/utils/errors.js +9 -1
  99. package/dist/utils/errors.js.map +1 -1
  100. package/dist/utils/format.d.ts +17 -1
  101. package/dist/utils/format.d.ts.map +1 -1
  102. package/dist/utils/format.js +1 -1
  103. package/dist/utils/format.js.map +1 -1
  104. package/dist/utils/harvest-catalog.d.ts +44 -0
  105. package/dist/utils/harvest-catalog.d.ts.map +1 -0
  106. package/dist/utils/harvest-catalog.js +164 -0
  107. package/dist/utils/harvest-catalog.js.map +1 -0
  108. package/dist/utils/harvest-lock.d.ts +3 -0
  109. package/dist/utils/harvest-lock.d.ts.map +1 -0
  110. package/dist/utils/harvest-lock.js +26 -0
  111. package/dist/utils/harvest-lock.js.map +1 -0
  112. package/dist/utils/harvest-receipts.d.ts +14 -0
  113. package/dist/utils/harvest-receipts.d.ts.map +1 -0
  114. package/dist/utils/harvest-receipts.js +110 -0
  115. package/dist/utils/harvest-receipts.js.map +1 -0
  116. package/dist/utils/listing-cursor.d.ts +31 -0
  117. package/dist/utils/listing-cursor.d.ts.map +1 -0
  118. package/dist/utils/listing-cursor.js +39 -0
  119. package/dist/utils/listing-cursor.js.map +1 -0
  120. package/dist/utils/pot-preflight.d.ts +53 -0
  121. package/dist/utils/pot-preflight.d.ts.map +1 -0
  122. package/dist/utils/pot-preflight.js +146 -0
  123. package/dist/utils/pot-preflight.js.map +1 -0
  124. package/dist/utils/sleep.d.ts +9 -0
  125. package/dist/utils/sleep.d.ts.map +1 -0
  126. package/dist/utils/sleep.js +25 -0
  127. package/dist/utils/sleep.js.map +1 -0
  128. package/dist/utils/store-health.d.ts +37 -0
  129. package/dist/utils/store-health.d.ts.map +1 -0
  130. package/dist/utils/store-health.js +92 -0
  131. package/dist/utils/store-health.js.map +1 -0
  132. package/dist/utils/store-migrations.d.ts +33 -0
  133. package/dist/utils/store-migrations.d.ts.map +1 -0
  134. package/dist/utils/store-migrations.js +41 -0
  135. package/dist/utils/store-migrations.js.map +1 -0
  136. package/dist/utils/store-paths.d.ts +33 -0
  137. package/dist/utils/store-paths.d.ts.map +1 -0
  138. package/dist/utils/store-paths.js +48 -0
  139. package/dist/utils/store-paths.js.map +1 -0
  140. package/dist/utils/store-rows.d.ts +22 -0
  141. package/dist/utils/store-rows.d.ts.map +1 -0
  142. package/dist/utils/store-rows.js +62 -0
  143. package/dist/utils/store-rows.js.map +1 -0
  144. package/dist/utils/store-schema.d.ts +16 -0
  145. package/dist/utils/store-schema.d.ts.map +1 -0
  146. package/dist/utils/store-schema.js +168 -0
  147. package/dist/utils/store-schema.js.map +1 -0
  148. package/dist/utils/store.d.ts +34 -0
  149. package/dist/utils/store.d.ts.map +1 -0
  150. package/dist/utils/store.js +157 -0
  151. package/dist/utils/store.js.map +1 -0
  152. package/dist/utils/video-stats-cache.d.ts +16 -0
  153. package/dist/utils/video-stats-cache.d.ts.map +1 -0
  154. package/dist/utils/video-stats-cache.js +81 -0
  155. package/dist/utils/video-stats-cache.js.map +1 -0
  156. package/dist/utils/video-stats.d.ts +186 -0
  157. package/dist/utils/video-stats.d.ts.map +1 -0
  158. package/dist/utils/video-stats.js +168 -0
  159. package/dist/utils/video-stats.js.map +1 -0
  160. package/dist/utils/video-store.d.ts +69 -0
  161. package/dist/utils/video-store.d.ts.map +1 -0
  162. package/dist/utils/video-store.js +77 -0
  163. package/dist/utils/video-store.js.map +1 -0
  164. package/dist/utils/youtube-channel.d.ts +9 -2
  165. package/dist/utils/youtube-channel.d.ts.map +1 -1
  166. package/dist/utils/youtube-channel.js +33 -3
  167. package/dist/utils/youtube-channel.js.map +1 -1
  168. package/dist/utils/youtube-schemas.d.ts +98 -0
  169. package/dist/utils/youtube-schemas.d.ts.map +1 -1
  170. package/dist/utils/youtube-schemas.js +30 -0
  171. package/dist/utils/youtube-schemas.js.map +1 -1
  172. package/dist/utils/youtube-video.d.ts +22 -1
  173. package/dist/utils/youtube-video.d.ts.map +1 -1
  174. package/dist/utils/youtube-video.js +40 -28
  175. package/dist/utils/youtube-video.js.map +1 -1
  176. package/dist/utils/ytdlp-limiter.d.ts +3 -0
  177. package/dist/utils/ytdlp-limiter.d.ts.map +1 -1
  178. package/dist/utils/ytdlp-limiter.js +19 -3
  179. package/dist/utils/ytdlp-limiter.js.map +1 -1
  180. package/dist/utils/ytdlp-pacer.d.ts +64 -0
  181. package/dist/utils/ytdlp-pacer.d.ts.map +1 -0
  182. package/dist/utils/ytdlp-pacer.js +165 -0
  183. package/dist/utils/ytdlp-pacer.js.map +1 -0
  184. package/dist/utils/ytdlp-stream.d.ts +32 -0
  185. package/dist/utils/ytdlp-stream.d.ts.map +1 -0
  186. package/dist/utils/ytdlp-stream.js +61 -0
  187. package/dist/utils/ytdlp-stream.js.map +1 -0
  188. package/dist/utils/ytdlp.d.ts +10 -0
  189. package/dist/utils/ytdlp.d.ts.map +1 -1
  190. package/dist/utils/ytdlp.js +36 -25
  191. package/dist/utils/ytdlp.js.map +1 -1
  192. package/package.json +2 -2
@@ -0,0 +1,41 @@
1
+ import { STORE_VERSION } from './store-schema.js';
2
+ import { writeUserVersion } from './store.js';
3
+ /**
4
+ * Empty on purpose at v1: `STORE_SCHEMA` is `CREATE TABLE IF NOT EXISTS`
5
+ * throughout, so a fresh database and an existing v1 database converge without
6
+ * one. The list exists so the second schema change is a one-line addition
7
+ * rather than a retrofit.
8
+ */
9
+ export const STORE_MIGRATIONS = [];
10
+ /**
11
+ * Brings a database up to STORE_VERSION.
12
+ *
13
+ * Each migration runs inside its own transaction so a failure leaves the
14
+ * store at the last version that fully applied, rather than half-way through
15
+ * one. A newer-than-known version is rejected before this is reached, in
16
+ * `store.ts`.
17
+ */
18
+ export function runStoreMigrations(database, fromVersion) {
19
+ let current = fromVersion;
20
+ for (const migration of STORE_MIGRATIONS) {
21
+ if (migration.to <= current)
22
+ continue;
23
+ database.exec('BEGIN');
24
+ try {
25
+ migration.apply(database);
26
+ writeUserVersion(database, migration.to);
27
+ database.exec('COMMIT');
28
+ }
29
+ catch (error) {
30
+ database.exec('ROLLBACK');
31
+ throw error;
32
+ }
33
+ current = migration.to;
34
+ }
35
+ if (current !== STORE_VERSION) {
36
+ writeUserVersion(database, STORE_VERSION);
37
+ current = STORE_VERSION;
38
+ }
39
+ return current;
40
+ }
41
+ //# sourceMappingURL=store-migrations.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store-migrations.js","sourceRoot":"","sources":["../../src/utils/store-migrations.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAmB9C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAA8B,EAAE,CAAC;AAE9D;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAsB,EAAE,WAAmB;IAC5E,IAAI,OAAO,GAAG,WAAW,CAAC;IAE1B,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;QACzC,IAAI,SAAS,CAAC,EAAE,IAAI,OAAO;YAAE,SAAS;QAEtC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,IAAI,CAAC;YACH,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC1B,gBAAgB,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1B,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,GAAG,SAAS,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAI,OAAO,KAAK,aAAa,EAAE,CAAC;QAC9B,gBAAgB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAC1C,OAAO,GAAG,aAAa,CAAC;IAC1B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
@@ -0,0 +1,33 @@
1
+ /**
2
+ * Every file the harvested store is made of.
3
+ *
4
+ * The only module that names `dataDir('store')`, so a later decision to shard
5
+ * the database by channel touches one file rather than every caller — the same
6
+ * reason `brain-paths.ts` exists.
7
+ *
8
+ * The `-wal` and `-shm` sidecars are created and removed by SQLite itself, not
9
+ * by us. They are the one place in this codebase where a file under the data
10
+ * directory is not written through `writeJsonAtomic`, and that exception is
11
+ * confined to the `store*.ts` modules.
12
+ */
13
+ export declare function storeDir(): string;
14
+ export declare function storeDatabasePath(): string;
15
+ /**
16
+ * Where a database that failed its integrity check is moved to.
17
+ *
18
+ * Renamed rather than deleted: a harvest costs hours of network that no local
19
+ * cache can replay, so nothing here removes one without the user asking.
20
+ */
21
+ export declare function corruptStorePath(at: number): string;
22
+ /** Taken before a migration that would change an existing `user_version`. */
23
+ export declare function preMigrationBackupPath(fromVersion: number): string;
24
+ /**
25
+ * Job-level exclusion, which WAL does not provide.
26
+ *
27
+ * WAL keeps concurrent writes consistent; it does not stop two harvests of the
28
+ * same channel from spawning yt-dlp twice and doubling the request rate at the
29
+ * one thing that throttles us.
30
+ */
31
+ export declare function channelHarvestLockPath(channelId: string): string;
32
+ export declare function videoHarvestLockPath(videoId: string): string;
33
+ //# sourceMappingURL=store-paths.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store-paths.d.ts","sourceRoot":"","sources":["../../src/utils/store-paths.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;GAWG;AAEH,wBAAgB,QAAQ,IAAI,MAAM,CAEjC;AAED,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED,6EAA6E;AAC7E,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAElE;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE5D"}
@@ -0,0 +1,48 @@
1
+ import { join } from 'node:path';
2
+ import { dataDir } from './paths.js';
3
+ import { assertChannelId, assertVideoId } from './validate.js';
4
+ /**
5
+ * Every file the harvested store is made of.
6
+ *
7
+ * The only module that names `dataDir('store')`, so a later decision to shard
8
+ * the database by channel touches one file rather than every caller — the same
9
+ * reason `brain-paths.ts` exists.
10
+ *
11
+ * The `-wal` and `-shm` sidecars are created and removed by SQLite itself, not
12
+ * by us. They are the one place in this codebase where a file under the data
13
+ * directory is not written through `writeJsonAtomic`, and that exception is
14
+ * confined to the `store*.ts` modules.
15
+ */
16
+ export function storeDir() {
17
+ return dataDir('store');
18
+ }
19
+ export function storeDatabasePath() {
20
+ return join(storeDir(), 'knowledge.db');
21
+ }
22
+ /**
23
+ * Where a database that failed its integrity check is moved to.
24
+ *
25
+ * Renamed rather than deleted: a harvest costs hours of network that no local
26
+ * cache can replay, so nothing here removes one without the user asking.
27
+ */
28
+ export function corruptStorePath(at) {
29
+ return join(storeDir(), `knowledge.corrupt-${String(at)}.db`);
30
+ }
31
+ /** Taken before a migration that would change an existing `user_version`. */
32
+ export function preMigrationBackupPath(fromVersion) {
33
+ return join(storeDir(), `knowledge.pre-v${String(fromVersion)}.db`);
34
+ }
35
+ /**
36
+ * Job-level exclusion, which WAL does not provide.
37
+ *
38
+ * WAL keeps concurrent writes consistent; it does not stop two harvests of the
39
+ * same channel from spawning yt-dlp twice and doubling the request rate at the
40
+ * one thing that throttles us.
41
+ */
42
+ export function channelHarvestLockPath(channelId) {
43
+ return join(storeDir(), `harvest.${assertChannelId(channelId)}.lock`);
44
+ }
45
+ export function videoHarvestLockPath(videoId) {
46
+ return join(storeDir(), `harvest.${assertVideoId(videoId)}.lock`);
47
+ }
48
+ //# sourceMappingURL=store-paths.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store-paths.js","sourceRoot":"","sources":["../../src/utils/store-paths.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE/D;;;;;;;;;;;GAWG;AAEH,MAAM,UAAU,QAAQ;IACtB,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,IAAI,CAAC,QAAQ,EAAE,EAAE,cAAc,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAAU;IACzC,OAAO,IAAI,CAAC,QAAQ,EAAE,EAAE,qBAAqB,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,sBAAsB,CAAC,WAAmB;IACxD,OAAO,IAAI,CAAC,QAAQ,EAAE,EAAE,kBAAkB,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;AACtE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CAAC,SAAiB;IACtD,OAAO,IAAI,CAAC,QAAQ,EAAE,EAAE,WAAW,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,OAAO,IAAI,CAAC,QAAQ,EAAE,EAAE,WAAW,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AACpE,CAAC"}
@@ -0,0 +1,22 @@
1
+ import type { StatementSync } from 'node:sqlite';
2
+ import { z } from 'zod';
3
+ /**
4
+ * Reading rows back out of the store, validated.
5
+ *
6
+ * `StatementSync.all()` is typed `Record<string, SQLOutputValue>[]` — usefully
7
+ * honest, and unusable directly. Every read goes through a zod schema here,
8
+ * the same contract `readJsonFile` and `parseYtDlpJson` already apply to the
9
+ * other two places untyped data enters this server. No `as` anywhere.
10
+ */
11
+ /**
12
+ * SQLite `STRICT` tables have no BOOLEAN, so flags are stored as 0 or 1.
13
+ * Decoded with an explicit transform rather than a cast, which also rejects a
14
+ * 2 that some future writer might smuggle in.
15
+ */
16
+ export declare const sqliteBoolean: z.ZodEffects<z.ZodUnion<[z.ZodLiteral<0>, z.ZodLiteral<1>]>, boolean, 0 | 1>;
17
+ export declare function queryRows<T extends z.ZodTypeAny>(statement: StatementSync, schema: T, ...params: readonly (string | number | null)[]): z.output<T>[];
18
+ /** The single-row form. Returns undefined when the statement matched nothing. */
19
+ export declare function queryRow<T extends z.ZodTypeAny>(statement: StatementSync, schema: T, ...params: readonly (string | number | null)[]): z.output<T> | undefined;
20
+ /** `SELECT COUNT(*)` is the one query whose shape is always the same. */
21
+ export declare function countRows(statement: StatementSync, ...params: readonly (string | number | null)[]): number;
22
+ //# sourceMappingURL=store-rows.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store-rows.d.ts","sourceRoot":"","sources":["../../src/utils/store-rows.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;;;;;;GAOG;AAEH;;;;GAIG;AACH,eAAO,MAAM,aAAa,8EAEU,CAAC;AA8BrC,wBAAgB,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC,UAAU,EAC9C,SAAS,EAAE,aAAa,EACxB,MAAM,EAAE,CAAC,EACT,GAAG,MAAM,EAAE,SAAS,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAC7C,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAEf;AAED,iFAAiF;AACjF,wBAAgB,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,UAAU,EAC7C,SAAS,EAAE,aAAa,EACxB,MAAM,EAAE,CAAC,EACT,GAAG,MAAM,EAAE,SAAS,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAC7C,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,SAAS,CAQzB;AAED,yEAAyE;AACzE,wBAAgB,SAAS,CACvB,SAAS,EAAE,aAAa,EACxB,GAAG,MAAM,EAAE,SAAS,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAC7C,MAAM,CAMR"}
@@ -0,0 +1,62 @@
1
+ import { z } from 'zod';
2
+ import { YouTubeError } from './errors.js';
3
+ /**
4
+ * Reading rows back out of the store, validated.
5
+ *
6
+ * `StatementSync.all()` is typed `Record<string, SQLOutputValue>[]` — usefully
7
+ * honest, and unusable directly. Every read goes through a zod schema here,
8
+ * the same contract `readJsonFile` and `parseYtDlpJson` already apply to the
9
+ * other two places untyped data enters this server. No `as` anywhere.
10
+ */
11
+ /**
12
+ * SQLite `STRICT` tables have no BOOLEAN, so flags are stored as 0 or 1.
13
+ * Decoded with an explicit transform rather than a cast, which also rejects a
14
+ * 2 that some future writer might smuggle in.
15
+ */
16
+ export const sqliteBoolean = z
17
+ .union([z.literal(0), z.literal(1)])
18
+ .transform((value) => value === 1);
19
+ function describe(error) {
20
+ return error.issues
21
+ .slice(0, 3)
22
+ .map((issue) => `${issue.path.join('.') || '(row)'}: ${issue.message}`)
23
+ .join('; ');
24
+ }
25
+ /**
26
+ * Runs a statement and validates every row.
27
+ *
28
+ * A row that does not match is a bug in this server, not something a caller
29
+ * can fix, so it fails loudly rather than being skipped — the opposite of
30
+ * `recordOfValid`, which drops a bad entry because there the bad entry came
31
+ * from YouTube.
32
+ */
33
+ function parseAll(schema, rows) {
34
+ const parsed = z.array(schema).safeParse(rows);
35
+ if (parsed.success)
36
+ return parsed.data;
37
+ throw new YouTubeError('MALFORMED_RESPONSE', 'A row in the harvested store had an unexpected shape.', {
38
+ nextStep: `This is a bug in the server, not in your request. Details: ${describe(parsed.error)}`,
39
+ });
40
+ }
41
+ export function queryRows(statement, schema, ...params) {
42
+ return parseAll(schema, statement.all(...params));
43
+ }
44
+ /** The single-row form. Returns undefined when the statement matched nothing. */
45
+ export function queryRow(statement, schema, ...params) {
46
+ const row = statement.get(...params);
47
+ if (row === undefined)
48
+ return undefined;
49
+ // Parsed through `z.array(schema)` rather than `schema` directly: a bare
50
+ // generic `safeParse` widens its result to `any`, which would let every
51
+ // later misuse through unchecked. This also shares one error message.
52
+ return parseAll(schema, [row])[0];
53
+ }
54
+ /** `SELECT COUNT(*)` is the one query whose shape is always the same. */
55
+ export function countRows(statement, ...params) {
56
+ const row = queryRow(statement, z.object({}).passthrough(), ...params);
57
+ if (row === undefined)
58
+ return 0;
59
+ const first = Object.values(row)[0];
60
+ return typeof first === 'number' ? first : 0;
61
+ }
62
+ //# sourceMappingURL=store-rows.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store-rows.js","sourceRoot":"","sources":["../../src/utils/store-rows.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C;;;;;;;GAOG;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC;KAC3B,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;KACnC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;AAErC,SAAS,QAAQ,CAAC,KAAiB;IACjC,OAAO,KAAK,CAAC,MAAM;SAChB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;SACtE,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,QAAQ,CAAyB,MAAS,EAAE,IAAe;IAClE,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC;IAEvC,MAAM,IAAI,YAAY,CACpB,oBAAoB,EACpB,uDAAuD,EACvD;QACE,QAAQ,EAAE,8DAA8D,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;KACjG,CACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,SAAwB,EACxB,MAAS,EACT,GAAG,MAA2C;IAE9C,OAAO,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,QAAQ,CACtB,SAAwB,EACxB,MAAS,EACT,GAAG,MAA2C;IAE9C,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IACrC,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAExC,yEAAyE;IACzE,wEAAwE;IACxE,sEAAsE;IACtE,OAAO,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,SAAS,CACvB,SAAwB,EACxB,GAAG,MAA2C;IAE9C,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,GAAG,MAAM,CAAC,CAAC;IACvE,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC;IAEhC,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/C,CAAC"}
@@ -0,0 +1,16 @@
1
+ /**
2
+ * The shape of the harvested store, as one reviewable unit.
3
+ *
4
+ * Kept apart from the module that opens the database so the schema can be read
5
+ * end to end without the connection handling in the way.
6
+ *
7
+ * `STRICT` on every table: SQLite's default type affinity would silently accept
8
+ * a string where a count belongs, and this store exists to make claims about
9
+ * counts. It has no BOOLEAN, so flags are 0/1 integers decoded explicitly on
10
+ * the way out (see store-rows.ts) rather than cast.
11
+ */
12
+ /** Bumped only by a migration in store-migrations.ts. Independent of the server's semver. */
13
+ export declare const STORE_VERSION = 1;
14
+ export declare const STORE_PRAGMAS: readonly ["PRAGMA journal_mode = WAL", "PRAGMA synchronous = NORMAL", "PRAGMA foreign_keys = ON", "PRAGMA temp_store = MEMORY"];
15
+ export declare const STORE_SCHEMA = "\nCREATE TABLE IF NOT EXISTS meta (\n key TEXT PRIMARY KEY,\n value TEXT NOT NULL\n) STRICT;\n\nCREATE TABLE IF NOT EXISTS channel (\n channel_id TEXT PRIMARY KEY,\n handle TEXT NOT NULL DEFAULT '',\n name TEXT NOT NULL DEFAULT '',\n channel_url TEXT NOT NULL DEFAULT '',\n subscriber_count INTEGER,\n description TEXT NOT NULL DEFAULT '',\n avatar_url TEXT,\n banner_url TEXT,\n first_seen_at INTEGER NOT NULL,\n updated_at INTEGER NOT NULL\n) STRICT;\n\nCREATE TABLE IF NOT EXISTS video (\n video_id TEXT PRIMARY KEY,\n channel_id TEXT REFERENCES channel(channel_id) ON DELETE CASCADE,\n title TEXT NOT NULL DEFAULT '',\n duration_s INTEGER,\n -- 'YYYY-MM-DD'. NULL, never '', when a flat listing carried none: a date\n -- filter must be able to tell \"not known yet\" from \"known to be empty\".\n upload_date TEXT,\n view_count INTEGER,\n like_count INTEGER,\n -- YouTube's own claim, read WITHOUT --write-comments, which overwrites it\n -- with the number extracted. This is the denominator every comment receipt\n -- is measured against, so it may only ever come from a metadata pass.\n comment_count INTEGER,\n live_status TEXT,\n availability TEXT,\n thumbnail_url TEXT,\n tab TEXT,\n also_in_tabs TEXT NOT NULL DEFAULT '[]',\n catalog_rank INTEGER,\n listed_at INTEGER,\n detail_at INTEGER,\n -- Stripped before storage: formats, automatic_captions, subtitles,\n -- thumbnails and comments are 96% of a raw -j payload and are mostly\n -- expiring URLs. See MAX_DETAIL_JSON_BYTES.\n detail_json TEXT,\n heatmap_json TEXT\n) STRICT;\n\nCREATE INDEX IF NOT EXISTS video_by_channel ON video(channel_id, catalog_rank);\nCREATE INDEX IF NOT EXISTS video_by_date ON video(channel_id, upload_date DESC);\n\nCREATE TABLE IF NOT EXISTS comment (\n comment_id TEXT PRIMARY KEY,\n video_id TEXT NOT NULL REFERENCES video(video_id) ON DELETE CASCADE,\n -- NULL for a top-level comment. yt-dlp writes the string 'root'; normalising\n -- it here means a reply count is a COUNT(parent_id) rather than a filter\n -- everyone has to remember.\n parent_id TEXT,\n text TEXT NOT NULL DEFAULT '',\n like_count INTEGER NOT NULL DEFAULT 0,\n author TEXT NOT NULL DEFAULT '',\n author_id TEXT,\n author_url TEXT,\n author_thumbnail TEXT,\n author_is_uploader INTEGER NOT NULL DEFAULT 0,\n author_is_verified INTEGER NOT NULL DEFAULT 0,\n is_pinned INTEGER NOT NULL DEFAULT 0,\n is_favorited INTEGER NOT NULL DEFAULT 0,\n time_text TEXT,\n published_at INTEGER,\n harvested_at INTEGER NOT NULL\n) STRICT;\n\nCREATE INDEX IF NOT EXISTS comment_by_video ON comment(video_id, parent_id, like_count DESC);\nCREATE INDEX IF NOT EXISTS comment_by_parent ON comment(parent_id);\nCREATE INDEX IF NOT EXISTS comment_by_author ON comment(author_id);\nCREATE INDEX IF NOT EXISTS comment_by_time ON comment(video_id, published_at);\n\nCREATE VIRTUAL TABLE IF NOT EXISTS comment_fts USING fts5(\n text,\n content = 'comment',\n content_rowid = 'rowid',\n tokenize = \"unicode61 remove_diacritics 2\"\n);\n\n-- Triggers rather than a periodic 'rebuild'. Measured: triggers cost 6.2x on\n-- bulk insert (2.3s vs 366ms per 200k rows) while 'rebuild' is O(whole table)\n-- and would take seconds per checkpoint at millions of rows. Against a network\n-- that delivers ~17 comments/second, the trigger cost is noise, and there is no\n-- \"did we remember to rebuild\" failure mode.\nCREATE TRIGGER IF NOT EXISTS comment_ai AFTER INSERT ON comment BEGIN\n INSERT INTO comment_fts(rowid, text) VALUES (new.rowid, new.text);\nEND;\nCREATE TRIGGER IF NOT EXISTS comment_ad AFTER DELETE ON comment BEGIN\n INSERT INTO comment_fts(comment_fts, rowid, text) VALUES ('delete', old.rowid, old.text);\nEND;\nCREATE TRIGGER IF NOT EXISTS comment_au AFTER UPDATE ON comment BEGIN\n INSERT INTO comment_fts(comment_fts, rowid, text) VALUES ('delete', old.rowid, old.text);\n INSERT INTO comment_fts(rowid, text) VALUES (new.rowid, new.text);\nEND;\n\n-- What this store can prove about one target. Written in the SAME transaction\n-- as the rows it describes, so there is no interval in which a receipt can\n-- claim something the store does not hold.\nCREATE TABLE IF NOT EXISTS harvest_receipt (\n scope TEXT NOT NULL,\n target_id TEXT NOT NULL,\n state TEXT NOT NULL,\n reason TEXT NOT NULL,\n have INTEGER NOT NULL DEFAULT 0,\n -- What the SOURCE stated. NULL when it stated nothing; never derived from\n -- 'have', which would turn \"we stopped\" into \"there was nothing more\".\n expected INTEGER,\n expected_source TEXT,\n source TEXT NOT NULL,\n limit_applied INTEGER,\n sort_applied TEXT,\n ran_to_exhaustion INTEGER,\n resume_token TEXT,\n started_at INTEGER NOT NULL,\n finished_at INTEGER,\n attempts INTEGER NOT NULL DEFAULT 0,\n error_code TEXT,\n note TEXT,\n PRIMARY KEY (scope, target_id)\n) STRICT;\n\nCREATE INDEX IF NOT EXISTS receipt_by_state ON harvest_receipt(state, finished_at DESC);\n\n-- Append-only audit trail: cheap, and it is what makes a receipt defensible\n-- after the fact rather than merely asserted.\nCREATE TABLE IF NOT EXISTS harvest_event (\n event_id INTEGER PRIMARY KEY AUTOINCREMENT,\n scope TEXT NOT NULL,\n target_id TEXT NOT NULL,\n at INTEGER NOT NULL,\n state TEXT NOT NULL,\n delta INTEGER NOT NULL DEFAULT 0,\n note TEXT\n) STRICT;\n\nCREATE INDEX IF NOT EXISTS event_by_target ON harvest_event(scope, target_id, at DESC);\n";
16
+ //# sourceMappingURL=store-schema.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store-schema.d.ts","sourceRoot":"","sources":["../../src/utils/store-schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,6FAA6F;AAC7F,eAAO,MAAM,aAAa,IAAI,CAAC;AAE/B,eAAO,MAAM,aAAa,iIAUhB,CAAC;AAEX,eAAO,MAAM,YAAY,0vLA8IxB,CAAC"}
@@ -0,0 +1,168 @@
1
+ /**
2
+ * The shape of the harvested store, as one reviewable unit.
3
+ *
4
+ * Kept apart from the module that opens the database so the schema can be read
5
+ * end to end without the connection handling in the way.
6
+ *
7
+ * `STRICT` on every table: SQLite's default type affinity would silently accept
8
+ * a string where a count belongs, and this store exists to make claims about
9
+ * counts. It has no BOOLEAN, so flags are 0/1 integers decoded explicitly on
10
+ * the way out (see store-rows.ts) rather than cast.
11
+ */
12
+ /** Bumped only by a migration in store-migrations.ts. Independent of the server's semver. */
13
+ export const STORE_VERSION = 1;
14
+ export const STORE_PRAGMAS = [
15
+ // One writer and many readers across processes, which rollback journalling
16
+ // cannot do. Required for the "kill the server mid-harvest" guarantee.
17
+ 'PRAGMA journal_mode = WAL',
18
+ // A crash loses at most the last committed transaction and can never corrupt
19
+ // the file. FULL would fsync per commit for a store whose worst case is
20
+ // re-fetching one video.
21
+ 'PRAGMA synchronous = NORMAL',
22
+ 'PRAGMA foreign_keys = ON',
23
+ 'PRAGMA temp_store = MEMORY',
24
+ ];
25
+ export const STORE_SCHEMA = `
26
+ CREATE TABLE IF NOT EXISTS meta (
27
+ key TEXT PRIMARY KEY,
28
+ value TEXT NOT NULL
29
+ ) STRICT;
30
+
31
+ CREATE TABLE IF NOT EXISTS channel (
32
+ channel_id TEXT PRIMARY KEY,
33
+ handle TEXT NOT NULL DEFAULT '',
34
+ name TEXT NOT NULL DEFAULT '',
35
+ channel_url TEXT NOT NULL DEFAULT '',
36
+ subscriber_count INTEGER,
37
+ description TEXT NOT NULL DEFAULT '',
38
+ avatar_url TEXT,
39
+ banner_url TEXT,
40
+ first_seen_at INTEGER NOT NULL,
41
+ updated_at INTEGER NOT NULL
42
+ ) STRICT;
43
+
44
+ CREATE TABLE IF NOT EXISTS video (
45
+ video_id TEXT PRIMARY KEY,
46
+ channel_id TEXT REFERENCES channel(channel_id) ON DELETE CASCADE,
47
+ title TEXT NOT NULL DEFAULT '',
48
+ duration_s INTEGER,
49
+ -- 'YYYY-MM-DD'. NULL, never '', when a flat listing carried none: a date
50
+ -- filter must be able to tell "not known yet" from "known to be empty".
51
+ upload_date TEXT,
52
+ view_count INTEGER,
53
+ like_count INTEGER,
54
+ -- YouTube's own claim, read WITHOUT --write-comments, which overwrites it
55
+ -- with the number extracted. This is the denominator every comment receipt
56
+ -- is measured against, so it may only ever come from a metadata pass.
57
+ comment_count INTEGER,
58
+ live_status TEXT,
59
+ availability TEXT,
60
+ thumbnail_url TEXT,
61
+ tab TEXT,
62
+ also_in_tabs TEXT NOT NULL DEFAULT '[]',
63
+ catalog_rank INTEGER,
64
+ listed_at INTEGER,
65
+ detail_at INTEGER,
66
+ -- Stripped before storage: formats, automatic_captions, subtitles,
67
+ -- thumbnails and comments are 96% of a raw -j payload and are mostly
68
+ -- expiring URLs. See MAX_DETAIL_JSON_BYTES.
69
+ detail_json TEXT,
70
+ heatmap_json TEXT
71
+ ) STRICT;
72
+
73
+ CREATE INDEX IF NOT EXISTS video_by_channel ON video(channel_id, catalog_rank);
74
+ CREATE INDEX IF NOT EXISTS video_by_date ON video(channel_id, upload_date DESC);
75
+
76
+ CREATE TABLE IF NOT EXISTS comment (
77
+ comment_id TEXT PRIMARY KEY,
78
+ video_id TEXT NOT NULL REFERENCES video(video_id) ON DELETE CASCADE,
79
+ -- NULL for a top-level comment. yt-dlp writes the string 'root'; normalising
80
+ -- it here means a reply count is a COUNT(parent_id) rather than a filter
81
+ -- everyone has to remember.
82
+ parent_id TEXT,
83
+ text TEXT NOT NULL DEFAULT '',
84
+ like_count INTEGER NOT NULL DEFAULT 0,
85
+ author TEXT NOT NULL DEFAULT '',
86
+ author_id TEXT,
87
+ author_url TEXT,
88
+ author_thumbnail TEXT,
89
+ author_is_uploader INTEGER NOT NULL DEFAULT 0,
90
+ author_is_verified INTEGER NOT NULL DEFAULT 0,
91
+ is_pinned INTEGER NOT NULL DEFAULT 0,
92
+ is_favorited INTEGER NOT NULL DEFAULT 0,
93
+ time_text TEXT,
94
+ published_at INTEGER,
95
+ harvested_at INTEGER NOT NULL
96
+ ) STRICT;
97
+
98
+ CREATE INDEX IF NOT EXISTS comment_by_video ON comment(video_id, parent_id, like_count DESC);
99
+ CREATE INDEX IF NOT EXISTS comment_by_parent ON comment(parent_id);
100
+ CREATE INDEX IF NOT EXISTS comment_by_author ON comment(author_id);
101
+ CREATE INDEX IF NOT EXISTS comment_by_time ON comment(video_id, published_at);
102
+
103
+ CREATE VIRTUAL TABLE IF NOT EXISTS comment_fts USING fts5(
104
+ text,
105
+ content = 'comment',
106
+ content_rowid = 'rowid',
107
+ tokenize = "unicode61 remove_diacritics 2"
108
+ );
109
+
110
+ -- Triggers rather than a periodic 'rebuild'. Measured: triggers cost 6.2x on
111
+ -- bulk insert (2.3s vs 366ms per 200k rows) while 'rebuild' is O(whole table)
112
+ -- and would take seconds per checkpoint at millions of rows. Against a network
113
+ -- that delivers ~17 comments/second, the trigger cost is noise, and there is no
114
+ -- "did we remember to rebuild" failure mode.
115
+ CREATE TRIGGER IF NOT EXISTS comment_ai AFTER INSERT ON comment BEGIN
116
+ INSERT INTO comment_fts(rowid, text) VALUES (new.rowid, new.text);
117
+ END;
118
+ CREATE TRIGGER IF NOT EXISTS comment_ad AFTER DELETE ON comment BEGIN
119
+ INSERT INTO comment_fts(comment_fts, rowid, text) VALUES ('delete', old.rowid, old.text);
120
+ END;
121
+ CREATE TRIGGER IF NOT EXISTS comment_au AFTER UPDATE ON comment BEGIN
122
+ INSERT INTO comment_fts(comment_fts, rowid, text) VALUES ('delete', old.rowid, old.text);
123
+ INSERT INTO comment_fts(rowid, text) VALUES (new.rowid, new.text);
124
+ END;
125
+
126
+ -- What this store can prove about one target. Written in the SAME transaction
127
+ -- as the rows it describes, so there is no interval in which a receipt can
128
+ -- claim something the store does not hold.
129
+ CREATE TABLE IF NOT EXISTS harvest_receipt (
130
+ scope TEXT NOT NULL,
131
+ target_id TEXT NOT NULL,
132
+ state TEXT NOT NULL,
133
+ reason TEXT NOT NULL,
134
+ have INTEGER NOT NULL DEFAULT 0,
135
+ -- What the SOURCE stated. NULL when it stated nothing; never derived from
136
+ -- 'have', which would turn "we stopped" into "there was nothing more".
137
+ expected INTEGER,
138
+ expected_source TEXT,
139
+ source TEXT NOT NULL,
140
+ limit_applied INTEGER,
141
+ sort_applied TEXT,
142
+ ran_to_exhaustion INTEGER,
143
+ resume_token TEXT,
144
+ started_at INTEGER NOT NULL,
145
+ finished_at INTEGER,
146
+ attempts INTEGER NOT NULL DEFAULT 0,
147
+ error_code TEXT,
148
+ note TEXT,
149
+ PRIMARY KEY (scope, target_id)
150
+ ) STRICT;
151
+
152
+ CREATE INDEX IF NOT EXISTS receipt_by_state ON harvest_receipt(state, finished_at DESC);
153
+
154
+ -- Append-only audit trail: cheap, and it is what makes a receipt defensible
155
+ -- after the fact rather than merely asserted.
156
+ CREATE TABLE IF NOT EXISTS harvest_event (
157
+ event_id INTEGER PRIMARY KEY AUTOINCREMENT,
158
+ scope TEXT NOT NULL,
159
+ target_id TEXT NOT NULL,
160
+ at INTEGER NOT NULL,
161
+ state TEXT NOT NULL,
162
+ delta INTEGER NOT NULL DEFAULT 0,
163
+ note TEXT
164
+ ) STRICT;
165
+
166
+ CREATE INDEX IF NOT EXISTS event_by_target ON harvest_event(scope, target_id, at DESC);
167
+ `;
168
+ //# sourceMappingURL=store-schema.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store-schema.js","sourceRoot":"","sources":["../../src/utils/store-schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,6FAA6F;AAC7F,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC;AAE/B,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,2EAA2E;IAC3E,uEAAuE;IACvE,2BAA2B;IAC3B,6EAA6E;IAC7E,wEAAwE;IACxE,yBAAyB;IACzB,6BAA6B;IAC7B,0BAA0B;IAC1B,4BAA4B;CACpB,CAAC;AAEX,MAAM,CAAC,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8I3B,CAAC"}
@@ -0,0 +1,34 @@
1
+ import type { DatabaseSync } from 'node:sqlite';
2
+ /**
3
+ * The one connection to the harvested store.
4
+ *
5
+ * `node:sqlite` is a runtime builtin, so this adds nothing to package.json —
6
+ * the same category as `node:fs`. It is reached through a dynamic import
7
+ * behind an async accessor so that a Node without it still loads every module
8
+ * and still serves the tools that do not touch the store; only the harvest
9
+ * tools fail, with a sentence naming the version to install.
10
+ *
11
+ * The reason for a database rather than more JSON files is transactional
12
+ * receipts. `brain-build.ts` carries a long comment explaining that its
13
+ * manifest and its chunks are two documents, only one can be written first,
14
+ * and a crash in between strands a brain. Here the rows and the receipt that
15
+ * describes them commit together, so that failure is not representable.
16
+ */
17
+ /** The exact Node where `node:sqlite` is unflagged and `StatementSync.iterate()` exists. */
18
+ export declare const MIN_NODE_VERSION = "22.13.0";
19
+ export declare function readUserVersion(database: DatabaseSync): number;
20
+ export declare function writeUserVersion(database: DatabaseSync, version: number): void;
21
+ /** The process-wide store connection, opened on first use. */
22
+ export declare function getStore(): Promise<DatabaseSync>;
23
+ export declare function closeStore(): void;
24
+ /**
25
+ * Moves a damaged database aside and starts a fresh one.
26
+ *
27
+ * Renames rather than deletes. Unlike `search-index.json`, which is derivable
28
+ * and is correctly rebuilt from disk, a harvest costs hours of network that no
29
+ * local cache can replay — so nothing here removes one without being asked.
30
+ */
31
+ export declare function quarantineStore(now?: number): Promise<string>;
32
+ /** Runs `work` in a transaction, rolling back if it throws. */
33
+ export declare function inTransaction<T>(database: DatabaseSync, work: () => T): T;
34
+ //# sourceMappingURL=store.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/utils/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAQhD;;;;;;;;;;;;;;GAcG;AAEH,4FAA4F;AAC5F,eAAO,MAAM,gBAAgB,YAAY,CAAC;AAsG1C,wBAAgB,eAAe,CAAC,QAAQ,EAAE,YAAY,GAAG,MAAM,CAI9D;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAI9E;AAED,8DAA8D;AAC9D,wBAAsB,QAAQ,IAAI,OAAO,CAAC,YAAY,CAAC,CAGtD;AAED,wBAAgB,UAAU,IAAI,IAAI,CAGjC;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,GAAG,SAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAKvE;AAED,+DAA+D;AAC/D,wBAAgB,aAAa,CAAC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAUzE"}
@@ -0,0 +1,157 @@
1
+ import { chmod, rename } from 'node:fs/promises';
2
+ import { YouTubeError } from './errors.js';
3
+ import { ensurePrivateDir } from './paths.js';
4
+ import { corruptStorePath, storeDatabasePath, storeDir } from './store-paths.js';
5
+ import { STORE_PRAGMAS, STORE_SCHEMA, STORE_VERSION } from './store-schema.js';
6
+ import { runStoreMigrations } from './store-migrations.js';
7
+ /**
8
+ * The one connection to the harvested store.
9
+ *
10
+ * `node:sqlite` is a runtime builtin, so this adds nothing to package.json —
11
+ * the same category as `node:fs`. It is reached through a dynamic import
12
+ * behind an async accessor so that a Node without it still loads every module
13
+ * and still serves the tools that do not touch the store; only the harvest
14
+ * tools fail, with a sentence naming the version to install.
15
+ *
16
+ * The reason for a database rather than more JSON files is transactional
17
+ * receipts. `brain-build.ts` carries a long comment explaining that its
18
+ * manifest and its chunks are two documents, only one can be written first,
19
+ * and a crash in between strands a brain. Here the rows and the receipt that
20
+ * describes them commit together, so that failure is not representable.
21
+ */
22
+ /** The exact Node where `node:sqlite` is unflagged and `StatementSync.iterate()` exists. */
23
+ export const MIN_NODE_VERSION = '22.13.0';
24
+ const BUSY_TIMEOUT_MS = 5_000;
25
+ const PRIVATE_FILE_MODE = 0o600;
26
+ let db;
27
+ let warningFilterInstalled = false;
28
+ /**
29
+ * Silence only Node's own "SQLite is experimental" notice.
30
+ *
31
+ * It goes to stderr, so it cannot corrupt the stdio protocol, but it is noise
32
+ * in every client's log. Matched on both the name and the message so that an
33
+ * unrelated ExperimentalWarning still reaches whoever was listening for it.
34
+ *
35
+ * Only listeners registered before the first store open are wrapped — which is
36
+ * the one that matters, Node's own default printer. A listener added later
37
+ * sees every warning, including this one, and that is the right trade: this
38
+ * exists to quieten a default, not to hide a warning from someone who went
39
+ * looking for it.
40
+ */
41
+ function installWarningFilter() {
42
+ if (warningFilterInstalled)
43
+ return;
44
+ warningFilterInstalled = true;
45
+ const existing = process.listeners('warning');
46
+ for (const listener of existing)
47
+ process.removeListener('warning', listener);
48
+ process.on('warning', (warning) => {
49
+ if (warning.name === 'ExperimentalWarning' && warning.message.includes('SQLite'))
50
+ return;
51
+ for (const listener of existing)
52
+ listener(warning);
53
+ });
54
+ }
55
+ async function loadSqlite() {
56
+ try {
57
+ installWarningFilter();
58
+ // Annotated rather than inferred: a bare dynamic import widens to `any`,
59
+ // which would let every later misuse of the module through unchecked.
60
+ const sqlite = await import('node:sqlite');
61
+ return sqlite;
62
+ }
63
+ catch {
64
+ throw new YouTubeError('STORE_UNAVAILABLE', 'This Node build has no node:sqlite, which the harvest tools need.', {
65
+ nextStep: `Install Node ${MIN_NODE_VERSION} or newer (this process is ${process.version}). Every other tool works without it.`,
66
+ });
67
+ }
68
+ }
69
+ function isCorruption(error) {
70
+ const message = error instanceof Error ? error.message : String(error);
71
+ return /malformed|not a database|SQLITE_CORRUPT|SQLITE_NOTADB/i.test(message);
72
+ }
73
+ function assertUsableVersion(found) {
74
+ if (found <= STORE_VERSION)
75
+ return;
76
+ // Operating on a schema written by a newer server is how data gets
77
+ // destroyed. Refusing is the only safe move, and naming both versions is
78
+ // what makes the refusal actionable.
79
+ throw new YouTubeError('STORE_CORRUPT', `This store was written by a newer version of the server (store v${String(found)}; this build understands v${String(STORE_VERSION)}).`, { nextStep: 'Upgrade youtube-knowledge-mcp, or point HOME at a different data directory.' });
80
+ }
81
+ async function openDatabase() {
82
+ const sqlite = await loadSqlite();
83
+ await ensurePrivateDir(storeDir());
84
+ const path = storeDatabasePath();
85
+ let opened;
86
+ try {
87
+ opened = new sqlite.DatabaseSync(path, {
88
+ timeout: BUSY_TIMEOUT_MS,
89
+ enableForeignKeyConstraints: true,
90
+ });
91
+ for (const pragma of STORE_PRAGMAS)
92
+ opened.exec(pragma);
93
+ }
94
+ catch (error) {
95
+ if (!isCorruption(error))
96
+ throw error;
97
+ throw new YouTubeError('STORE_CORRUPT', 'The harvested store could not be opened.', {
98
+ nextStep: 'Call repair_store to move the damaged file aside and start a fresh store.',
99
+ cause: error,
100
+ });
101
+ }
102
+ const version = readUserVersion(opened);
103
+ assertUsableVersion(version);
104
+ opened.exec(STORE_SCHEMA);
105
+ runStoreMigrations(opened, version);
106
+ // The database holds harvested personal data — commenter names, ids and
107
+ // free text — so it gets the same bits as the transcript cache rather than
108
+ // the umask default.
109
+ await chmod(path, PRIVATE_FILE_MODE).catch(() => undefined);
110
+ return opened;
111
+ }
112
+ export function readUserVersion(database) {
113
+ const row = database.prepare('PRAGMA user_version').get();
114
+ const value = row?.user_version;
115
+ return typeof value === 'number' ? value : 0;
116
+ }
117
+ export function writeUserVersion(database, version) {
118
+ // PRAGMA does not accept a bound parameter, so the value is constrained to an
119
+ // integer here rather than interpolated from anything a caller controls.
120
+ database.exec(`PRAGMA user_version = ${String(Math.trunc(version))}`);
121
+ }
122
+ /** The process-wide store connection, opened on first use. */
123
+ export async function getStore() {
124
+ db ??= await openDatabase();
125
+ return db;
126
+ }
127
+ export function closeStore() {
128
+ db?.close();
129
+ db = undefined;
130
+ }
131
+ /**
132
+ * Moves a damaged database aside and starts a fresh one.
133
+ *
134
+ * Renames rather than deletes. Unlike `search-index.json`, which is derivable
135
+ * and is correctly rebuilt from disk, a harvest costs hours of network that no
136
+ * local cache can replay — so nothing here removes one without being asked.
137
+ */
138
+ export async function quarantineStore(now = Date.now()) {
139
+ closeStore();
140
+ const target = corruptStorePath(now);
141
+ await rename(storeDatabasePath(), target);
142
+ return target;
143
+ }
144
+ /** Runs `work` in a transaction, rolling back if it throws. */
145
+ export function inTransaction(database, work) {
146
+ database.exec('BEGIN');
147
+ try {
148
+ const result = work();
149
+ database.exec('COMMIT');
150
+ return result;
151
+ }
152
+ catch (error) {
153
+ database.exec('ROLLBACK');
154
+ throw error;
155
+ }
156
+ }
157
+ //# sourceMappingURL=store.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/utils/store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACjF,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D;;;;;;;;;;;;;;GAcG;AAEH,4FAA4F;AAC5F,MAAM,CAAC,MAAM,gBAAgB,GAAG,SAAS,CAAC;AAE1C,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,iBAAiB,GAAG,KAAK,CAAC;AAEhC,IAAI,EAA4B,CAAC;AACjC,IAAI,sBAAsB,GAAG,KAAK,CAAC;AAEnC;;;;;;;;;;;;GAYG;AACH,SAAS,oBAAoB;IAC3B,IAAI,sBAAsB;QAAE,OAAO;IACnC,sBAAsB,GAAG,IAAI,CAAC;IAE9B,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAC9C,KAAK,MAAM,QAAQ,IAAI,QAAQ;QAAE,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAE7E,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAc,EAAE,EAAE;QACvC,IAAI,OAAO,CAAC,IAAI,KAAK,qBAAqB,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO;QACzF,KAAK,MAAM,QAAQ,IAAI,QAAQ;YAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI,CAAC;QACH,oBAAoB,EAAE,CAAC;QACvB,yEAAyE;QACzE,sEAAsE;QACtE,MAAM,MAAM,GAAiC,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzE,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,YAAY,CACpB,mBAAmB,EACnB,mEAAmE,EACnE;YACE,QAAQ,EAAE,gBAAgB,gBAAgB,8BAA8B,OAAO,CAAC,OAAO,uCAAuC;SAC/H,CACF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,wDAAwD,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACxC,IAAI,KAAK,IAAI,aAAa;QAAE,OAAO;IAEnC,mEAAmE;IACnE,yEAAyE;IACzE,qCAAqC;IACrC,MAAM,IAAI,YAAY,CACpB,eAAe,EACf,mEAAmE,MAAM,CAAC,KAAK,CAAC,6BAA6B,MAAM,CAAC,aAAa,CAAC,IAAI,EACtI,EAAE,QAAQ,EAAE,6EAA6E,EAAE,CAC5F,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY;IACzB,MAAM,MAAM,GAAG,MAAM,UAAU,EAAE,CAAC;IAClC,MAAM,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,iBAAiB,EAAE,CAAC;IAEjC,IAAI,MAAoB,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE;YACrC,OAAO,EAAE,eAAe;YACxB,2BAA2B,EAAE,IAAI;SAClC,CAAC,CAAC;QACH,KAAK,MAAM,MAAM,IAAI,aAAa;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;YAAE,MAAM,KAAK,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,eAAe,EAAE,0CAA0C,EAAE;YAClF,QAAQ,EAAE,2EAA2E;YACrF,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IACxC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1B,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEpC,wEAAwE;IACxE,2EAA2E;IAC3E,qBAAqB;IACrB,MAAM,KAAK,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC5D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAAsB;IACpD,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,GAAG,EAAE,CAAC;IAC1D,MAAM,KAAK,GAAG,GAAG,EAAE,YAAY,CAAC;IAChC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAAsB,EAAE,OAAe;IACtE,8EAA8E;IAC9E,yEAAyE;IACzE,QAAQ,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,8DAA8D;AAC9D,MAAM,CAAC,KAAK,UAAU,QAAQ;IAC5B,EAAE,KAAK,MAAM,YAAY,EAAE,CAAC;IAC5B,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,EAAE,EAAE,KAAK,EAAE,CAAC;IACZ,EAAE,GAAG,SAAS,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;IACpD,UAAU,EAAE,CAAC;IACb,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,MAAM,CAAC,iBAAiB,EAAE,EAAE,MAAM,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+DAA+D;AAC/D,MAAM,UAAU,aAAa,CAAI,QAAsB,EAAE,IAAa;IACpE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxB,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1B,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}