youtube-knowledge-mcp 2.2.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -18
- package/dist/harvest-schemas.d.ts +75 -0
- package/dist/harvest-schemas.d.ts.map +1 -0
- package/dist/harvest-schemas.js +106 -0
- package/dist/harvest-schemas.js.map +1 -0
- package/dist/registry/harvest-tools.d.ts +10 -0
- package/dist/registry/harvest-tools.d.ts.map +1 -0
- package/dist/registry/harvest-tools.js +109 -0
- package/dist/registry/harvest-tools.js.map +1 -0
- package/dist/registry/health-tools.d.ts.map +1 -1
- package/dist/registry/health-tools.js +18 -0
- package/dist/registry/health-tools.js.map +1 -1
- package/dist/registry/index.d.ts.map +1 -1
- package/dist/registry/index.js +2 -0
- package/dist/registry/index.js.map +1 -1
- package/dist/registry/video-tools.js +1 -1
- package/dist/registry/video-tools.js.map +1 -1
- package/dist/schemas.d.ts +239 -0
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +56 -0
- package/dist/schemas.js.map +1 -1
- package/dist/thumbnail-schemas.d.ts +28 -28
- package/dist/tools/ask-brain.js +1 -1
- package/dist/tools/ask-brain.js.map +1 -1
- package/dist/tools/brain-info.js +1 -1
- package/dist/tools/brain-info.js.map +1 -1
- package/dist/tools/check-health.d.ts +59 -0
- package/dist/tools/check-health.d.ts.map +1 -1
- package/dist/tools/check-health.js +85 -6
- package/dist/tools/check-health.js.map +1 -1
- package/dist/tools/fetch-channel-thumbnails.d.ts +8 -8
- package/dist/tools/fetch-videos.d.ts +15 -4
- package/dist/tools/fetch-videos.d.ts.map +1 -1
- package/dist/tools/fetch-videos.js +63 -22
- package/dist/tools/fetch-videos.js.map +1 -1
- package/dist/tools/get-comments.d.ts +238 -6
- package/dist/tools/get-comments.d.ts.map +1 -1
- package/dist/tools/get-comments.js +85 -21
- package/dist/tools/get-comments.js.map +1 -1
- package/dist/tools/get-coverage.d.ts +122 -0
- package/dist/tools/get-coverage.d.ts.map +1 -0
- package/dist/tools/get-coverage.js +105 -0
- package/dist/tools/get-coverage.js.map +1 -0
- package/dist/tools/harvest-channel.d.ts +92 -0
- package/dist/tools/harvest-channel.d.ts.map +1 -0
- package/dist/tools/harvest-channel.js +62 -0
- package/dist/tools/harvest-channel.js.map +1 -0
- package/dist/tools/harvest-comments.d.ts +81 -0
- package/dist/tools/harvest-comments.d.ts.map +1 -0
- package/dist/tools/harvest-comments.js +115 -0
- package/dist/tools/harvest-comments.js.map +1 -0
- package/dist/tools/library.js +1 -1
- package/dist/tools/library.js.map +1 -1
- package/dist/tools/list-channel-thumbnails.d.ts +10 -10
- package/dist/tools/list-channel-thumbnails.js +1 -1
- package/dist/tools/list-channel-thumbnails.js.map +1 -1
- package/dist/tools/list-library.js +1 -1
- package/dist/tools/list-library.js.map +1 -1
- package/dist/tools/prune-harvest.d.ts +59 -0
- package/dist/tools/prune-harvest.d.ts.map +1 -0
- package/dist/tools/prune-harvest.js +108 -0
- package/dist/tools/prune-harvest.js.map +1 -0
- package/dist/tools/query-comments.d.ts +125 -0
- package/dist/tools/query-comments.d.ts.map +1 -0
- package/dist/tools/query-comments.js +106 -0
- package/dist/tools/query-comments.js.map +1 -0
- package/dist/tools/query-videos.d.ts +118 -0
- package/dist/tools/query-videos.d.ts.map +1 -0
- package/dist/tools/query-videos.js +97 -0
- package/dist/tools/query-videos.js.map +1 -0
- package/dist/tools/repair-store.d.ts +34 -0
- package/dist/tools/repair-store.d.ts.map +1 -0
- package/dist/tools/repair-store.js +58 -0
- package/dist/tools/repair-store.js.map +1 -0
- package/dist/tools/search-channels.d.ts.map +1 -1
- package/dist/tools/search-channels.js +5 -1
- package/dist/tools/search-channels.js.map +1 -1
- package/dist/tools/search-videos.js +1 -1
- package/dist/tools/search-videos.js.map +1 -1
- package/dist/utils/comment-store.d.ts +71 -0
- package/dist/utils/comment-store.d.ts.map +1 -0
- package/dist/utils/comment-store.js +129 -0
- package/dist/utils/comment-store.js.map +1 -0
- package/dist/utils/comment-threads.d.ts +51 -0
- package/dist/utils/comment-threads.d.ts.map +1 -0
- package/dist/utils/comment-threads.js +76 -0
- package/dist/utils/comment-threads.js.map +1 -0
- package/dist/utils/coverage-text.d.ts +3 -0
- package/dist/utils/coverage-text.d.ts.map +1 -0
- package/dist/utils/coverage-text.js +62 -0
- package/dist/utils/coverage-text.js.map +1 -0
- package/dist/utils/coverage.d.ts +57 -0
- package/dist/utils/coverage.d.ts.map +1 -0
- package/dist/utils/coverage.js +142 -0
- package/dist/utils/coverage.js.map +1 -0
- package/dist/utils/errors.d.ts +1 -1
- package/dist/utils/errors.d.ts.map +1 -1
- package/dist/utils/errors.js +9 -1
- package/dist/utils/errors.js.map +1 -1
- package/dist/utils/format.d.ts +17 -1
- package/dist/utils/format.d.ts.map +1 -1
- package/dist/utils/format.js +1 -1
- package/dist/utils/format.js.map +1 -1
- package/dist/utils/harvest-catalog.d.ts +44 -0
- package/dist/utils/harvest-catalog.d.ts.map +1 -0
- package/dist/utils/harvest-catalog.js +164 -0
- package/dist/utils/harvest-catalog.js.map +1 -0
- package/dist/utils/harvest-lock.d.ts +3 -0
- package/dist/utils/harvest-lock.d.ts.map +1 -0
- package/dist/utils/harvest-lock.js +26 -0
- package/dist/utils/harvest-lock.js.map +1 -0
- package/dist/utils/harvest-receipts.d.ts +14 -0
- package/dist/utils/harvest-receipts.d.ts.map +1 -0
- package/dist/utils/harvest-receipts.js +110 -0
- package/dist/utils/harvest-receipts.js.map +1 -0
- package/dist/utils/listing-cursor.d.ts +31 -0
- package/dist/utils/listing-cursor.d.ts.map +1 -0
- package/dist/utils/listing-cursor.js +39 -0
- package/dist/utils/listing-cursor.js.map +1 -0
- package/dist/utils/pot-preflight.d.ts +53 -0
- package/dist/utils/pot-preflight.d.ts.map +1 -0
- package/dist/utils/pot-preflight.js +146 -0
- package/dist/utils/pot-preflight.js.map +1 -0
- package/dist/utils/sleep.d.ts +9 -0
- package/dist/utils/sleep.d.ts.map +1 -0
- package/dist/utils/sleep.js +25 -0
- package/dist/utils/sleep.js.map +1 -0
- package/dist/utils/store-health.d.ts +37 -0
- package/dist/utils/store-health.d.ts.map +1 -0
- package/dist/utils/store-health.js +92 -0
- package/dist/utils/store-health.js.map +1 -0
- package/dist/utils/store-migrations.d.ts +33 -0
- package/dist/utils/store-migrations.d.ts.map +1 -0
- package/dist/utils/store-migrations.js +41 -0
- package/dist/utils/store-migrations.js.map +1 -0
- package/dist/utils/store-paths.d.ts +33 -0
- package/dist/utils/store-paths.d.ts.map +1 -0
- package/dist/utils/store-paths.js +48 -0
- package/dist/utils/store-paths.js.map +1 -0
- package/dist/utils/store-rows.d.ts +22 -0
- package/dist/utils/store-rows.d.ts.map +1 -0
- package/dist/utils/store-rows.js +62 -0
- package/dist/utils/store-rows.js.map +1 -0
- package/dist/utils/store-schema.d.ts +16 -0
- package/dist/utils/store-schema.d.ts.map +1 -0
- package/dist/utils/store-schema.js +168 -0
- package/dist/utils/store-schema.js.map +1 -0
- package/dist/utils/store.d.ts +34 -0
- package/dist/utils/store.d.ts.map +1 -0
- package/dist/utils/store.js +157 -0
- package/dist/utils/store.js.map +1 -0
- package/dist/utils/video-stats-cache.d.ts +16 -0
- package/dist/utils/video-stats-cache.d.ts.map +1 -0
- package/dist/utils/video-stats-cache.js +81 -0
- package/dist/utils/video-stats-cache.js.map +1 -0
- package/dist/utils/video-stats.d.ts +186 -0
- package/dist/utils/video-stats.d.ts.map +1 -0
- package/dist/utils/video-stats.js +168 -0
- package/dist/utils/video-stats.js.map +1 -0
- package/dist/utils/video-store.d.ts +69 -0
- package/dist/utils/video-store.d.ts.map +1 -0
- package/dist/utils/video-store.js +77 -0
- package/dist/utils/video-store.js.map +1 -0
- package/dist/utils/youtube-channel.d.ts +9 -2
- package/dist/utils/youtube-channel.d.ts.map +1 -1
- package/dist/utils/youtube-channel.js +33 -3
- package/dist/utils/youtube-channel.js.map +1 -1
- package/dist/utils/youtube-schemas.d.ts +98 -0
- package/dist/utils/youtube-schemas.d.ts.map +1 -1
- package/dist/utils/youtube-schemas.js +30 -0
- package/dist/utils/youtube-schemas.js.map +1 -1
- package/dist/utils/youtube-video.d.ts +22 -1
- package/dist/utils/youtube-video.d.ts.map +1 -1
- package/dist/utils/youtube-video.js +40 -28
- package/dist/utils/youtube-video.js.map +1 -1
- package/dist/utils/ytdlp-limiter.d.ts +3 -0
- package/dist/utils/ytdlp-limiter.d.ts.map +1 -1
- package/dist/utils/ytdlp-limiter.js +19 -3
- package/dist/utils/ytdlp-limiter.js.map +1 -1
- package/dist/utils/ytdlp-pacer.d.ts +64 -0
- package/dist/utils/ytdlp-pacer.d.ts.map +1 -0
- package/dist/utils/ytdlp-pacer.js +165 -0
- package/dist/utils/ytdlp-pacer.js.map +1 -0
- package/dist/utils/ytdlp-stream.d.ts +32 -0
- package/dist/utils/ytdlp-stream.d.ts.map +1 -0
- package/dist/utils/ytdlp-stream.js +61 -0
- package/dist/utils/ytdlp-stream.js.map +1 -0
- package/dist/utils/ytdlp.d.ts +10 -0
- package/dist/utils/ytdlp.d.ts.map +1 -1
- package/dist/utils/ytdlp.js +36 -25
- package/dist/utils/ytdlp.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { coverageSchema, } from '../harvest-schemas.js';
|
|
2
|
+
/**
|
|
3
|
+
* The one constructor for a completeness receipt.
|
|
4
|
+
*
|
|
5
|
+
* A wrong `complete: true` is the worst thing this server can emit: it is the
|
|
6
|
+
* difference between a consumer saying "here is a sample of the comments" and
|
|
7
|
+
* "here is the video's full comment history". So `complete` is not a field a
|
|
8
|
+
* caller sets — it is derived here, and the derivation is guarded by
|
|
9
|
+
* invariants that are asserted in code rather than merely documented.
|
|
10
|
+
*
|
|
11
|
+
* The rule that does the most work: `expected` is what the SOURCE said, and is
|
|
12
|
+
* never derived from `have`. Copying the number we collected into the field
|
|
13
|
+
* that means "how many there are" would turn "we stopped" into "there was
|
|
14
|
+
* nothing more", silently, and every check downstream would agree with it.
|
|
15
|
+
*/
|
|
16
|
+
/** A complete receipt is still only as fresh as its harvest; comments accrue. */
|
|
17
|
+
export const RECEIPT_STALE_MS = 7 * 86_400_000;
|
|
18
|
+
/** Beyond this, receipts are summarised rather than listed in a tool result. */
|
|
19
|
+
export const COVERAGE_IN_RESULT_LIMIT = 100;
|
|
20
|
+
/** Thrown for a receipt that contradicts itself. Not a YouTubeError: no caller can fix it. */
|
|
21
|
+
export class CoverageInvariantError extends Error {
|
|
22
|
+
constructor(message) {
|
|
23
|
+
super(`Coverage invariant violated: ${message}`);
|
|
24
|
+
this.name = 'CoverageInvariantError';
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
function capWasBinding(input) {
|
|
28
|
+
return input.limitApplied !== undefined && input.have >= input.limitApplied;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Whether this server can prove it holds everything in scope.
|
|
32
|
+
*
|
|
33
|
+
* Two independent ways to prove it, and a cap overrides both:
|
|
34
|
+
*
|
|
35
|
+
* - the source stated a total and we hold at least that many, or
|
|
36
|
+
* - the source's own iteration ran out (`source-exhausted`).
|
|
37
|
+
*
|
|
38
|
+
* Exhaustion alone is not proof. Measured against yt-dlp 2026.08.19: a run
|
|
39
|
+
* capped at 40 parent comments returned a non-null `comment_count` of 58,
|
|
40
|
+
* because `itertools.islice` ends iteration at the cap exactly as a natural
|
|
41
|
+
* end would. So a binding cap disqualifies completeness even when the source
|
|
42
|
+
* says it finished.
|
|
43
|
+
*/
|
|
44
|
+
function isProvablyComplete(input) {
|
|
45
|
+
if (capWasBinding(input))
|
|
46
|
+
return false;
|
|
47
|
+
if (input.expected !== undefined) {
|
|
48
|
+
if (input.expected.source === 'source-exhausted')
|
|
49
|
+
return input.ranToExhaustion !== false;
|
|
50
|
+
return input.have >= input.expected.value;
|
|
51
|
+
}
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
function derivedReason(input, complete) {
|
|
55
|
+
if (complete)
|
|
56
|
+
return 'COMPLETE';
|
|
57
|
+
if (input.reason !== undefined && input.reason !== 'COMPLETE')
|
|
58
|
+
return input.reason;
|
|
59
|
+
if (capWasBinding(input))
|
|
60
|
+
return 'CAP_REACHED';
|
|
61
|
+
// Nothing said there was more, and nothing proved there was not. That is not
|
|
62
|
+
// completeness; it is the absence of evidence, and it has its own name.
|
|
63
|
+
return 'SOURCE_SILENT';
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Checks a receipt against every invariant.
|
|
67
|
+
*
|
|
68
|
+
* Called inside `coverageOf` and again by handlers before a result goes out,
|
|
69
|
+
* because the cost of the second call is nothing and the cost of a wrong
|
|
70
|
+
* `complete: true` reaching a consumer is the whole point of this module.
|
|
71
|
+
*/
|
|
72
|
+
export function assertCoverageConsistent(coverage) {
|
|
73
|
+
const fail = (message) => {
|
|
74
|
+
throw new CoverageInvariantError(message);
|
|
75
|
+
};
|
|
76
|
+
if (coverage.complete && coverage.reason !== 'COMPLETE') {
|
|
77
|
+
fail(`complete is true but reason is ${coverage.reason}`);
|
|
78
|
+
}
|
|
79
|
+
if (!coverage.complete && coverage.reason === 'COMPLETE') {
|
|
80
|
+
fail('reason is COMPLETE but complete is false');
|
|
81
|
+
}
|
|
82
|
+
if ((coverage.expected === undefined) !== (coverage.expectedSource === undefined)) {
|
|
83
|
+
fail('expected and expectedSource must be present or absent together');
|
|
84
|
+
}
|
|
85
|
+
if (coverage.complete) {
|
|
86
|
+
const exhausted = coverage.expectedSource === 'source-exhausted';
|
|
87
|
+
const enough = coverage.expected !== undefined && coverage.have >= coverage.expected;
|
|
88
|
+
if (!exhausted && !enough) {
|
|
89
|
+
fail('complete is true without either a met expected total or an exhausted source');
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
// There is deliberately no separate "COMPLETE without an expected total"
|
|
93
|
+
// guard. It reads like a necessary invariant and is unreachable: reaching it
|
|
94
|
+
// needs reason COMPLETE, which the checks above already force to pair with
|
|
95
|
+
// complete:true, which in turn already demands either a met total or an
|
|
96
|
+
// exhausted source. An unreachable guard is not defence in depth, it is code
|
|
97
|
+
// no test can justify.
|
|
98
|
+
// One-directional on purpose. "A token exactly when incomplete" is the
|
|
99
|
+
// tempting symmetry, and it is wrong: plenty of incomplete receipts have
|
|
100
|
+
// nothing to resume — SOURCE_REFUSED, COMMENTS_DISABLED, a caller that
|
|
101
|
+
// simply did not supply one. What must never happen is a token on a
|
|
102
|
+
// complete receipt, which would invite a caller to page past the end.
|
|
103
|
+
if (coverage.resumeToken !== undefined && coverage.complete) {
|
|
104
|
+
fail('a complete receipt cannot carry a resumeToken');
|
|
105
|
+
}
|
|
106
|
+
if (coverage.limitApplied !== undefined &&
|
|
107
|
+
coverage.have >= coverage.limitApplied &&
|
|
108
|
+
coverage.reason === 'COMPLETE') {
|
|
109
|
+
fail('a binding cap cannot produce COMPLETE');
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/** Builds a validated receipt. The only way to make one. */
|
|
113
|
+
export function coverageOf(input) {
|
|
114
|
+
const complete = isProvablyComplete(input);
|
|
115
|
+
const reason = derivedReason(input, complete);
|
|
116
|
+
const harvestedAt = input.harvestedAt ?? new Date();
|
|
117
|
+
const coverage = coverageSchema.parse({
|
|
118
|
+
scope: input.scope,
|
|
119
|
+
targetId: input.targetId,
|
|
120
|
+
complete,
|
|
121
|
+
reason,
|
|
122
|
+
have: input.have,
|
|
123
|
+
...(input.expected === undefined
|
|
124
|
+
? {}
|
|
125
|
+
: { expected: input.expected.value, expectedSource: input.expected.source }),
|
|
126
|
+
...(input.ranToExhaustion === undefined ? {} : { ranToExhaustion: input.ranToExhaustion }),
|
|
127
|
+
source: input.source,
|
|
128
|
+
harvestedAt: harvestedAt.toISOString(),
|
|
129
|
+
...(complete
|
|
130
|
+
? { staleAfter: new Date(harvestedAt.getTime() + RECEIPT_STALE_MS).toISOString() }
|
|
131
|
+
: {}),
|
|
132
|
+
...(input.limitApplied === undefined ? {} : { limitApplied: input.limitApplied }),
|
|
133
|
+
...(input.sortApplied === undefined ? {} : { sortApplied: input.sortApplied }),
|
|
134
|
+
// A resume token is meaningless on a complete receipt, and its absence is
|
|
135
|
+
// how a caller knows there is nothing left to ask for.
|
|
136
|
+
...(complete || input.resumeToken === undefined ? {} : { resumeToken: input.resumeToken }),
|
|
137
|
+
...(input.note === undefined ? {} : { note: input.note }),
|
|
138
|
+
});
|
|
139
|
+
assertCoverageConsistent(coverage);
|
|
140
|
+
return coverage;
|
|
141
|
+
}
|
|
142
|
+
//# sourceMappingURL=coverage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coverage.js","sourceRoot":"","sources":["../../src/utils/coverage.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,GAKf,MAAM,uBAAuB,CAAC;AAE/B;;;;;;;;;;;;;GAaG;AAEH,iFAAiF;AACjF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,GAAG,UAAU,CAAC;AAE/C,gFAAgF;AAChF,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AA2B5C,8FAA8F;AAC9F,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAC/C,YAAY,OAAe;QACzB,KAAK,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED,SAAS,aAAa,CAAC,KAAoB;IACzC,OAAO,KAAK,CAAC,YAAY,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,YAAY,CAAC;AAC9E,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,kBAAkB,CAAC,KAAoB;IAC9C,IAAI,aAAa,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACjC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,kBAAkB;YAAE,OAAO,KAAK,CAAC,eAAe,KAAK,KAAK,CAAC;QACzF,OAAO,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC5C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAAoB,EAAE,QAAiB;IAC5D,IAAI,QAAQ;QAAE,OAAO,UAAU,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU;QAAE,OAAO,KAAK,CAAC,MAAM,CAAC;IACnF,IAAI,aAAa,CAAC,KAAK,CAAC;QAAE,OAAO,aAAa,CAAC;IAC/C,6EAA6E;IAC7E,wEAAwE;IACxE,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CAAC,QAAkB;IACzD,MAAM,IAAI,GAAG,CAAC,OAAe,EAAS,EAAE;QACtC,MAAM,IAAI,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC,CAAC;IAEF,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACxD,IAAI,CAAC,kCAAkC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACzD,IAAI,CAAC,0CAA0C,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,cAAc,KAAK,SAAS,CAAC,EAAE,CAAC;QAClF,IAAI,CAAC,gEAAgE,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACtB,MAAM,SAAS,GAAG,QAAQ,CAAC,cAAc,KAAK,kBAAkB,CAAC;QACjE,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,QAAQ,CAAC;QACrF,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,6EAA6E,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;IACD,yEAAyE;IACzE,6EAA6E;IAC7E,2EAA2E;IAC3E,wEAAwE;IACxE,6EAA6E;IAC7E,uBAAuB;IACvB,uEAAuE;IACvE,yEAAyE;IACzE,uEAAuE;IACvE,oEAAoE;IACpE,sEAAsE;IACtE,IAAI,QAAQ,CAAC,WAAW,KAAK,SAAS,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC5D,IAAI,CAAC,+CAA+C,CAAC,CAAC;IACxD,CAAC;IACD,IACE,QAAQ,CAAC,YAAY,KAAK,SAAS;QACnC,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,YAAY;QACtC,QAAQ,CAAC,MAAM,KAAK,UAAU,EAC9B,CAAC;QACD,IAAI,CAAC,uCAAuC,CAAC,CAAC;IAChD,CAAC;AACH,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,UAAU,CAAC,KAAoB;IAC7C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC9C,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,IAAI,IAAI,EAAE,CAAC;IAEpD,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC;QACpC,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,QAAQ;QACR,MAAM;QACN,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS;YAC9B,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC9E,GAAG,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC;QAC1F,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE;QACtC,GAAG,CAAC,QAAQ;YACV,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,gBAAgB,CAAC,CAAC,WAAW,EAAE,EAAE;YAClF,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,KAAK,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC;QACjF,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;QAC9E,0EAA0E;QAC1E,uDAAuD;QACvD,GAAG,CAAC,QAAQ,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;QAC1F,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;KAC1D,CAAC,CAAC;IAEH,wBAAwB,CAAC,QAAQ,CAAC,CAAC;IACnC,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
package/dist/utils/errors.d.ts
CHANGED
|
@@ -7,7 +7,7 @@ import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
|
|
7
7
|
* than a wall of yt-dlp stderr it has to guess at — and never a leaked command
|
|
8
8
|
* line or stack trace.
|
|
9
9
|
*/
|
|
10
|
-
export type YouTubeErrorCode = 'PRIVATE' | 'AGE_GATED' | 'MEMBERS_ONLY' | 'PREMIUM_ONLY' | 'LOGIN_REQUIRED' | 'BOT_CHECK' | 'NOT_FOUND' | 'NO_CAPTIONS' | 'LIVE_NOT_ENDED' | 'RATE_LIMITED' | 'TIMEOUT' | 'CANCELLED' | 'YTDLP_MISSING' | 'FFMPEG_MISSING' | 'YTDLP_FAILED' | 'FETCH_FAILED' | 'MALFORMED_RESPONSE' | 'INVALID_INPUT';
|
|
10
|
+
export type YouTubeErrorCode = 'PRIVATE' | 'AGE_GATED' | 'MEMBERS_ONLY' | 'PREMIUM_ONLY' | 'LOGIN_REQUIRED' | 'BOT_CHECK' | 'NOT_FOUND' | 'NO_CAPTIONS' | 'LIVE_NOT_ENDED' | 'RATE_LIMITED' | 'TIMEOUT' | 'CANCELLED' | 'YTDLP_MISSING' | 'FFMPEG_MISSING' | 'YTDLP_FAILED' | 'FETCH_FAILED' | 'MALFORMED_RESPONSE' | 'INVALID_INPUT' | 'STORE_UNAVAILABLE' | 'STORE_CORRUPT';
|
|
11
11
|
export interface YouTubeErrorOptions {
|
|
12
12
|
/** What the caller should try next. Shown to the model verbatim. */
|
|
13
13
|
nextStep?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAEzE;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GACxB,SAAS,GACT,WAAW,GACX,cAAc,GACd,cAAc,GACd,gBAAgB,GAChB,WAAW,GACX,WAAW,GACX,aAAa,GACb,gBAAgB,GAChB,cAAc,GACd,SAAS,GACT,WAAW,GACX,eAAe,GACf,gBAAgB,GAChB,cAAc,GACd,cAAc,GACd,oBAAoB,GACpB,eAAe,CAAC;AAEpB,MAAM,WAAW,mBAAmB;IAClC,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8DAA8D;IAC9D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACrC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;gBAEhB,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB;IAQtF,iFAAiF;IACjF,aAAa,IAAI,MAAM;CAGxB;
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAEzE;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GACxB,SAAS,GACT,WAAW,GACX,cAAc,GACd,cAAc,GACd,gBAAgB,GAChB,WAAW,GACX,WAAW,GACX,aAAa,GACb,gBAAgB,GAChB,cAAc,GACd,SAAS,GACT,WAAW,GACX,eAAe,GACf,gBAAgB,GAChB,cAAc,GACd,cAAc,GACd,oBAAoB,GACpB,eAAe,GACf,mBAAmB,GACnB,eAAe,CAAC;AAEpB,MAAM,WAAW,mBAAmB;IAClC,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8DAA8D;IAC9D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACrC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;gBAEhB,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB;IAQtF,iFAAiF;IACjF,aAAa,IAAI,MAAM;CAGxB;AAwGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,GAAG,YAAY,CAkBlF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,YAAY,GAAG,SAAS,CA2C3B;AAED,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,CAQ3D;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,cAAc,CAM1D"}
|
package/dist/utils/errors.js
CHANGED
|
@@ -31,7 +31,15 @@ const RULES = [
|
|
|
31
31
|
emits: 'Sign in to confirm you',
|
|
32
32
|
source: 'playabilityStatus.reason, re-raised by yt_dlp/extractor/youtube/_video.py',
|
|
33
33
|
message: 'YouTube is asking this client to prove it is not a bot.',
|
|
34
|
-
|
|
34
|
+
// Ordered by what was measured on 2026-09-06, not by what sounds most
|
|
35
|
+
// technical. From a datacenter address every account-free lever was
|
|
36
|
+
// installed and tested — a PO token provider (yt-dlp never even reaches
|
|
37
|
+
// the point of requesting one; it fails at the player response first),
|
|
38
|
+
// eight player clients, a 15-minute cooldown, and curl_cffi TLS
|
|
39
|
+
// impersonation — and none of them changed the answer. So the address
|
|
40
|
+
// comes first and the PO token provider comes last: leading with a remedy
|
|
41
|
+
// that was measured not to work sends the reader down a dead end.
|
|
42
|
+
nextStep: `Per-video reads are refused from this address. Datacenter and VPN/exit-node addresses are gated regardless of client settings, so check whether this host egresses through one. Otherwise: ${COOKIE_HINT} On a residential address a missing PO token provider can be the cause instead — check \`check_health\`.`,
|
|
35
43
|
},
|
|
36
44
|
{
|
|
37
45
|
code: 'RATE_LIMITED',
|
package/dist/utils/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"AAwCA,MAAM,OAAO,YAAa,SAAQ,KAAK;IAC5B,IAAI,CAAmB;IACvB,QAAQ,CAAU;IAClB,SAAS,CAAU;IAE5B,YAAY,IAAsB,EAAE,OAAe,EAAE,UAA+B,EAAE;QACpF,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAC9C,CAAC;IAED,iFAAiF;IACjF,aAAa;QACX,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;IAC9E,CAAC;CACF;AA8BD,6EAA6E;AAC7E,MAAM,WAAW,GACf,4JAA4J,CAAC;AAE/J,MAAM,KAAK,GAAW;IACpB;QACE,yEAAyE;QACzE,wEAAwE;QACxE,wEAAwE;QACxE,qEAAqE;QACrE,sEAAsE;QACtE,wEAAwE;QACxE,2EAA2E;QAC3E,sEAAsE;QACtE,qCAAqC;QACrC,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,2EAA2E;QACnF,OAAO,EAAE,yDAAyD;QAClE,sEAAsE;QACtE,oEAAoE;QACpE,wEAAwE;QACxE,uEAAuE;QACvE,gEAAgE;QAChE,sEAAsE;QACtE,0EAA0E;QAC1E,kEAAkE;QAClE,QAAQ,EAAE,8LAA8L,WAAW,0GAA0G;KAC9T;IACD;QACE,IAAI,EAAE,cAAc;QACpB,4EAA4E;QAC5E,wDAAwD;QACxD,KAAK,EAAE,gBAAgB;QACvB,MAAM,EAAE,oCAAoC;QAC5C,OAAO,EAAE,uCAAuC;QAChD,QAAQ,EAAE,kEAAkE;QAC5E,SAAS,EAAE,IAAI;KAChB;IACD;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,oDAAoD;QAC3D,MAAM,EAAE,0BAA0B;QAClC,OAAO,EAAE,qEAAqE;QAC9E,QAAQ,EACN,uFAAuF;KAC1F;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,6BAA6B;QACpC,MAAM,EAAE,0BAA0B;QAClC,OAAO,EAAE,6DAA6D;QACtE,QAAQ,EAAE,gEAAgE;KAC3E;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,8EAA8E;QACrF,MAAM,EAAE,oCAAoC;QAC5C,OAAO,EAAE,6DAA6D;QACtE,QAAQ,EAAE,gEAAgE;KAC3E;IACD;QACE,oEAAoE;QACpE,yEAAyE;QACzE,yEAAyE;QACzE,iEAAiE;QACjE,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,mDAAmD;QAC1D,MAAM,EAAE,uCAAuC;QAC/C,OAAO,EAAE,kDAAkD;QAC3D,QAAQ,EAAE,sDAAsD,WAAW,EAAE;KAC9E;CACF,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAc,EAAE,KAAe;IAClE,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAChD,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE;gBAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,yCAAyC,EAAE;QACjF,QAAQ,EACN,kLAAkL;QACpL,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAGnC;IACC,QAAQ,MAAM,CAAC,YAAY,EAAE,CAAC;QAC5B,KAAK,SAAS;YACZ,OAAO,IAAI,YAAY,CAAC,SAAS,EAAE,wBAAwB,EAAE;gBAC3D,QAAQ,EAAE,0EAA0E;aACrF,CAAC,CAAC;QACL,KAAK,iBAAiB;YACpB,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,8CAA8C,EAAE;gBACtF,QAAQ,EAAE,iEAAiE;aAC5E,CAAC,CAAC;QACL,KAAK,cAAc;YACjB,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,gDAAgD,EAAE;gBACxF,QAAQ,EAAE,4DAA4D;aACvE,CAAC,CAAC;QACL,KAAK,YAAY;YACf,OAAO,IAAI,YAAY,CACrB,WAAW,EACX,gEAAgE,EAChE;gBACE,QAAQ,EAAE,2DAA2D,WAAW,EAAE;aACnF,CACF,CAAC;IACN,CAAC;IAED,4EAA4E;IAC5E,gDAAgD;IAChD,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1B,KAAK,aAAa;YAChB,OAAO,IAAI,YAAY,CAAC,gBAAgB,EAAE,uCAAuC,EAAE;gBACjF,QAAQ,EAAE,qEAAqE;aAChF,CAAC,CAAC;QACL,KAAK,WAAW;YACd,OAAO,IAAI,YAAY,CACrB,gBAAgB,EAChB,qDAAqD,EACrD;gBACE,QAAQ,EAAE,+DAA+D;gBACzE,SAAS,EAAE,IAAI;aAChB,CACF,CAAC;IACN,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,IAAI,KAAK,YAAY,YAAY;QAAE,OAAO,KAAK,CAAC;IAEhD,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,sCAAsC,EAAE;QAC9E,QAAQ,EAAE,0EAA0E;QACpF,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC;KAC3D,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,KAAc;IACxC,MAAM,YAAY,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAC3C,OAAO;QACL,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,YAAY,CAAC,IAAI,KAAK,YAAY,CAAC,aAAa,EAAE,EAAE,EAAE,CAAC;KAC5F,CAAC;AACJ,CAAC"}
|
package/dist/utils/format.d.ts
CHANGED
|
@@ -20,7 +20,23 @@ export interface PageInfo {
|
|
|
20
20
|
hasMore: boolean;
|
|
21
21
|
nextOffset?: number;
|
|
22
22
|
}
|
|
23
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Takes an object, not three adjacent numbers.
|
|
25
|
+
*
|
|
26
|
+
* The bug this replaces was not in the arithmetic: it was five call sites that
|
|
27
|
+
* passed `count` where `total` belonged, so `hasMore` computed `n < n` and a
|
|
28
|
+
* caller with 20 of a channel's 5,000 videos was told it had all of them.
|
|
29
|
+
* Positional arguments of the same type made that invisible; an object makes
|
|
30
|
+
* it impossible, and changing the signature forced every call site to be
|
|
31
|
+
* revisited, which was the point.
|
|
32
|
+
*/
|
|
33
|
+
export interface PageInput {
|
|
34
|
+
/** Items available in total. Never `count` — that is the bug this shape prevents. */
|
|
35
|
+
total: number;
|
|
36
|
+
count: number;
|
|
37
|
+
offset?: number;
|
|
38
|
+
}
|
|
39
|
+
export declare function pageInfo({ total, count, offset }: PageInput): PageInfo;
|
|
24
40
|
/** A link to a file this server produced, so clients can offer to open it. */
|
|
25
41
|
export declare function fileResourceLink(filePath: string, name: string, mimeType?: string): {
|
|
26
42
|
type: 'resource_link';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/utils/format.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAEzE,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKjD;AAOD,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAG1D;AAED,wBAAgB,cAAc,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAKrD;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,cAAc,CAK5F;AAED,wEAAwE;AACxE,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,
|
|
1
|
+
{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/utils/format.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAEzE,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKjD;AAOD,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAG1D;AAED,wBAAgB,cAAc,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAKrD;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,cAAc,CAK5F;AAED,wEAAwE;AACxE,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,SAAS;IACxB,qFAAqF;IACrF,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,QAAQ,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAU,EAAE,EAAE,SAAS,GAAG,QAAQ,CAS1E;AAED,8EAA8E;AAC9E,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,MAAM,GAChB;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAOzE;AAED,wEAAwE;AACxE,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GACtD,cAAc,CAQhB"}
|
package/dist/utils/format.js
CHANGED
|
@@ -40,7 +40,7 @@ export function toolResult(text, structured) {
|
|
|
40
40
|
structuredContent: structured,
|
|
41
41
|
};
|
|
42
42
|
}
|
|
43
|
-
export function pageInfo(total, count, offset = 0) {
|
|
43
|
+
export function pageInfo({ total, count, offset = 0 }) {
|
|
44
44
|
const hasMore = offset + count < total;
|
|
45
45
|
return {
|
|
46
46
|
total,
|
package/dist/utils/format.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/utils/format.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,IAAI,KAAK,IAAI,aAAa;QAAE,OAAO,GAAG,gBAAgB,CAAC,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC;IACjF,IAAI,KAAK,IAAI,SAAS;QAAE,OAAO,GAAG,gBAAgB,CAAC,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC;IACzE,IAAI,KAAK,IAAI,KAAK;QAAE,OAAO,GAAG,gBAAgB,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC;IACjE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;AAC1B,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAS;IACjC,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3B,OAAO,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAChD,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAClE,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAChF,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;AACzD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,UAAmC;IAC1E,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,CAAC;QAC1C,iBAAiB,EAAE,UAAU;KAC9B,CAAC;AACJ,CAAC;
|
|
1
|
+
{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/utils/format.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,IAAI,KAAK,IAAI,aAAa;QAAE,OAAO,GAAG,gBAAgB,CAAC,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC;IACjF,IAAI,KAAK,IAAI,SAAS;QAAE,OAAO,GAAG,gBAAgB,CAAC,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC;IACzE,IAAI,KAAK,IAAI,KAAK;QAAE,OAAO,GAAG,gBAAgB,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC;IACjE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;AAC1B,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAS;IACjC,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3B,OAAO,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAChD,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAClE,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAChF,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;AACzD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,UAAmC;IAC1E,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,CAAC;QAC1C,iBAAiB,EAAE,UAAU;KAC9B,CAAC;AACJ,CAAC;AA4BD,MAAM,UAAU,QAAQ,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,GAAG,CAAC,EAAa;IAC9D,MAAM,OAAO,GAAG,MAAM,GAAG,KAAK,GAAG,KAAK,CAAC;IACvC,OAAO;QACL,KAAK;QACL,KAAK;QACL,MAAM;QACN,OAAO;QACP,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,gBAAgB,CAC9B,QAAgB,EAChB,IAAY,EACZ,QAAiB;IAEjB,OAAO;QACL,IAAI,EAAE,eAAe;QACrB,GAAG,EAAE,UAAU,QAAQ,EAAE;QACzB,IAAI;QACJ,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;KAChD,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,UAAU,CACxB,IAAY,EACZ,UAAmC,EACnC,IAAuD;IAEvD,OAAO;QACL,OAAO,EAAE;YACP,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE;YAC/B,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC;SACtD;QACD,iBAAiB,EAAE,UAAU;KAC9B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import type { Coverage } from '../harvest-schemas.js';
|
|
3
|
+
/**
|
|
4
|
+
* Reading a channel's whole catalogue.
|
|
5
|
+
*
|
|
6
|
+
* The listing plane is cheap and is not bot-checked: measured 1,716 videos in
|
|
7
|
+
* 31 seconds from one spawn, so a catalogue costs about a minute per channel
|
|
8
|
+
* and the old 100-video cap was self-imposed rather than forced.
|
|
9
|
+
*
|
|
10
|
+
* Every tab is walked because a channel's videos are not all under /videos —
|
|
11
|
+
* shorts, streams, releases and podcasts each have their own. The first tab
|
|
12
|
+
* that lists a video owns it; later tabs record themselves in `also_in_tabs`.
|
|
13
|
+
* Last-writer-wins would mislabel most of a music channel's catalogue.
|
|
14
|
+
*/
|
|
15
|
+
export declare const HARVEST_TABS: readonly ["videos", "shorts", "streams", "releases", "podcasts"];
|
|
16
|
+
export type HarvestTab = (typeof HARVEST_TABS)[number];
|
|
17
|
+
export declare const MAX_VIDEOS_PER_CATALOG = 10000;
|
|
18
|
+
export interface TabResult {
|
|
19
|
+
tab: HarvestTab;
|
|
20
|
+
listed: number;
|
|
21
|
+
added: number;
|
|
22
|
+
complete: boolean;
|
|
23
|
+
error?: string;
|
|
24
|
+
}
|
|
25
|
+
export interface CatalogResult {
|
|
26
|
+
channelId: string;
|
|
27
|
+
name: string;
|
|
28
|
+
handle: string;
|
|
29
|
+
tabs: TabResult[];
|
|
30
|
+
videosSeen: number;
|
|
31
|
+
videosAdded: number;
|
|
32
|
+
coverage: Coverage;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Walks every tab of a channel into the store.
|
|
36
|
+
*
|
|
37
|
+
* A tab the channel simply does not have is not a failure — yt-dlp says so in
|
|
38
|
+
* words, and treating that as an error would make every channel without a
|
|
39
|
+
* podcasts tab look broken.
|
|
40
|
+
*/
|
|
41
|
+
export declare function harvestCatalog(database: DatabaseSync, channel: string, options?: {
|
|
42
|
+
maxVideos?: number;
|
|
43
|
+
}): Promise<CatalogResult>;
|
|
44
|
+
//# sourceMappingURL=harvest-catalog.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harvest-catalog.d.ts","sourceRoot":"","sources":["../../src/utils/harvest-catalog.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAahD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEtD;;;;;;;;;;;GAWG;AAEH,eAAO,MAAM,YAAY,kEAAmE,CAAC;AAC7F,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvD,eAAO,MAAM,sBAAsB,QAAS,CAAC;AAK7C,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,UAAU,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAyCD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GACnC,OAAO,CAAC,aAAa,CAAC,CAQxB"}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { runYtDlp, TIMEOUTS } from './ytdlp.js';
|
|
2
|
+
import { FLAT_PRINT_TEMPLATE, flatEntrySchema, toVideoListItem } from './flat-listing.js';
|
|
3
|
+
import { parseYtDlpJsonLines } from './ytdlp-parse.js';
|
|
4
|
+
import { resolveListTarget } from './youtube-url.js';
|
|
5
|
+
import { getChannelInfo } from './youtube-channel.js';
|
|
6
|
+
import { coverageOf } from './coverage.js';
|
|
7
|
+
import { saveReceipt } from './harvest-receipts.js';
|
|
8
|
+
import { inTransaction } from './store.js';
|
|
9
|
+
import { throwIfAborted, reportProgress } from './context.js';
|
|
10
|
+
import { countRows } from './store-rows.js';
|
|
11
|
+
import { withChannelHarvestLock } from './harvest-lock.js';
|
|
12
|
+
import { YouTubeError } from './errors.js';
|
|
13
|
+
/**
|
|
14
|
+
* Reading a channel's whole catalogue.
|
|
15
|
+
*
|
|
16
|
+
* The listing plane is cheap and is not bot-checked: measured 1,716 videos in
|
|
17
|
+
* 31 seconds from one spawn, so a catalogue costs about a minute per channel
|
|
18
|
+
* and the old 100-video cap was self-imposed rather than forced.
|
|
19
|
+
*
|
|
20
|
+
* Every tab is walked because a channel's videos are not all under /videos —
|
|
21
|
+
* shorts, streams, releases and podcasts each have their own. The first tab
|
|
22
|
+
* that lists a video owns it; later tabs record themselves in `also_in_tabs`.
|
|
23
|
+
* Last-writer-wins would mislabel most of a music channel's catalogue.
|
|
24
|
+
*/
|
|
25
|
+
export const HARVEST_TABS = ['videos', 'shorts', 'streams', 'releases', 'podcasts'];
|
|
26
|
+
export const MAX_VIDEOS_PER_CATALOG = 10_000;
|
|
27
|
+
/** yt-dlp says this, in these words, for a tab the channel does not have. */
|
|
28
|
+
const NO_SUCH_TAB = 'does not have a';
|
|
29
|
+
const UPSERT_VIDEO = `
|
|
30
|
+
INSERT INTO video (
|
|
31
|
+
video_id, channel_id, title, duration_s, upload_date, view_count,
|
|
32
|
+
live_status, thumbnail_url, tab, also_in_tabs, catalog_rank, listed_at
|
|
33
|
+
) VALUES (?,?,?,?,?,?,?,?,?,'[]',?,?)
|
|
34
|
+
ON CONFLICT(video_id) DO UPDATE SET
|
|
35
|
+
title = excluded.title,
|
|
36
|
+
duration_s = excluded.duration_s,
|
|
37
|
+
view_count = excluded.view_count,
|
|
38
|
+
live_status = excluded.live_status,
|
|
39
|
+
thumbnail_url = excluded.thumbnail_url,
|
|
40
|
+
listed_at = excluded.listed_at,
|
|
41
|
+
-- The owning tab is kept; a later tab only records that it also lists this
|
|
42
|
+
-- video, so a re-walk in a different order cannot relabel the catalogue.
|
|
43
|
+
also_in_tabs = CASE
|
|
44
|
+
WHEN video.tab = excluded.tab THEN video.also_in_tabs
|
|
45
|
+
WHEN instr(video.also_in_tabs, excluded.tab) > 0 THEN video.also_in_tabs
|
|
46
|
+
ELSE json_insert(video.also_in_tabs, '$[#]', excluded.tab)
|
|
47
|
+
END`;
|
|
48
|
+
async function listTab(channelUrl, tab, limit) {
|
|
49
|
+
return runYtDlp([
|
|
50
|
+
'--skip-download',
|
|
51
|
+
'--flat-playlist',
|
|
52
|
+
'--lazy-playlist',
|
|
53
|
+
'--print',
|
|
54
|
+
FLAT_PRINT_TEMPLATE,
|
|
55
|
+
'--playlist-end',
|
|
56
|
+
String(limit),
|
|
57
|
+
], {
|
|
58
|
+
label: `harvest_channel:${tab}`,
|
|
59
|
+
timeoutMs: TIMEOUTS.transcript,
|
|
60
|
+
target: resolveListTarget(`${channelUrl}/${tab}`),
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Walks every tab of a channel into the store.
|
|
65
|
+
*
|
|
66
|
+
* A tab the channel simply does not have is not a failure — yt-dlp says so in
|
|
67
|
+
* words, and treating that as an error would make every channel without a
|
|
68
|
+
* podcasts tab look broken.
|
|
69
|
+
*/
|
|
70
|
+
export async function harvestCatalog(database, channel, options = {}) {
|
|
71
|
+
const maxVideos = Math.min(options.maxVideos ?? 500, MAX_VIDEOS_PER_CATALOG);
|
|
72
|
+
// Resolving the channel comes before the lock because the lock is keyed on
|
|
73
|
+
// the channel id, and only YouTube can turn a handle into one.
|
|
74
|
+
const info = await getChannelInfo(channel);
|
|
75
|
+
return withChannelHarvestLock(info.channelId, () => walkTabs(database, info, maxVideos));
|
|
76
|
+
}
|
|
77
|
+
async function walkTabs(database, info, maxVideos) {
|
|
78
|
+
const now = Date.now();
|
|
79
|
+
database
|
|
80
|
+
.prepare(`INSERT INTO channel (channel_id, handle, name, channel_url, subscriber_count, description, first_seen_at, updated_at)
|
|
81
|
+
VALUES (?,?,?,?,?,?,?,?)
|
|
82
|
+
ON CONFLICT(channel_id) DO UPDATE SET
|
|
83
|
+
handle = excluded.handle, name = excluded.name, channel_url = excluded.channel_url,
|
|
84
|
+
subscriber_count = excluded.subscriber_count, description = excluded.description,
|
|
85
|
+
updated_at = excluded.updated_at`)
|
|
86
|
+
.run(info.channelId, info.handle, info.name, info.channelUrl, info.subscriberCount, info.description, now, now);
|
|
87
|
+
const tabs = [];
|
|
88
|
+
const seen = new Set();
|
|
89
|
+
let rank = 0;
|
|
90
|
+
// Counted from the store, not from `seen`: on a second run every video is
|
|
91
|
+
// new to the walk and none is new to the catalogue, and reporting the former
|
|
92
|
+
// as "added" would make a re-run look like a fresh harvest.
|
|
93
|
+
const countVideos = () => countRows(database.prepare('SELECT COUNT(*) FROM video WHERE channel_id = ?'), info.channelId);
|
|
94
|
+
const before = countVideos();
|
|
95
|
+
for (const tab of HARVEST_TABS) {
|
|
96
|
+
throwIfAborted();
|
|
97
|
+
reportProgress(tabs.length, HARVEST_TABS.length, `harvest_channel: ${tab}`);
|
|
98
|
+
let stdout;
|
|
99
|
+
try {
|
|
100
|
+
stdout = await listTab(info.channelUrl, tab, maxVideos);
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
const message = error instanceof YouTubeError ? error.message : String(error);
|
|
104
|
+
const absent = error instanceof YouTubeError && error.message.includes(NO_SUCH_TAB);
|
|
105
|
+
tabs.push({
|
|
106
|
+
tab,
|
|
107
|
+
listed: 0,
|
|
108
|
+
added: 0,
|
|
109
|
+
// A tab the channel does not have is a complete answer, not a gap.
|
|
110
|
+
complete: absent,
|
|
111
|
+
...(absent ? {} : { error: message }),
|
|
112
|
+
});
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
const entries = parseYtDlpJsonLines(stdout, flatEntrySchema).map(toVideoListItem);
|
|
116
|
+
const insert = database.prepare(UPSERT_VIDEO);
|
|
117
|
+
const tabBefore = countVideos();
|
|
118
|
+
inTransaction(database, () => {
|
|
119
|
+
for (const entry of entries) {
|
|
120
|
+
if (seen.size >= maxVideos)
|
|
121
|
+
break;
|
|
122
|
+
seen.add(entry.id);
|
|
123
|
+
insert.run(entry.id, info.channelId, entry.title, entry.duration, entry.uploadDate === '' ? null : entry.uploadDate, entry.viewCount ?? null, entry.liveStatus ?? null, entry.thumbnailUrl ?? null, tab, rank, now);
|
|
124
|
+
rank += 1;
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
tabs.push({
|
|
128
|
+
tab,
|
|
129
|
+
listed: entries.length,
|
|
130
|
+
added: countVideos() - tabBefore,
|
|
131
|
+
complete: entries.length < maxVideos,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
const everyTabComplete = tabs.every((tab) => tab.complete);
|
|
135
|
+
const coverage = coverageOf({
|
|
136
|
+
scope: 'channel-catalog',
|
|
137
|
+
targetId: info.channelId,
|
|
138
|
+
have: seen.size,
|
|
139
|
+
source: `yt-dlp --flat-playlist over ${HARVEST_TABS.join(', ')}`,
|
|
140
|
+
// Only an unbroken walk of every tab proves the catalogue is whole. A tab
|
|
141
|
+
// that errored, or one that filled to the cap, leaves it unprovable.
|
|
142
|
+
...(everyTabComplete
|
|
143
|
+
? {
|
|
144
|
+
expected: { value: seen.size, source: 'source-exhausted' },
|
|
145
|
+
ranToExhaustion: true,
|
|
146
|
+
}
|
|
147
|
+
: {}),
|
|
148
|
+
...(seen.size >= maxVideos ? { limitApplied: maxVideos } : {}),
|
|
149
|
+
...(everyTabComplete ? {} : { resumeToken: `catalog:${info.channelId}` }),
|
|
150
|
+
});
|
|
151
|
+
inTransaction(database, () => {
|
|
152
|
+
saveReceipt(database, coverage, { startedAt: now });
|
|
153
|
+
});
|
|
154
|
+
return {
|
|
155
|
+
channelId: info.channelId,
|
|
156
|
+
name: info.name,
|
|
157
|
+
handle: info.handle,
|
|
158
|
+
tabs,
|
|
159
|
+
videosSeen: seen.size,
|
|
160
|
+
videosAdded: countVideos() - before,
|
|
161
|
+
coverage,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
//# sourceMappingURL=harvest-catalog.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harvest-catalog.js","sourceRoot":"","sources":["../../src/utils/harvest-catalog.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC1F,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C;;;;;;;;;;;GAWG;AAEH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAU,CAAC;AAG7F,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAE7C,6EAA6E;AAC7E,MAAM,WAAW,GAAG,iBAAiB,CAAC;AAoBtC,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;MAkBf,CAAC;AAEP,KAAK,UAAU,OAAO,CAAC,UAAkB,EAAE,GAAe,EAAE,KAAa;IACvE,OAAO,QAAQ,CACb;QACE,iBAAiB;QACjB,iBAAiB;QACjB,iBAAiB;QACjB,SAAS;QACT,mBAAmB;QACnB,gBAAgB;QAChB,MAAM,CAAC,KAAK,CAAC;KACd,EACD;QACE,KAAK,EAAE,mBAAmB,GAAG,EAAE;QAC/B,SAAS,EAAE,QAAQ,CAAC,UAAU;QAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,UAAU,IAAI,GAAG,EAAE,CAAC;KAClD,CACF,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAsB,EACtB,OAAe,EACf,UAAkC,EAAE;IAEpC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,EAAE,sBAAsB,CAAC,CAAC;IAE7E,2EAA2E;IAC3E,+DAA+D;IAC/D,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAE3C,OAAO,sBAAsB,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,QAAsB,EACtB,IAAgD,EAChD,SAAiB;IAEjB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,QAAQ;SACL,OAAO,CACN;;;;;0CAKoC,CACrC;SACA,GAAG,CACF,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,GAAG,EACH,GAAG,CACJ,CAAC;IAEJ,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,0EAA0E;IAC1E,6EAA6E;IAC7E,4DAA4D;IAC5D,MAAM,WAAW,GAAG,GAAW,EAAE,CAC/B,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,iDAAiD,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACjG,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAC/B,cAAc,EAAE,CAAC;QACjB,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,oBAAoB,GAAG,EAAE,CAAC,CAAC;QAE5E,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QAC1D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9E,MAAM,MAAM,GAAG,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YACpF,IAAI,CAAC,IAAI,CAAC;gBACR,GAAG;gBACH,MAAM,EAAE,CAAC;gBACT,KAAK,EAAE,CAAC;gBACR,mEAAmE;gBACnE,QAAQ,EAAE,MAAM;gBAChB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;aACtC,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC;QAEhC,aAAa,CAAC,QAAQ,EAAE,GAAG,EAAE;YAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,IAAI,IAAI,SAAS;oBAAE,MAAM;gBAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACnB,MAAM,CAAC,GAAG,CACR,KAAK,CAAC,EAAE,EACR,IAAI,CAAC,SAAS,EACd,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,QAAQ,EACd,KAAK,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,EACjD,KAAK,CAAC,SAAS,IAAI,IAAI,EACvB,KAAK,CAAC,UAAU,IAAI,IAAI,EACxB,KAAK,CAAC,YAAY,IAAI,IAAI,EAC1B,GAAG,EACH,IAAI,EACJ,GAAG,CACJ,CAAC;gBACF,IAAI,IAAI,CAAC,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC;YACR,GAAG;YACH,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,WAAW,EAAE,GAAG,SAAS;YAChC,QAAQ,EAAE,OAAO,CAAC,MAAM,GAAG,SAAS;SACrC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,UAAU,CAAC;QAC1B,KAAK,EAAE,iBAAiB;QACxB,QAAQ,EAAE,IAAI,CAAC,SAAS;QACxB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,+BAA+B,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAChE,0EAA0E;QAC1E,qEAAqE;QACrE,GAAG,CAAC,gBAAgB;YAClB,CAAC,CAAC;gBACE,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,kBAA2B,EAAE;gBACnE,eAAe,EAAE,IAAI;aACtB;YACH,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,WAAW,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;KAC1E,CAAC,CAAC;IAEH,aAAa,CAAC,QAAQ,EAAE,GAAG,EAAE;QAC3B,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,IAAI;QACJ,UAAU,EAAE,IAAI,CAAC,IAAI;QACrB,WAAW,EAAE,WAAW,EAAE,GAAG,MAAM;QACnC,QAAQ;KACT,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harvest-lock.d.ts","sourceRoot":"","sources":["../../src/utils/harvest-lock.ts"],"names":[],"mappings":"AAsBA,wBAAsB,sBAAsB,CAAC,CAAC,EAC5C,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACrB,OAAO,CAAC,CAAC,CAAC,CAKZ;AAED,wBAAsB,oBAAoB,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAKjG"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { YouTubeError } from './errors.js';
|
|
2
|
+
import { ensurePrivateDir } from './paths.js';
|
|
3
|
+
import { withFileLock } from './file-lock.js';
|
|
4
|
+
import { channelHarvestLockPath, storeDir, videoHarvestLockPath } from './store-paths.js';
|
|
5
|
+
/**
|
|
6
|
+
* Exclusive access to one harvest target.
|
|
7
|
+
*
|
|
8
|
+
* WAL already keeps concurrent writes consistent, so this is not about the
|
|
9
|
+
* rows. It is about the requests: two harvests of the same channel would each
|
|
10
|
+
* spawn yt-dlp and double the rate at the one resource that throttles us.
|
|
11
|
+
*/
|
|
12
|
+
function alreadyRunning(what, id, existing) {
|
|
13
|
+
const owner = existing === undefined ? '' : `Started at ${existing.startedAt} by process ${existing.pid}. `;
|
|
14
|
+
return new YouTubeError('INVALID_INPUT', `A harvest of ${what} ${id} is already running.`, {
|
|
15
|
+
nextStep: `${owner}Wait for it to finish, then call the tool again to continue where it stopped.`,
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
export async function withChannelHarvestLock(channelId, work) {
|
|
19
|
+
await ensurePrivateDir(storeDir());
|
|
20
|
+
return withFileLock(channelHarvestLockPath(channelId), work, (existing) => alreadyRunning('channel', channelId, existing));
|
|
21
|
+
}
|
|
22
|
+
export async function withVideoHarvestLock(videoId, work) {
|
|
23
|
+
await ensurePrivateDir(storeDir());
|
|
24
|
+
return withFileLock(videoHarvestLockPath(videoId), work, (existing) => alreadyRunning('video', videoId, existing));
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=harvest-lock.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harvest-lock.js","sourceRoot":"","sources":["../../src/utils/harvest-lock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAmB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,sBAAsB,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE1F;;;;;;GAMG;AAEH,SAAS,cAAc,CAAC,IAAY,EAAE,EAAU,EAAE,QAAgC;IAChF,MAAM,KAAK,GACT,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,QAAQ,CAAC,SAAS,eAAe,QAAQ,CAAC,GAAG,IAAI,CAAC;IAEhG,OAAO,IAAI,YAAY,CAAC,eAAe,EAAE,gBAAgB,IAAI,IAAI,EAAE,sBAAsB,EAAE;QACzF,QAAQ,EAAE,GAAG,KAAK,+EAA+E;KAClG,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,SAAiB,EACjB,IAAsB;IAEtB,MAAM,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnC,OAAO,YAAY,CAAC,sBAAsB,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,EAAE,CACxE,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,CAAC,CAC/C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAI,OAAe,EAAE,IAAsB;IACnF,MAAM,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnC,OAAO,YAAY,CAAC,oBAAoB,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,EAAE,CACpE,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAC3C,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import { type Coverage } from '../harvest-schemas.js';
|
|
3
|
+
export declare function saveReceipt(database: DatabaseSync, coverage: Coverage, options?: {
|
|
4
|
+
startedAt?: number;
|
|
5
|
+
errorCode?: string;
|
|
6
|
+
}): void;
|
|
7
|
+
export declare function readReceipt(database: DatabaseSync, scope: Coverage['scope'], targetId: string): Coverage | undefined;
|
|
8
|
+
export declare function listReceipts(database: DatabaseSync, options?: {
|
|
9
|
+
scope?: Coverage['scope'];
|
|
10
|
+
limit?: number;
|
|
11
|
+
offset?: number;
|
|
12
|
+
}): Coverage[];
|
|
13
|
+
export declare function countReceipts(database: DatabaseSync, scope?: Coverage['scope']): number;
|
|
14
|
+
//# sourceMappingURL=harvest-receipts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harvest-receipts.d.ts","sourceRoot":"","sources":["../../src/utils/harvest-receipts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAsC,KAAK,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAgE1F,wBAAgB,WAAW,CACzB,QAAQ,EAAE,YAAY,EACtB,QAAQ,EAAE,QAAQ,EAClB,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GACvD,IAAI,CAsCN;AAuBD,wBAAgB,WAAW,CACzB,QAAQ,EAAE,YAAY,EACtB,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,EACxB,QAAQ,EAAE,MAAM,GACf,QAAQ,GAAG,SAAS,CAQtB;AAED,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAC3E,QAAQ,EAAE,CAwBZ;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,MAAM,CAavF"}
|