youtube-knowledge-mcp 3.0.0 → 3.0.1

package/dist/utils/detail-batch.d.ts

@@ -0,0 +1,25 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { CompletenessReason } from '../harvest-schemas.js';
+import { type YouTubeErrorCode } from './errors.js';
+import { fetchVideoStatsRow } from './video-stats.js';
+/** Reading one batch of videos and keeping what came back, a batch at a time. */
+export interface DetailFailure {
+    videoId: string;
+    code: YouTubeErrorCode;
+}
+/** Refusals that say the next video will be refused too: stop rather than hammer. */
+export declare const STOPPING: Partial<Record<YouTubeErrorCode, CompletenessReason>>;
+export type Read = {
+    videoId: string;
+    row: Awaited<ReturnType<typeof fetchVideoStatsRow>>;
+} | {
+    videoId: string;
+    code: YouTubeErrorCode;
+};
+export declare function read(videoId: string): Promise<Read>;
+export declare function stoppingCode(batch: Read[]): YouTubeErrorCode | undefined;
+export declare function persist(database: DatabaseSync, batch: Read[]): {
+    stored: number;
+    failures: DetailFailure[];
+};
+//# sourceMappingURL=detail-batch.d.ts.map

package/dist/utils/detail-batch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"detail-batch.d.ts","sourceRoot":"","sources":["../../src/utils/detail-batch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAqC,KAAK,gBAAgB,EAAE,MAAM,aAAa,CAAC;AASvF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,iFAAiF;AAEjF,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,gBAAgB,CAAC;CACxB;AAED,qFAAqF;AACrF,eAAO,MAAM,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,CAK1E,CAAC;AAEF,MAAM,MAAM,IAAI,GACZ;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAC,CAAA;CAAE,GACxE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,gBAAgB,CAAA;CAAE,CAAC;AAEhD,wBAAsB,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAczD;AAUD,wBAAgB,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,gBAAgB,GAAG,SAAS,CAIxE;AAED,wBAAgB,OAAO,CACrB,QAAQ,EAAE,YAAY,EACtB,KAAK,EAAE,IAAI,EAAE,GACZ;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,aAAa,EAAE,CAAA;CAAE,CAqB/C"}

package/dist/utils/detail-batch.js

@@ -0,0 +1,58 @@
+import { YouTubeError, classifyPlayability } from './errors.js';
+import { inTransaction } from './store.js';
+import { READ_FAILED_AVAILABILITY, UNREADABLE_AVAILABILITY, markUnreadable, saveVideoDetail, } from './video-detail-store.js';
+import { fetchVideoStatsRow } from './video-stats.js';
+/** Refusals that say the next video will be refused too: stop rather than hammer. */
+export const STOPPING = {
+    BOT_CHECK: 'SOURCE_REFUSED',
+    LOGIN_REQUIRED: 'SOURCE_REFUSED',
+    RATE_LIMITED: 'RATE_LIMITED',
+    CANCELLED: 'CANCELLED',
+};
+export async function read(videoId) {
+    try {
+        // Tolerant, so a private or members-only video comes back as a row whose
+        // availability says so, not as a failure that asks to sign in and would be taken
+        // for a problem with the whole session.
+        const row = await fetchVideoStatsRow(videoId, { tolerateRefusal: true });
+        const refusal = classifyPlayability({
+            availability: row.availability ?? undefined,
+            liveStatus: row.live_status ?? undefined,
+        });
+        return refusal === undefined ? { videoId, row } : { videoId, code: refusal.code };
+    }
+    catch (error) {
+        return { videoId, code: error instanceof YouTubeError ? error.code : 'YTDLP_FAILED' };
+    }
+}
+/** What to record for a video that failed, so it is not tried first on every run. */
+function unreadableMarker(code) {
+    return (Object.entries(UNREADABLE_AVAILABILITY).find(([key]) => key === code)?.[1] ??
+        READ_FAILED_AVAILABILITY);
+}
+export function stoppingCode(batch) {
+    return batch
+        .flatMap((outcome) => ('code' in outcome ? [outcome.code] : []))
+        .find((code) => STOPPING[code] !== undefined);
+}
+export function persist(database, batch) {
+    const failures = [];
+    let stored = 0;
+    inTransaction(database, () => {
+        for (const outcome of batch) {
+            if ('row' in outcome) {
+                saveVideoDetail(database, { ...outcome.row, id: outcome.videoId }, Date.now());
+                stored += 1;
+                continue;
+            }
+            failures.push({ videoId: outcome.videoId, code: outcome.code });
+            // A refusal that will hit every video is a session problem: the run stops for it
+            // below, and marking this video would blame it for something it did not do.
+            if (STOPPING[outcome.code] === undefined) {
+                markUnreadable(database, outcome.videoId, unreadableMarker(outcome.code));
+            }
+        }
+    });
+    return { stored, failures };
+}
+//# sourceMappingURL=detail-batch.js.map

package/dist/utils/detail-batch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"detail-batch.js","sourceRoot":"","sources":["../../src/utils/detail-batch.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAyB,MAAM,aAAa,CAAC;AACvF,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EAEvB,cAAc,EACd,eAAe,GAChB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAStD,qFAAqF;AACrF,MAAM,CAAC,MAAM,QAAQ,GAA0D;IAC7E,SAAS,EAAE,gBAAgB;IAC3B,cAAc,EAAE,gBAAgB;IAChC,YAAY,EAAE,cAAc;IAC5B,SAAS,EAAE,WAAW;CACvB,CAAC;AAMF,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,OAAe;IACxC,IAAI,CAAC;QACH,yEAAyE;QACzE,iFAAiF;QACjF,wCAAwC;QACxC,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,OAAO,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,mBAAmB,CAAC;YAClC,YAAY,EAAE,GAAG,CAAC,YAAY,IAAI,SAAS;YAC3C,UAAU,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS;SACzC,CAAC,CAAC;QACH,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;IACpF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,YAAY,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC;IACxF,CAAC;AACH,CAAC;AAED,qFAAqF;AACrF,SAAS,gBAAgB,CAAC,IAAsB;IAC9C,OAAO,CACL,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC1E,wBAAwB,CACzB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,OAAO,KAAK;SACT,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SAC/D,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,OAAO,CACrB,QAAsB,EACtB,KAAa;IAEb,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,aAAa,CAAC,QAAQ,EAAE,GAAG,EAAE;QAC3B,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC5B,IAAI,KAAK,IAAI,OAAO,EAAE,CAAC;gBACrB,eAAe,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,EAAE,EAAE,OAAO,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC/E,MAAM,IAAI,CAAC,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YAChE,iFAAiF;YACjF,4EAA4E;YAC5E,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;gBACzC,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;AAC9B,CAAC"}

package/dist/utils/errors.d.ts

@@ -7,7 +7,7 @@ import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
  * than a wall of yt-dlp stderr it has to guess at — and never a leaked command
  * line or stack trace.
  */
-export type YouTubeErrorCode = 'PRIVATE' | 'AGE_GATED' | 'MEMBERS_ONLY' | 'PREMIUM_ONLY' | 'LOGIN_REQUIRED' | 'BOT_CHECK' | 'NOT_FOUND' | 'NO_CAPTIONS' | 'LIVE_NOT_ENDED' | 'RATE_LIMITED' | 'TIMEOUT' | 'CANCELLED' | 'YTDLP_MISSING' | 'FFMPEG_MISSING' | 'YTDLP_FAILED' | 'FETCH_FAILED' | 'MALFORMED_RESPONSE' | 'INVALID_INPUT' | 'STORE_UNAVAILABLE' | 'STORE_CORRUPT';
+export type YouTubeErrorCode = 'PRIVATE' | 'AGE_GATED' | 'MEMBERS_ONLY' | 'PREMIUM_ONLY' | 'LOGIN_REQUIRED' | 'BOT_CHECK' | 'NOT_FOUND' | 'NO_CAPTIONS' | 'LIVE_NOT_ENDED' | 'RATE_LIMITED' | 'TIMEOUT' | 'CANCELLED' | 'YTDLP_MISSING' | 'FFMPEG_MISSING' | 'WHISPER_MISSING' | 'YTDLP_FAILED' | 'FETCH_FAILED' | 'MALFORMED_RESPONSE' | 'INVALID_INPUT' | 'STORE_UNAVAILABLE' | 'STORE_CORRUPT';
 export interface YouTubeErrorOptions {
     /** What the caller should try next. Shown to the model verbatim. */
     nextStep?: string;

package/dist/utils/errors.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAEzE;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GACxB,SAAS,GACT,WAAW,GACX,cAAc,GACd,cAAc,GACd,gBAAgB,GAChB,WAAW,GACX,WAAW,GACX,aAAa,GACb,gBAAgB,GAChB,cAAc,GACd,SAAS,GACT,WAAW,GACX,eAAe,GACf,gBAAgB,GAChB,cAAc,GACd,cAAc,GACd,oBAAoB,GACpB,eAAe,GACf,mBAAmB,GACnB,eAAe,CAAC;AAEpB,MAAM,WAAW,mBAAmB;IAClC,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8DAA8D;IAC9D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACrC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;gBAEhB,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB;IAQtF,iFAAiF;IACjF,aAAa,IAAI,MAAM;CAGxB;AAwGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,GAAG,YAAY,CAkBlF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,YAAY,GAAG,SAAS,CA2C3B;AAED,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,CAQ3D;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,cAAc,CAM1D"}
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAEzE;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GACxB,SAAS,GACT,WAAW,GACX,cAAc,GACd,cAAc,GACd,gBAAgB,GAChB,WAAW,GACX,WAAW,GACX,aAAa,GACb,gBAAgB,GAChB,cAAc,GACd,SAAS,GACT,WAAW,GACX,eAAe,GACf,gBAAgB,GAChB,iBAAiB,GACjB,cAAc,GACd,cAAc,GACd,oBAAoB,GACpB,eAAe,GACf,mBAAmB,GACnB,eAAe,CAAC;AAEpB,MAAM,WAAW,mBAAmB;IAClC,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8DAA8D;IAC9D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACrC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;gBAEhB,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB;IAQtF,iFAAiF;IACjF,aAAa,IAAI,MAAM;CAGxB;AAkHD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,GAAG,YAAY,CAkBlF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,YAAY,GAAG,SAAS,CA2C3B;AAED,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,CAQ3D;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,cAAc,CAM1D"}

package/dist/utils/errors.js

@@ -51,6 +51,16 @@ const RULES = [
         nextStep: 'Wait a minute before retrying, and request fewer items per call.',
         retryable: true,
     },
+    {
+        // Raised as an expected error, so a harvest that walks every tab of a channel meets
+        // it for each tab the channel lacks. It is an answer, not a failure, and has to be
+        // recognisable as one: the generic fallback below carries none of yt-dlp's words.
+        code: 'NOT_FOUND',
+        emits: 'This channel does not have a',
+        source: 'yt_dlp/extractor/youtube/_tab.py:2393',
+        message: 'This channel does not have that tab.',
+        nextStep: 'Not every channel has every tab; this is not an error with the channel.',
+    },
     {
         code: 'NO_CAPTIONS',
         emits: 'There are no subtitles for the requested languages',

package/dist/utils/errors.js.map

@@ -1 +1 @@
-{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"AAwCA,MAAM,OAAO,YAAa,SAAQ,KAAK;IAC5B,IAAI,CAAmB;IACvB,QAAQ,CAAU;IAClB,SAAS,CAAU;IAE5B,YAAY,IAAsB,EAAE,OAAe,EAAE,UAA+B,EAAE;QACpF,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAC9C,CAAC;IAED,iFAAiF;IACjF,aAAa;QACX,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;IAC9E,CAAC;CACF;AA8BD,6EAA6E;AAC7E,MAAM,WAAW,GACf,4JAA4J,CAAC;AAE/J,MAAM,KAAK,GAAW;IACpB;QACE,yEAAyE;QACzE,wEAAwE;QACxE,wEAAwE;QACxE,qEAAqE;QACrE,sEAAsE;QACtE,wEAAwE;QACxE,2EAA2E;QAC3E,sEAAsE;QACtE,qCAAqC;QACrC,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,2EAA2E;QACnF,OAAO,EAAE,yDAAyD;QAClE,sEAAsE;QACtE,oEAAoE;QACpE,wEAAwE;QACxE,uEAAuE;QACvE,gEAAgE;QAChE,sEAAsE;QACtE,0EAA0E;QAC1E,kEAAkE;QAClE,QAAQ,EAAE,8LAA8L,WAAW,0GAA0G;KAC9T;IACD;QACE,IAAI,EAAE,cAAc;QACpB,4EAA4E;QAC5E,wDAAwD;QACxD,KAAK,EAAE,gBAAgB;QACvB,MAAM,EAAE,oCAAoC;QAC5C,OAAO,EAAE,uCAAuC;QAChD,QAAQ,EAAE,kEAAkE;QAC5E,SAAS,EAAE,IAAI;KAChB;IACD;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,oDAAoD;QAC3D,MAAM,EAAE,0BAA0B;QAClC,OAAO,EAAE,qEAAqE;QAC9E,QAAQ,EACN,uFAAuF;KAC1F;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,6BAA6B;QACpC,MAAM,EAAE,0BAA0B;QAClC,OAAO,EAAE,6DAA6D;QACtE,QAAQ,EAAE,gEAAgE;KAC3E;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,8EAA8E;QACrF,MAAM,EAAE,oCAAoC;QAC5C,OAAO,EAAE,6DAA6D;QACtE,QAAQ,EAAE,gEAAgE;KAC3E;IACD;QACE,oEAAoE;QACpE,yEAAyE;QACzE,yEAAyE;QACzE,iEAAiE;QACjE,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,mDAAmD;QAC1D,MAAM,EAAE,uCAAuC;QAC/C,OAAO,EAAE,kDAAkD;QAC3D,QAAQ,EAAE,sDAAsD,WAAW,EAAE;KAC9E;CACF,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAc,EAAE,KAAe;IAClE,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAChD,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE;gBAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,yCAAyC,EAAE;QACjF,QAAQ,EACN,kLAAkL;QACpL,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAGnC;IACC,QAAQ,MAAM,CAAC,YAAY,EAAE,CAAC;QAC5B,KAAK,SAAS;YACZ,OAAO,IAAI,YAAY,CAAC,SAAS,EAAE,wBAAwB,EAAE;gBAC3D,QAAQ,EAAE,0EAA0E;aACrF,CAAC,CAAC;QACL,KAAK,iBAAiB;YACpB,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,8CAA8C,EAAE;gBACtF,QAAQ,EAAE,iEAAiE;aAC5E,CAAC,CAAC;QACL,KAAK,cAAc;YACjB,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,gDAAgD,EAAE;gBACxF,QAAQ,EAAE,4DAA4D;aACvE,CAAC,CAAC;QACL,KAAK,YAAY;YACf,OAAO,IAAI,YAAY,CACrB,WAAW,EACX,gEAAgE,EAChE;gBACE,QAAQ,EAAE,2DAA2D,WAAW,EAAE;aACnF,CACF,CAAC;IACN,CAAC;IAED,4EAA4E;IAC5E,gDAAgD;IAChD,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1B,KAAK,aAAa;YAChB,OAAO,IAAI,YAAY,CAAC,gBAAgB,EAAE,uCAAuC,EAAE;gBACjF,QAAQ,EAAE,qEAAqE;aAChF,CAAC,CAAC;QACL,KAAK,WAAW;YACd,OAAO,IAAI,YAAY,CACrB,gBAAgB,EAChB,qDAAqD,EACrD;gBACE,QAAQ,EAAE,+DAA+D;gBACzE,SAAS,EAAE,IAAI;aAChB,CACF,CAAC;IACN,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,IAAI,KAAK,YAAY,YAAY;QAAE,OAAO,KAAK,CAAC;IAEhD,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,sCAAsC,EAAE;QAC9E,QAAQ,EAAE,0EAA0E;QACpF,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC;KAC3D,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,KAAc;IACxC,MAAM,YAAY,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAC3C,OAAO;QACL,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,YAAY,CAAC,IAAI,KAAK,YAAY,CAAC,aAAa,EAAE,EAAE,EAAE,CAAC;KAC5F,CAAC;AACJ,CAAC"}
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"AAyCA,MAAM,OAAO,YAAa,SAAQ,KAAK;IAC5B,IAAI,CAAmB;IACvB,QAAQ,CAAU;IAClB,SAAS,CAAU;IAE5B,YAAY,IAAsB,EAAE,OAAe,EAAE,UAA+B,EAAE;QACpF,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAC9C,CAAC;IAED,iFAAiF;IACjF,aAAa;QACX,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;IAC9E,CAAC;CACF;AA8BD,6EAA6E;AAC7E,MAAM,WAAW,GACf,4JAA4J,CAAC;AAE/J,MAAM,KAAK,GAAW;IACpB;QACE,yEAAyE;QACzE,wEAAwE;QACxE,wEAAwE;QACxE,qEAAqE;QACrE,sEAAsE;QACtE,wEAAwE;QACxE,2EAA2E;QAC3E,sEAAsE;QACtE,qCAAqC;QACrC,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,2EAA2E;QACnF,OAAO,EAAE,yDAAyD;QAClE,sEAAsE;QACtE,oEAAoE;QACpE,wEAAwE;QACxE,uEAAuE;QACvE,gEAAgE;QAChE,sEAAsE;QACtE,0EAA0E;QAC1E,kEAAkE;QAClE,QAAQ,EAAE,8LAA8L,WAAW,0GAA0G;KAC9T;IACD;QACE,IAAI,EAAE,cAAc;QACpB,4EAA4E;QAC5E,wDAAwD;QACxD,KAAK,EAAE,gBAAgB;QACvB,MAAM,EAAE,oCAAoC;QAC5C,OAAO,EAAE,uCAAuC;QAChD,QAAQ,EAAE,kEAAkE;QAC5E,SAAS,EAAE,IAAI;KAChB;IACD;QACE,oFAAoF;QACpF,mFAAmF;QACnF,kFAAkF;QAClF,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,8BAA8B;QACrC,MAAM,EAAE,uCAAuC;QAC/C,OAAO,EAAE,sCAAsC;QAC/C,QAAQ,EAAE,yEAAyE;KACpF;IACD;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,oDAAoD;QAC3D,MAAM,EAAE,0BAA0B;QAClC,OAAO,EAAE,qEAAqE;QAC9E,QAAQ,EACN,uFAAuF;KAC1F;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,6BAA6B;QACpC,MAAM,EAAE,0BAA0B;QAClC,OAAO,EAAE,6DAA6D;QACtE,QAAQ,EAAE,gEAAgE;KAC3E;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,8EAA8E;QACrF,MAAM,EAAE,oCAAoC;QAC5C,OAAO,EAAE,6DAA6D;QACtE,QAAQ,EAAE,gEAAgE;KAC3E;IACD;QACE,oEAAoE;QACpE,yEAAyE;QACzE,yEAAyE;QACzE,iEAAiE;QACjE,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,mDAAmD;QAC1D,MAAM,EAAE,uCAAuC;QAC/C,OAAO,EAAE,kDAAkD;QAC3D,QAAQ,EAAE,sDAAsD,WAAW,EAAE;KAC9E;CACF,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAc,EAAE,KAAe;IAClE,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAChD,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE;gBAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,yCAAyC,EAAE;QACjF,QAAQ,EACN,kLAAkL;QACpL,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAGnC;IACC,QAAQ,MAAM,CAAC,YAAY,EAAE,CAAC;QAC5B,KAAK,SAAS;YACZ,OAAO,IAAI,YAAY,CAAC,SAAS,EAAE,wBAAwB,EAAE;gBAC3D,QAAQ,EAAE,0EAA0E;aACrF,CAAC,CAAC;QACL,KAAK,iBAAiB;YACpB,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,8CAA8C,EAAE;gBACtF,QAAQ,EAAE,iEAAiE;aAC5E,CAAC,CAAC;QACL,KAAK,cAAc;YACjB,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,gDAAgD,EAAE;gBACxF,QAAQ,EAAE,4DAA4D;aACvE,CAAC,CAAC;QACL,KAAK,YAAY;YACf,OAAO,IAAI,YAAY,CACrB,WAAW,EACX,gEAAgE,EAChE;gBACE,QAAQ,EAAE,2DAA2D,WAAW,EAAE;aACnF,CACF,CAAC;IACN,CAAC;IAED,4EAA4E;IAC5E,gDAAgD;IAChD,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1B,KAAK,aAAa;YAChB,OAAO,IAAI,YAAY,CAAC,gBAAgB,EAAE,uCAAuC,EAAE;gBACjF,QAAQ,EAAE,qEAAqE;aAChF,CAAC,CAAC;QACL,KAAK,WAAW;YACd,OAAO,IAAI,YAAY,CACrB,gBAAgB,EAChB,qDAAqD,EACrD;gBACE,QAAQ,EAAE,+DAA+D;gBACzE,SAAS,EAAE,IAAI;aAChB,CACF,CAAC;IACN,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,IAAI,KAAK,YAAY,YAAY;QAAE,OAAO,KAAK,CAAC;IAEhD,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,sCAAsC,EAAE;QAC9E,QAAQ,EAAE,0EAA0E;QACpF,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC;KAC3D,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,KAAc;IACxC,MAAM,YAAY,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAC3C,OAAO;QACL,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,YAAY,CAAC,IAAI,KAAK,YAAY,CAAC,aAAa,EAAE,EAAE,EAAE,CAAC;KAC5F,CAAC;AACJ,CAAC"}

package/dist/utils/export-data.d.ts

@@ -0,0 +1,41 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { Coverage } from '../harvest-schemas.js';
+/**
+ * Taking harvested data out of the store, as a file.
+ *
+ * What is harvested is only as whole as its receipt says, and a file is read without
+ * the tool that made it. So an export carries its receipts with it: inside the file
+ * for JSON and Markdown, and beside it for CSV, which has nowhere to put them.
+ */
+export declare const DEFAULT_EXPORT_ROWS = 10000;
+export declare const MAX_EXPORT_ROWS = 100000;
+export type ExportDataset = 'videos' | 'comments';
+export type ExportFormat = 'json' | 'csv' | 'markdown';
+export interface ExportRequest {
+    dataset: ExportDataset;
+    format: ExportFormat;
+    /** Restrict to one channel id. */
+    channel?: string;
+    /** Restrict to one video id; comments only. */
+    video?: string;
+    limit: number;
+    outputDir?: string;
+}
+export interface ExportResult {
+    filePath: string;
+    /** Present for CSV: where the receipts went, since a CSV cannot hold them. */
+    coveragePath?: string;
+    dataset: ExportDataset;
+    format: ExportFormat;
+    rows: number;
+    total: number;
+    truncated: boolean;
+    coverage: Coverage[];
+    /** Receipts that should exist for these rows and do not, as `scope:id`. */
+    withoutReceipt: string[];
+    /** Receipts left out of the list, past the limit a result carries. */
+    omittedReceipts: number;
+    bytes: number;
+}
+export declare function exportData(database: DatabaseSync, request: ExportRequest): Promise<ExportResult>;
+//# sourceMappingURL=export-data.d.ts.map

package/dist/utils/export-data.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"export-data.d.ts","sourceRoot":"","sources":["../../src/utils/export-data.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAUtD;;;;;;GAMG;AAEH,eAAO,MAAM,mBAAmB,QAAS,CAAC;AAC1C,eAAO,MAAM,eAAe,SAAU,CAAC;AAOvC,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,UAAU,CAAC;AAClD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,KAAK,GAAG,UAAU,CAAC;AAEvD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,aAAa,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,8EAA8E;IAC9E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,aAAa,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACrB,2EAA2E;IAC3E,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,sEAAsE;IACtE,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;CACf;AAsCD,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,YAAY,CAAC,CA4DvB"}

package/dist/utils/export-data.js

@@ -0,0 +1,114 @@
+import { mkdir, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { renderCoverage } from './coverage-text.js';
+import { YouTubeError } from './errors.js';
+import { toCsv, toMarkdownTable } from './export-format.js';
+import { loadComments, loadVideos } from './export-load.js';
+import { COMMENT_COLUMNS, VIDEO_COLUMNS } from './export-rows.js';
+import { writeFileAtomic, writeJsonAtomic } from './json-file.js';
+import { dataDir } from './paths.js';
+import { assertChannelId, assertVideoId, resolveOutputDir } from './validate.js';
+/**
+ * Taking harvested data out of the store, as a file.
+ *
+ * What is harvested is only as whole as its receipt says, and a file is read without
+ * the tool that made it. So an export carries its receipts with it: inside the file
+ * for JSON and Markdown, and beside it for CSV, which has nowhere to put them.
+ */
+export const DEFAULT_EXPORT_ROWS = 10_000;
+export const MAX_EXPORT_ROWS = 100_000;
+/** Resolved per call, not at import, so the home directory is the one in force when it runs. */
+function exportsDir() {
+    return dataDir('exports');
+}
+function csv(loaded) {
+    return loaded.dataset === 'videos'
+        ? toCsv(VIDEO_COLUMNS, loaded.rows)
+        : toCsv(COMMENT_COLUMNS, loaded.rows);
+}
+function markdownTable(loaded) {
+    return loaded.dataset === 'videos'
+        ? toMarkdownTable(VIDEO_COLUMNS, loaded.rows)
+        : toMarkdownTable(COMMENT_COLUMNS, loaded.rows);
+}
+function markdown(loaded, truncated) {
+    const { receipts } = loaded;
+    return [
+        `# ${loaded.dataset === 'videos' ? 'Videos' : 'Comments'}`,
+        '',
+        `${String(loaded.rows.length)} of ${String(loaded.total)} rows${truncated ? ' (truncated by the limit)' : ''}.`,
+        '',
+        '## Completeness',
+        '',
+        ...(receipts.coverage.length === 0
+            ? ['No receipt covers these rows, so how much of the source they are is unknown.']
+            : receipts.coverage.flatMap((receipt) => [renderCoverage(receipt), ''])),
+        ...(receipts.withoutReceipt.length === 0
+            ? []
+            : [`No receipt for: ${receipts.withoutReceipt.join(', ')}`, '']),
+        ...(receipts.omittedReceipts === 0
+            ? []
+            : [`${String(receipts.omittedReceipts)} further receipts are not listed.`, '']),
+        markdownTable(loaded),
+        '',
+    ].join('\n');
+}
+export async function exportData(database, request) {
+    if (request.channel !== undefined)
+        assertChannelId(request.channel);
+    if (request.video !== undefined) {
+        if (request.dataset !== 'comments') {
+            throw new YouTubeError('INVALID_INPUT', 'video only applies when exporting comments.', {
+                nextStep: 'Pass channel to restrict a video export, or export dataset comments.',
+            });
+        }
+        assertVideoId(request.video);
+    }
+    const targetDir = resolveOutputDir(request.outputDir, exportsDir());
+    const loaded = request.dataset === 'videos' ? loadVideos(database, request) : loadComments(database, request);
+    const truncated = loaded.rows.length < loaded.total;
+    const base = `${request.dataset}-${request.video ?? request.channel ?? 'all'}`;
+    const extension = request.format === 'markdown' ? 'md' : request.format;
+    const filePath = join(targetDir, `${base}.${extension}`);
+    const meta = {
+        exportedAt: new Date().toISOString(),
+        dataset: request.dataset,
+        filters: {
+            ...(request.channel === undefined ? {} : { channel: request.channel }),
+            ...(request.video === undefined ? {} : { video: request.video }),
+        },
+        total: loaded.total,
+        exported: loaded.rows.length,
+        truncated,
+        coverage: loaded.receipts.coverage,
+        withoutReceipt: loaded.receipts.withoutReceipt,
+        omittedReceipts: loaded.receipts.omittedReceipts,
+    };
+    await mkdir(targetDir, { recursive: true });
+    let coveragePath;
+    if (request.format === 'json') {
+        await writeJsonAtomic(filePath, { ...meta, rows: loaded.rows });
+    }
+    else if (request.format === 'markdown') {
+        await writeFileAtomic(filePath, markdown(loaded, truncated));
+    }
+    else {
+        await writeFileAtomic(filePath, csv(loaded));
+        coveragePath = join(targetDir, `${base}.coverage.json`);
+        await writeJsonAtomic(coveragePath, meta);
+    }
+    return {
+        filePath,
+        ...(coveragePath === undefined ? {} : { coveragePath }),
+        dataset: request.dataset,
+        format: request.format,
+        rows: loaded.rows.length,
+        total: loaded.total,
+        truncated,
+        coverage: loaded.receipts.coverage,
+        withoutReceipt: loaded.receipts.withoutReceipt,
+        omittedReceipts: loaded.receipts.omittedReceipts,
+        bytes: (await stat(filePath)).size,
+    };
+}
+//# sourceMappingURL=export-data.js.map

package/dist/utils/export-data.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"export-data.js","sourceRoot":"","sources":["../../src/utils/export-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,UAAU,EAAe,MAAM,kBAAkB,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEjF;;;;;;GAMG;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAC1C,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC;AAEvC,gGAAgG;AAChG,SAAS,UAAU;IACjB,OAAO,OAAO,CAAC,SAAS,CAAC,CAAC;AAC5B,CAAC;AAiCD,SAAS,GAAG,CAAC,MAAc;IACzB,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ;QAChC,CAAC,CAAC,KAAK,CAAC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC;QACnC,CAAC,CAAC,KAAK,CAAC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ;QAChC,CAAC,CAAC,eAAe,CAAC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC;QAC7C,CAAC,CAAC,eAAe,CAAC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,QAAQ,CAAC,MAAc,EAAE,SAAkB;IAClD,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IAE5B,OAAO;QACL,KAAK,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,EAAE;QAC1D,EAAE;QACF,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,GAAG;QAC/G,EAAE;QACF,iBAAiB;QACjB,EAAE;QACF,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAChC,CAAC,CAAC,CAAC,8EAA8E,CAAC;YAClF,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1E,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC;YACtC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,CAAC,mBAAmB,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAClE,GAAG,CAAC,QAAQ,CAAC,eAAe,KAAK,CAAC;YAChC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,mCAAmC,EAAE,EAAE,CAAC,CAAC;QACjF,aAAa,CAAC,MAAM,CAAC;QACrB,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,QAAsB,EACtB,OAAsB;IAEtB,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;QAAE,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpE,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAChC,IAAI,OAAO,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YACnC,MAAM,IAAI,YAAY,CAAC,eAAe,EAAE,6CAA6C,EAAE;gBACrF,QAAQ,EAAE,sEAAsE;aACjF,CAAC,CAAC;QACL,CAAC;QACD,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC;IAEpE,MAAM,MAAM,GACV,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACjG,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC;IAEpD,MAAM,IAAI,GAAG,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,IAAI,KAAK,EAAE,CAAC;IAC/E,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG;QACX,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE;YACP,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;YACtE,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;SACjE;QACD,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;QAC5B,SAAS;QACT,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ;QAClC,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc;QAC9C,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,eAAe;KACjD,CAAC;IAEF,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,IAAI,YAAgC,CAAC;IAErC,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAC9B,MAAM,eAAe,CAAC,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAClE,CAAC;SAAM,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACzC,MAAM,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/D,CAAC;SAAM,CAAC;QACN,MAAM,eAAe,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7C,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,gBAAgB,CAAC,CAAC;QACxD,MAAM,eAAe,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO;QACL,QAAQ;QACR,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC;QACvD,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;QACxB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,SAAS;QACT,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ;QAClC,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc;QAC9C,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,eAAe;QAChD,KAAK,EAAE,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;KACnC,CAAC;AACJ,CAAC"}

package/dist/utils/export-format.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Writing rows out as CSV and Markdown tables.
+ *
+ * Exported comments are text written by strangers, and a spreadsheet will run a cell
+ * that begins like a formula. Every text field is therefore defused on the way out,
+ * as OWASP's CSV-injection guidance prescribes, rather than trusted because it came
+ * from a platform.
+ */
+export type Cell = string | number | boolean | null | undefined;
+export interface Column<T> {
+    header: string;
+    value: (row: T) => Cell;
+    /**
+     * Written as it is, never defused. For identifiers, which are validated elsewhere and
+     * must come back exact: a YouTube id may begin with `-`, and a leading apostrophe
+     * would make it a different id.
+     */
+    verbatim?: boolean;
+}
+/** RFC 4180: a header row, CRLF line endings, every record ended with one. */
+export declare function toCsv<T>(columns: Column<T>[], rows: T[]): string;
+export declare function toMarkdownTable<T>(columns: Column<T>[], rows: T[]): string;
+//# sourceMappingURL=export-format.d.ts.map

package/dist/utils/export-format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"export-format.d.ts","sourceRoot":"","sources":["../../src/utils/export-format.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;AAEhE,MAAM,WAAW,MAAM,CAAC,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC;IACxB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAmBD,8EAA8E;AAC9E,wBAAgB,KAAK,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,MAAM,CAMhE;AAwBD,wBAAgB,eAAe,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,MAAM,CAQ1E"}

package/dist/utils/export-format.js

@@ -0,0 +1,59 @@
+/**
+ * Writing rows out as CSV and Markdown tables.
+ *
+ * Exported comments are text written by strangers, and a spreadsheet will run a cell
+ * that begins like a formula. Every text field is therefore defused on the way out,
+ * as OWASP's CSV-injection guidance prescribes, rather than trusted because it came
+ * from a platform.
+ */
+/** Characters that make a spreadsheet read a cell as a formula (or as a command). */
+const FORMULA_TRIGGERS = new Set(['=', '+', '-', '@', '\t', '\r']);
+function defused(text) {
+    const first = text[0];
+    return first !== undefined && FORMULA_TRIGGERS.has(first) ? `'${text}` : text;
+}
+function csvField(cell, verbatim = false) {
+    if (cell === null || cell === undefined)
+        return '';
+    // Numbers and booleans are not text: a negative number is data, not a formula.
+    if (typeof cell !== 'string')
+        return String(cell);
+    const text = verbatim ? cell : defused(cell);
+    return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
+}
+/** RFC 4180: a header row, CRLF line endings, every record ended with one. */
+export function toCsv(columns, rows) {
+    const header = columns.map((column) => csvField(column.header)).join(',');
+    const body = rows.map((row) => columns.map((column) => csvField(column.value(row), column.verbatim)).join(','));
+    return [header, ...body].map((line) => `${line}\r\n`).join('');
+}
+/** Characters that carry meaning in Markdown or HTML, and so must not be left to the reader's renderer. */
+const MARKDOWN_ESCAPES = {
+    '\\': '\\\\',
+    '|': '\\|',
+    '`': '\\`',
+    '[': '\\[',
+    ']': '\\]',
+    '<': '&lt;',
+    '>': '&gt;',
+};
+/**
+ * One table cell. Comment text is written by strangers and may hold HTML or links; it is
+ * shown as text, never as markup, and cannot end its cell or its row.
+ */
+function markdownCell(cell) {
+    if (cell === null || cell === undefined)
+        return '';
+    return String(cell)
+        .replace(/[\\|`[\]<>]/g, (character) => MARKDOWN_ESCAPES[character] ?? character)
+        .replace(/\r\n|\r|\n/g, ' ');
+}
+export function toMarkdownTable(columns, rows) {
+    const line = (cells) => `| ${cells.join(' | ')} |`;
+    return [
+        line(columns.map((column) => markdownCell(column.header))),
+        line(columns.map(() => '---')),
+        ...rows.map((row) => line(columns.map((column) => markdownCell(column.value(row))))),
+    ].join('\n');
+}
+//# sourceMappingURL=export-format.js.map

package/dist/utils/export-format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"export-format.js","sourceRoot":"","sources":["../../src/utils/export-format.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAeH,qFAAqF;AACrF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAEnE,SAAS,OAAO,CAAC,IAAY;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,OAAO,KAAK,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAChF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAU,EAAE,QAAQ,GAAG,KAAK;IAC5C,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACnD,+EAA+E;IAC/E,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IAElD,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AACxE,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,KAAK,CAAI,OAAoB,EAAE,IAAS;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1E,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC5B,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAChF,CAAC;IACF,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,2GAA2G;AAC3G,MAAM,gBAAgB,GAA2B;IAC/C,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;CACZ,CAAC;AAEF;;;GAGG;AACH,SAAS,YAAY,CAAC,IAAU;IAC9B,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACnD,OAAO,MAAM,CAAC,IAAI,CAAC;SAChB,OAAO,CAAC,cAAc,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;SAChF,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,eAAe,CAAI,OAAoB,EAAE,IAAS;IAChE,MAAM,IAAI,GAAG,CAAC,KAAe,EAAU,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IAErE,OAAO;QACL,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QAC9B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;KACrF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC"}

package/dist/utils/export-load.d.ts

@@ -0,0 +1,28 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { Coverage } from '../harvest-schemas.js';
+import { type CommentRow, type VideoRow } from './export-rows.js';
+/** What is loaded: the part of an export request that decides which rows. */
+export interface LoadRequest {
+    channel?: string;
+    video?: string;
+    limit: number;
+}
+export interface Receipts {
+    coverage: Coverage[];
+    withoutReceipt: string[];
+    omittedReceipts: number;
+}
+export type Loaded = {
+    dataset: 'videos';
+    rows: VideoRow[];
+    total: number;
+    receipts: Receipts;
+} | {
+    dataset: 'comments';
+    rows: CommentRow[];
+    total: number;
+    receipts: Receipts;
+};
+export declare function loadVideos(database: DatabaseSync, request: LoadRequest): Loaded;
+export declare function loadComments(database: DatabaseSync, request: LoadRequest): Loaded;
+//# sourceMappingURL=export-load.d.ts.map

package/dist/utils/export-load.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"export-load.d.ts","sourceRoot":"","sources":["../../src/utils/export-load.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGtD,OAAO,EAA4B,KAAK,UAAU,EAAE,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAU5F,6EAA6E;AAC7E,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,MAAM,GACd;IAAE,OAAO,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,QAAQ,CAAA;CAAE,GAC1E;IAAE,OAAO,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,UAAU,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,QAAQ,CAAA;CAAE,CAAC;AAsCnF,wBAAgB,UAAU,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,GAAG,MAAM,CAgB/E;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,GAAG,MAAM,CAiBjF"}

package/dist/utils/export-load.js

@@ -0,0 +1,70 @@
+import { countComments, queryComments } from './comment-store.js';
+import { COVERAGE_IN_RESULT_LIMIT } from './coverage.js';
+import { toCommentRow, toVideoRow } from './export-rows.js';
+import { readReceipt } from './harvest-receipts.js';
+import { countVideos, queryVideos } from './video-store.js';
+/**
+ * Reading the rows an export holds out of the store, and the receipts that say how whole they are.
+ */
+const PAGE = 1_000;
+/** Pages through a query until `limit` rows, so memory holds rows and not a second copy of them. */
+function collect(limit, page) {
+    const rows = [];
+    while (rows.length < limit) {
+        const got = page(rows.length, Math.min(PAGE, limit - rows.length));
+        rows.push(...got);
+        if (got.length === 0)
+            break;
+    }
+    return rows;
+}
+function unique(values) {
+    return [...new Set(values.filter((value) => value !== undefined))];
+}
+/** The receipts for the targets in these rows, up to what a result carries. */
+function receiptsFor(database, targets) {
+    const coverage = [];
+    const withoutReceipt = [];
+    let omittedReceipts = 0;
+    for (const { scope, ids, required } of targets) {
+        for (const id of ids.slice(0, COVERAGE_IN_RESULT_LIMIT)) {
+            const receipt = readReceipt(database, scope, id);
+            if (receipt !== undefined)
+                coverage.push(receipt);
+            else if (required)
+                withoutReceipt.push(`${scope}:${id}`);
+        }
+        omittedReceipts += Math.max(0, ids.length - COVERAGE_IN_RESULT_LIMIT);
+    }
+    return { coverage, withoutReceipt, omittedReceipts };
+}
+export function loadVideos(database, request) {
+    const filter = request.channel === undefined ? {} : { channelId: request.channel };
+    const rows = collect(request.limit, (offset, size) => queryVideos(database, { ...filter, order: 'catalog', limit: size, offset }).map(toVideoRow));
+    const channels = unique([request.channel, ...rows.map((row) => row.channelId)]);
+    return {
+        dataset: 'videos',
+        rows,
+        total: countVideos(database, filter),
+        receipts: receiptsFor(database, [
+            { scope: 'channel-catalog', ids: channels, required: true },
+            { scope: 'video-details', ids: channels, required: false },
+        ]),
+    };
+}
+export function loadComments(database, request) {
+    const filter = {
+        ...(request.video === undefined ? {} : { videoId: request.video }),
+        ...(request.channel === undefined ? {} : { channelId: request.channel }),
+    };
+    const rows = collect(request.limit, (offset, size) => queryComments(database, { ...filter, order: 'oldest', limit: size, offset }).map(toCommentRow));
+    return {
+        dataset: 'comments',
+        rows,
+        total: countComments(database, filter),
+        receipts: receiptsFor(database, [
+            { scope: 'video-comments', ids: unique(rows.map((row) => row.videoId)), required: true },
+        ]),
+    };
+}
+//# sourceMappingURL=export-load.js.map

package/dist/utils/export-load.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"export-load.js","sourceRoot":"","sources":["../../src/utils/export-load.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAkC,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE5D;;GAEG;AAEH,MAAM,IAAI,GAAG,KAAK,CAAC;AAmBnB,oGAAoG;AACpG,SAAS,OAAO,CAAI,KAAa,EAAE,IAA2C;IAC5E,MAAM,IAAI,GAAQ,EAAE,CAAC;IACrB,OAAO,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;QAClB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;IAC9B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,MAAM,CAAC,MAA8B;IAC5C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAED,+EAA+E;AAC/E,SAAS,WAAW,CAClB,QAAsB,EACtB,OAAyE;IAEzE,MAAM,QAAQ,GAAe,EAAE,CAAC;IAChC,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,KAAK,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,OAAO,EAAE,CAAC;QAC/C,KAAK,MAAM,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,EAAE,CAAC;YACxD,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YACjD,IAAI,OAAO,KAAK,SAAS;gBAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBAC7C,IAAI,QAAQ;gBAAE,cAAc,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,EAAE,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,eAAe,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC;IACxE,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,QAAsB,EAAE,OAAoB;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;IACnF,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CACnD,WAAW,CAAC,QAAQ,EAAE,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAC5F,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAEhF,OAAO;QACL,OAAO,EAAE,QAAQ;QACjB,IAAI;QACJ,KAAK,EAAE,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC;QACpC,QAAQ,EAAE,WAAW,CAAC,QAAQ,EAAE;YAC9B,EAAE,KAAK,EAAE,iBAAiB,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;YAC3D,EAAE,KAAK,EAAE,eAAe,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE;SAC3D,CAAC;KACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAsB,EAAE,OAAoB;IACvE,MAAM,MAAM,GAAG;QACb,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;QAClE,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;KACzE,CAAC;IACF,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CACnD,aAAa,CAAC,QAAQ,EAAE,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAC/F,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,UAAU;QACnB,IAAI;QACJ,KAAK,EAAE,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC;QACtC,QAAQ,EAAE,WAAW,CAAC,QAAQ,EAAE;YAC9B,EAAE,KAAK,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE;SACzF,CAAC;KACH,CAAC;AACJ,CAAC"}

package/dist/utils/export-rows.d.ts

@@ -0,0 +1,35 @@
+import type { StoredComment } from './comment-store.js';
+import type { Column } from './export-format.js';
+import type { StoredVideo } from './video-store.js';
+/** The rows an export holds, as they are written, and the columns a CSV or table gives them. */
+export interface VideoRow {
+    videoId: string;
+    channelId?: string;
+    title: string;
+    durationSeconds?: number;
+    uploadDate?: string;
+    viewCount?: number;
+    commentCount?: number;
+    tab?: string;
+    alsoInTabs: string[];
+    detailed: boolean;
+    storedComments: number;
+}
+export interface CommentRow {
+    commentId: string;
+    videoId: string;
+    parentId?: string;
+    author: string;
+    authorId?: string;
+    likeCount: number;
+    isPinned: boolean;
+    isUploader: boolean;
+    isVerified: boolean;
+    publishedAt?: string;
+    text: string;
+}
+export declare const VIDEO_COLUMNS: Column<VideoRow>[];
+export declare const COMMENT_COLUMNS: Column<CommentRow>[];
+export declare function toVideoRow(row: StoredVideo): VideoRow;
+export declare function toCommentRow(row: StoredComment): CommentRow;
+//# sourceMappingURL=export-rows.d.ts.map

package/dist/utils/export-rows.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"export-rows.d.ts","sourceRoot":"","sources":["../../src/utils/export-rows.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,gGAAgG;AAEhG,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,EAAE,OAAO,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,QAAQ,CAAC,EAY3C,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC,EAY/C,CAAC;AAWF,wBAAgB,UAAU,CAAC,GAAG,EAAE,WAAW,GAAG,QAAQ,CAcrD;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,aAAa,GAAG,UAAU,CAgB3D"}