yt-liked 0.2.0-alpha.0

package/dist/videos-db.js

@@ -0,0 +1,534 @@
+import { openDb, saveDb } from './db.js';
+import { readJsonLines, writeJsonLines } from './jsonl.js';
+import { videosDbPath, videosJsonlPath } from './paths.js';
+import { readProbeReport } from './report.js';
+function initSchema(db) {
+    db.run(`
+    CREATE TABLE IF NOT EXISTS videos (
+      id TEXT PRIMARY KEY,
+      video_id TEXT,
+      playlist_item_id TEXT,
+      url TEXT NOT NULL,
+      title TEXT NOT NULL,
+      description TEXT,
+      channel_id TEXT,
+      channel_title TEXT,
+      liked_at TEXT,
+      video_published_at TEXT,
+      duration TEXT,
+      privacy_status TEXT,
+      position INTEGER,
+      categories TEXT,
+      primary_category TEXT,
+      domains TEXT,
+      primary_domain TEXT,
+      classification_reason TEXT,
+      classification_engine TEXT,
+      classification_model TEXT,
+      classified_at TEXT,
+      imported_at TEXT NOT NULL
+    );
+  `);
+    db.run(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS videos_fts
+    USING fts5(title, description, channel_title, content='videos', content_rowid='rowid');
+  `);
+}
+function escapeSql(value) {
+    return value.replace(/'/g, "''");
+}
+function insertRecord(db, record) {
+    const stmt = db.prepare(`
+    INSERT OR REPLACE INTO videos (
+      id, video_id, playlist_item_id, url, title, description, channel_id, channel_title,
+      liked_at, video_published_at, duration, privacy_status, position,
+      categories, primary_category, domains, primary_domain,
+      classification_reason, classification_engine, classification_model, classified_at, imported_at
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+    stmt.run([
+        record.id,
+        record.video_id,
+        record.playlist_item_id,
+        record.url,
+        record.title,
+        record.description,
+        record.channel_id,
+        record.channel_title,
+        record.liked_at,
+        record.video_published_at,
+        record.duration,
+        record.privacy_status,
+        record.position,
+        record.categories ? JSON.stringify(record.categories) : null,
+        record.primary_category,
+        record.domains ? JSON.stringify(record.domains) : null,
+        record.primary_domain,
+        record.classification_reason,
+        record.classification_engine,
+        record.classification_model,
+        record.classified_at,
+        record.imported_at,
+    ]);
+    stmt.free();
+}
+function rowToRecord(row) {
+    return {
+        id: row[0],
+        video_id: row[1],
+        playlist_item_id: row[2],
+        url: row[3],
+        title: row[4],
+        description: row[5],
+        channel_id: row[6],
+        channel_title: row[7],
+        liked_at: row[8],
+        video_published_at: row[9],
+        duration: row[10],
+        privacy_status: row[11],
+        position: typeof row[12] === 'number' ? row[12] : null,
+        categories: row[13] ? JSON.parse(row[13]) : null,
+        primary_category: row[14],
+        domains: row[15] ? JSON.parse(row[15]) : null,
+        primary_domain: row[16],
+        classification_reason: row[17],
+        classification_engine: row[18],
+        classification_model: row[19],
+        classified_at: row[20],
+        imported_at: row[21],
+    };
+}
+export async function buildIndex(options) {
+    const db = await openDb(videosDbPath());
+    const records = await readJsonLines(videosJsonlPath());
+    try {
+        if (options?.force) {
+            db.run('DROP TABLE IF EXISTS videos_fts');
+            db.run('DROP TABLE IF EXISTS videos');
+        }
+        initSchema(db);
+        db.run('DELETE FROM videos');
+        db.run('BEGIN TRANSACTION');
+        for (const record of records) {
+            insertRecord(db, record);
+        }
+        db.run('COMMIT');
+        db.run(`INSERT INTO videos_fts(videos_fts) VALUES('rebuild')`);
+        saveDb(db, videosDbPath());
+        return { recordCount: records.length, newRecords: records.length };
+    }
+    finally {
+        db.close();
+    }
+}
+export async function exportDbToJsonl(db) {
+    const localDb = db ?? await openDb(videosDbPath());
+    const ownsDb = !db;
+    try {
+        initSchema(localDb);
+        const rows = localDb.exec(`
+      SELECT
+        id, video_id, playlist_item_id, url, title, description, channel_id, channel_title,
+        liked_at, video_published_at, duration, privacy_status, position,
+        categories, primary_category, domains, primary_domain,
+        classification_reason, classification_engine, classification_model, classified_at, imported_at
+      FROM videos
+      ORDER BY COALESCE(position, 999999999) ASC, title ASC
+    `);
+        const records = (rows[0]?.values ?? []).map((row) => rowToRecord(row));
+        writeJsonLines(videosJsonlPath(), records);
+        if (ownsDb) {
+            saveDb(localDb, videosDbPath());
+        }
+    }
+    finally {
+        if (ownsDb)
+            localDb.close();
+    }
+}
+export async function getVideoStatusView() {
+    const db = await openDb(videosDbPath());
+    let lastProbe = null;
+    try {
+        initSchema(db);
+        const totals = db.exec(`
+      SELECT
+        COUNT(*) AS imported_count,
+        SUM(CASE WHEN primary_category IS NOT NULL THEN 1 ELSE 0 END) AS categorized_count,
+        SUM(CASE WHEN primary_domain IS NOT NULL THEN 1 ELSE 0 END) AS domain_count
+      FROM videos
+    `);
+        const lastRunRows = db.exec(`
+      SELECT classification_engine, classification_model
+      FROM videos
+      WHERE classification_engine IS NOT NULL OR classification_model IS NOT NULL
+      ORDER BY classified_at DESC
+      LIMIT 1
+    `);
+        const values = totals[0]?.values?.[0] ?? [0, 0, 0];
+        lastProbe = readProbeReport();
+        return {
+            importedCount: Number(values[0] ?? 0),
+            categorizedCount: Number(values[1] ?? 0),
+            domainCount: Number(values[2] ?? 0),
+            lastClassificationEngine: lastRunRows[0]?.values?.[0]?.[0] ?? null,
+            lastClassificationModel: lastRunRows[0]?.values?.[0]?.[1] ?? null,
+            lastProbe,
+        };
+    }
+    finally {
+        db.close();
+    }
+}
+function rowsToLabelCounts(rows) {
+    return rows.map((row) => ({
+        label: String(row[0] ?? 'unknown'),
+        count: Number(row[1] ?? 0),
+    }));
+}
+function rowsToChannelCounts(rows) {
+    return rows.map((row) => ({
+        channelTitle: String(row[0] ?? 'Unknown channel'),
+        count: Number(row[1] ?? 0),
+    }));
+}
+function rowToSearchResult(row) {
+    return {
+        id: row[0],
+        url: row[1],
+        title: row[2],
+        description: row[3],
+        channelTitle: row[4],
+        likedAt: row[5],
+        primaryCategory: row[6],
+        primaryDomain: row[7],
+        score: Number(row[8] ?? 0),
+    };
+}
+function buildVideoWhereClause(filters) {
+    const conditions = [];
+    const params = [];
+    if (filters.query) {
+        conditions.push(`v.rowid IN (SELECT rowid FROM videos_fts WHERE videos_fts MATCH ?)`);
+        params.push(filters.query);
+    }
+    if (filters.channel) {
+        conditions.push(`v.channel_title = ? COLLATE NOCASE`);
+        params.push(filters.channel);
+    }
+    if (filters.after) {
+        conditions.push(`COALESCE(v.liked_at, v.video_published_at, v.imported_at) >= ?`);
+        params.push(filters.after);
+    }
+    if (filters.before) {
+        conditions.push(`COALESCE(v.liked_at, v.video_published_at, v.imported_at) <= ?`);
+        params.push(filters.before);
+    }
+    if (filters.category) {
+        conditions.push(`v.primary_category = ? COLLATE NOCASE`);
+        params.push(filters.category);
+    }
+    if (filters.domain) {
+        conditions.push(`v.primary_domain = ? COLLATE NOCASE`);
+        params.push(filters.domain);
+    }
+    if (filters.privacy) {
+        conditions.push(`COALESCE(v.privacy_status, 'unknown') = ? COLLATE NOCASE`);
+        params.push(filters.privacy);
+    }
+    return {
+        where: conditions.length ? `WHERE ${conditions.join(' AND ')}` : '',
+        params,
+    };
+}
+function videoSortClause(direction = 'desc') {
+    const normalized = direction === 'asc' ? 'ASC' : 'DESC';
+    return `
+    ORDER BY
+      COALESCE(v.liked_at, v.video_published_at, v.imported_at) ${normalized},
+      v.title ${normalized}
+  `;
+}
+export async function getVideoVizView() {
+    const db = await openDb(videosDbPath());
+    try {
+        initSchema(db);
+        const totals = db.exec(`
+      SELECT
+        COUNT(*) AS imported_count,
+        SUM(CASE WHEN primary_category IS NOT NULL THEN 1 ELSE 0 END) AS categorized_count,
+        SUM(CASE WHEN primary_domain IS NOT NULL THEN 1 ELSE 0 END) AS domain_count
+      FROM videos
+    `);
+        const categoryRows = db.exec(`
+      SELECT primary_category, COUNT(*) AS n
+      FROM videos
+      WHERE primary_category IS NOT NULL
+      GROUP BY primary_category
+      ORDER BY n DESC, primary_category ASC
+      LIMIT 12
+    `);
+        const domainRows = db.exec(`
+      SELECT primary_domain, COUNT(*) AS n
+      FROM videos
+      WHERE primary_domain IS NOT NULL
+      GROUP BY primary_domain
+      ORDER BY n DESC, primary_domain ASC
+      LIMIT 12
+    `);
+        const channelRows = db.exec(`
+      SELECT COALESCE(NULLIF(channel_title, ''), 'Unknown channel') AS channel_title, COUNT(*) AS n
+      FROM videos
+      GROUP BY channel_title
+      ORDER BY n DESC, channel_title ASC
+      LIMIT 10
+    `);
+        const monthlyRows = db.exec(`
+      SELECT substr(liked_at, 1, 7) AS ym, COUNT(*) AS n
+      FROM videos
+      WHERE liked_at IS NOT NULL AND liked_at != ''
+      GROUP BY ym
+      ORDER BY ym ASC
+      LIMIT 18
+    `);
+        const privacyRows = db.exec(`
+      SELECT COALESCE(NULLIF(privacy_status, ''), 'unknown') AS privacy_status, COUNT(*) AS n
+      FROM videos
+      GROUP BY privacy_status
+      ORDER BY n DESC, privacy_status ASC
+      LIMIT 6
+    `);
+        const distinctChannelRows = db.exec(`
+      SELECT
+        COUNT(DISTINCT COALESCE(NULLIF(channel_title, ''), '__missing__')) AS distinct_titles,
+        COUNT(DISTINCT COALESCE(NULLIF(channel_id, ''), '__missing__')) AS distinct_ids
+      FROM videos
+    `);
+        const values = totals[0]?.values?.[0] ?? [0, 0, 0];
+        const importedCount = Number(values[0] ?? 0);
+        const categorizedCount = Number(values[1] ?? 0);
+        const domainCount = Number(values[2] ?? 0);
+        const distinctValues = distinctChannelRows[0]?.values?.[0] ?? [0, 0];
+        const distinctChannelTitles = Number(distinctValues[0] ?? 0);
+        const distinctChannelIds = Number(distinctValues[1] ?? 0);
+        const channelMetadataLikelyOwnerFallback = importedCount > 25 && (distinctChannelTitles <= 1 || distinctChannelIds <= 1);
+        const dominantFallbackChannelTitle = String(channelRows[0]?.values?.[0]?.[0] ?? '') || null;
+        const dominantFallbackChannelCount = Number(channelRows[0]?.values?.[0]?.[1] ?? 0);
+        return {
+            importedCount,
+            categorizedCount,
+            domainCount,
+            uncategorizedCount: Math.max(0, importedCount - categorizedCount),
+            undomainedCount: Math.max(0, importedCount - domainCount),
+            topCategories: rowsToLabelCounts(categoryRows[0]?.values ?? []),
+            topDomains: rowsToLabelCounts(domainRows[0]?.values ?? []),
+            topChannels: rowsToChannelCounts(channelRows[0]?.values ?? []),
+            monthlyLikes: rowsToLabelCounts(monthlyRows[0]?.values ?? []),
+            privacyBreakdown: rowsToLabelCounts(privacyRows[0]?.values ?? []),
+            distinctChannelTitles,
+            distinctChannelIds,
+            channelMetadataLikelyOwnerFallback,
+            dominantFallbackChannelTitle,
+            dominantFallbackChannelCount,
+        };
+    }
+    finally {
+        db.close();
+    }
+}
+export async function searchVideos(filters) {
+    const db = await openDb(videosDbPath());
+    try {
+        initSchema(db);
+        const limit = filters.limit ?? 20;
+        const { where, params } = buildVideoWhereClause(filters);
+        const sql = `
+      SELECT
+        v.id,
+        v.url,
+        v.title,
+        v.description,
+        v.channel_title,
+        v.liked_at,
+        v.primary_category,
+        v.primary_domain,
+        bm25(videos_fts, 3.5, 1.5, 1.0) AS score
+      FROM videos v
+      JOIN videos_fts ON videos_fts.rowid = v.rowid
+      ${where}
+      ORDER BY bm25(videos_fts, 3.5, 1.5, 1.0) ASC,
+               COALESCE(v.liked_at, v.video_published_at, v.imported_at) DESC
+      LIMIT ?
+    `;
+        params.push(limit);
+        const rows = db.exec(sql, params);
+        return (rows[0]?.values ?? []).map((row) => rowToSearchResult(row));
+    }
+    finally {
+        db.close();
+    }
+}
+export async function listVideos(filters = {}) {
+    const db = await openDb(videosDbPath());
+    try {
+        initSchema(db);
+        const limit = filters.limit ?? 30;
+        const offset = filters.offset ?? 0;
+        const { where, params } = buildVideoWhereClause(filters);
+        const sql = `
+      SELECT
+        v.id, v.video_id, v.playlist_item_id, v.url, v.title, v.description, v.channel_id, v.channel_title,
+        v.liked_at, v.video_published_at, v.duration, v.privacy_status, v.position,
+        v.categories, v.primary_category, v.domains, v.primary_domain,
+        v.classification_reason, v.classification_engine, v.classification_model, v.classified_at, v.imported_at
+      FROM videos v
+      ${where}
+      ${videoSortClause(filters.sort)}
+      LIMIT ?
+      OFFSET ?
+    `;
+        params.push(limit, offset);
+        const rows = db.exec(sql, params);
+        return (rows[0]?.values ?? []).map((row) => rowToRecord(row));
+    }
+    finally {
+        db.close();
+    }
+}
+export async function getVideoById(id) {
+    const db = await openDb(videosDbPath());
+    try {
+        initSchema(db);
+        const rows = db.exec(`
+      SELECT
+        v.id, v.video_id, v.playlist_item_id, v.url, v.title, v.description, v.channel_id, v.channel_title,
+        v.liked_at, v.video_published_at, v.duration, v.privacy_status, v.position,
+        v.categories, v.primary_category, v.domains, v.primary_domain,
+        v.classification_reason, v.classification_engine, v.classification_model, v.classified_at, v.imported_at
+      FROM videos v
+      WHERE v.id = ?
+      LIMIT 1
+    `, [id]);
+        const row = rows[0]?.values?.[0];
+        return row ? rowToRecord(row) : null;
+    }
+    finally {
+        db.close();
+    }
+}
+export async function getVideoByLookupKey(key) {
+    const db = await openDb(videosDbPath());
+    try {
+        initSchema(db);
+        const rows = db.exec(`
+      SELECT
+        v.id, v.video_id, v.playlist_item_id, v.url, v.title, v.description, v.channel_id, v.channel_title,
+        v.liked_at, v.video_published_at, v.duration, v.privacy_status, v.position,
+        v.categories, v.primary_category, v.domains, v.primary_domain,
+        v.classification_reason, v.classification_engine, v.classification_model, v.classified_at, v.imported_at
+      FROM videos v
+      WHERE v.id = ?
+         OR COALESCE(v.video_id, '') = ?
+         OR v.url = ?
+      LIMIT 1
+    `, [key, key, key]);
+        const row = rows[0]?.values?.[0];
+        return row ? rowToRecord(row) : null;
+    }
+    finally {
+        db.close();
+    }
+}
+export async function loadClassificationItems(kind, options = {}) {
+    const db = await openDb(videosDbPath());
+    try {
+        initSchema(db);
+        const where = kind === 'categories'
+            ? (options.all ? '1=1' : 'primary_category IS NULL')
+            : (options.all ? '1=1' : 'primary_domain IS NULL');
+        const limitClause = options.limit ? ` LIMIT ${Math.max(1, options.limit)}` : '';
+        const rows = db.exec(`
+      SELECT id, title, description, channel_title, duration, privacy_status, categories
+      FROM videos
+      WHERE ${where}
+      ORDER BY RANDOM()
+      ${limitClause}
+    `);
+        return (rows[0]?.values ?? []).map((row) => ({
+            id: row[0],
+            title: row[1],
+            description: row[2],
+            channelTitle: row[3],
+            duration: row[4],
+            privacyStatus: row[5],
+            existingCategories: row[6] ? JSON.parse(row[6]) : null,
+        }));
+    }
+    finally {
+        db.close();
+    }
+}
+export async function applyCategoryUpdates(updates) {
+    const db = await openDb(videosDbPath());
+    try {
+        initSchema(db);
+        const stmt = db.prepare(`
+      UPDATE videos
+      SET categories = ?, primary_category = ?, classification_reason = ?, classification_engine = ?,
+          classification_model = ?, classified_at = ?
+      WHERE id = ?
+    `);
+        const classifiedAt = new Date().toISOString();
+        for (const update of updates) {
+            stmt.run([
+                JSON.stringify(update.categories),
+                update.primary,
+                update.reason,
+                update.engine,
+                update.model ?? null,
+                classifiedAt,
+                update.id,
+            ]);
+        }
+        stmt.free();
+        saveDb(db, videosDbPath());
+        await exportDbToJsonl(db);
+    }
+    finally {
+        db.close();
+    }
+}
+export async function applyDomainUpdates(updates) {
+    const db = await openDb(videosDbPath());
+    try {
+        initSchema(db);
+        const stmt = db.prepare(`
+      UPDATE videos
+      SET domains = ?, primary_domain = ?, classification_reason = COALESCE(?, classification_reason),
+          classification_engine = ?, classification_model = ?, classified_at = ?
+      WHERE id = ?
+    `);
+        const classifiedAt = new Date().toISOString();
+        for (const update of updates) {
+            stmt.run([
+                JSON.stringify(update.domains),
+                update.primary,
+                update.reason,
+                update.engine,
+                update.model ?? null,
+                classifiedAt,
+                update.id,
+            ]);
+        }
+        stmt.free();
+        saveDb(db, videosDbPath());
+        await exportDbToJsonl(db);
+    }
+    finally {
+        db.close();
+    }
+}
+export async function requireVideoData() {
+    const records = await readJsonLines(videosJsonlPath());
+    return records.length > 0;
+}

package/dist/videos-import.js

@@ -0,0 +1,122 @@
+import fs from 'node:fs';
+import { videosJsonlPath } from './paths.js';
+import { readVideoArchive, writeJsonLines } from './jsonl.js';
+import { buildIndex } from './videos-db.js';
+function toStringOrNull(value) {
+    if (typeof value !== 'string')
+        return null;
+    const trimmed = value.trim();
+    return trimmed.length > 0 ? trimmed : null;
+}
+function toStringArray(value) {
+    if (Array.isArray(value)) {
+        const items = value
+            .map((item) => (typeof item === 'string' ? item.trim().toLowerCase() : ''))
+            .filter(Boolean);
+        return items.length > 0 ? Array.from(new Set(items)) : null;
+    }
+    if (typeof value === 'string') {
+        const parts = value
+            .split(',')
+            .map((item) => item.trim().toLowerCase())
+            .filter(Boolean);
+        return parts.length > 0 ? Array.from(new Set(parts)) : null;
+    }
+    return null;
+}
+function normalizeUrl(input) {
+    const explicit = toStringOrNull(input.url);
+    if (explicit)
+        return explicit;
+    const videoId = toStringOrNull(input.video_id);
+    if (videoId)
+        return `https://www.youtube.com/watch?v=${videoId}`;
+    throw new Error('Could not determine a video URL for one imported item.');
+}
+export function normalizeImportedVideo(input, importedAt = new Date().toISOString()) {
+    const playlistItemId = toStringOrNull(input.playlist_item_id);
+    const videoId = toStringOrNull(input.video_id);
+    const url = normalizeUrl(input);
+    const id = playlistItemId ?? videoId ?? url;
+    const primaryCategory = toStringOrNull(input.primary_category)?.toLowerCase() ?? null;
+    let categories = toStringArray(input.categories) ?? toStringArray(input.tags);
+    if (!categories && primaryCategory) {
+        categories = [primaryCategory];
+    }
+    const primaryDomain = toStringOrNull(input.primary_domain)?.toLowerCase() ?? null;
+    let domains = toStringArray(input.domains);
+    if (!domains && primaryDomain) {
+        domains = [primaryDomain];
+    }
+    return {
+        id,
+        video_id: videoId,
+        playlist_item_id: playlistItemId,
+        url,
+        title: toStringOrNull(input.title) ?? '(untitled)',
+        description: toStringOrNull(input.description),
+        channel_id: toStringOrNull(input.channel_id),
+        channel_title: toStringOrNull(input.channel_title),
+        liked_at: toStringOrNull(input.published_at),
+        video_published_at: toStringOrNull(input.video_published_at),
+        duration: toStringOrNull(input.duration),
+        privacy_status: toStringOrNull(input.privacy_status),
+        position: typeof input.position === 'number' ? input.position : null,
+        categories,
+        primary_category: primaryCategory,
+        domains,
+        primary_domain: primaryDomain,
+        classification_reason: toStringOrNull(input.classification_reason),
+        classification_engine: toStringOrNull(input.classification_engine),
+        classification_model: toStringOrNull(input.classification_model),
+        classified_at: toStringOrNull(input.classified_at),
+        imported_at: importedAt,
+    };
+}
+export function mergeVideoRecords(existing, incoming) {
+    return {
+        ...existing,
+        ...incoming,
+        url: incoming.url || existing.url,
+        title: incoming.title !== '(untitled)' ? incoming.title : existing.title,
+        description: incoming.description ?? existing.description,
+        channel_id: incoming.channel_id ?? existing.channel_id,
+        channel_title: incoming.channel_title ?? existing.channel_title,
+        liked_at: incoming.liked_at ?? existing.liked_at,
+        video_published_at: incoming.video_published_at ?? existing.video_published_at,
+        duration: incoming.duration ?? existing.duration,
+        privacy_status: incoming.privacy_status ?? existing.privacy_status,
+        position: incoming.position ?? existing.position,
+        categories: incoming.categories ?? existing.categories,
+        primary_category: incoming.primary_category ?? existing.primary_category,
+        domains: incoming.domains ?? existing.domains,
+        primary_domain: incoming.primary_domain ?? existing.primary_domain,
+        classification_reason: incoming.classification_reason ?? existing.classification_reason,
+        classification_engine: incoming.classification_engine ?? existing.classification_engine,
+        classification_model: incoming.classification_model ?? existing.classification_model,
+        classified_at: incoming.classified_at ?? existing.classified_at,
+        imported_at: incoming.imported_at ?? existing.imported_at,
+    };
+}
+export async function importVideoArchive(filePath) {
+    const raw = fs.readFileSync(filePath, 'utf8');
+    const parsed = JSON.parse(raw);
+    if (!Array.isArray(parsed)) {
+        throw new Error('Imported file must be a JSON array.');
+    }
+    const importedAt = new Date().toISOString();
+    const incoming = parsed.map((item) => normalizeImportedVideo(item, importedAt));
+    const existing = await readVideoArchive();
+    const merged = new Map();
+    for (const record of existing) {
+        merged.set(record.id, record);
+    }
+    for (const record of incoming) {
+        const prior = merged.get(record.id);
+        merged.set(record.id, prior ? mergeVideoRecords(prior, record) : record);
+    }
+    const records = Array.from(merged.values());
+    writeJsonLines(videosJsonlPath(), records);
+    await buildIndex({ force: true });
+    return { imported: incoming.length, total: records.length };
+}