playlist-admin 2.6.81 → 2.6.82

package/cli/index.js

@@ -1,33 +1,95 @@
 #!/usr/bin/env node
-import { loadConfig } from './lib/config.js';
-import { PipelineOrchestrator } from './lib/pipeline.js';
-import { PodcastPipeline } from './lib/podcast.js';
-import { SpotubeController } from './lib/spotube.js';
-import { cleanupMp3 } from './lib/cleanup.js';
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
 
-const HELP = `Playlist Administrator CLI (Node)
+// Thin launcher: the CLI engine lives in the Flutter app binary
+// (lib/cli_main.dart), so GUI and CLI share one implementation.
+// This script just locates the built exe / rag scripts and forwards args.
+
+const HELP = `Playlist Administrator CLI
+
+  Runs the same engine as the GUI app (shared Dart code). Requires a built
+  release: flutter build windows --release
 
 Usage:
-  playlist-admin pipeline                   Run full pipeline
-  playlist-admin pipeline --step N          Run single step
-  playlist-admin spotube-download <name>    Download one playlist
-  playlist-admin spotube-download-all       Download all playlists
-  playlist-admin spotube-move               Move M4A files
-  playlist-admin spotube-cleanup            Remove metadata-renamed mp3 duplicates
-  playlist-admin podcast                    Run podcast pipeline
-  playlist-admin status                     Show status
+  playlist-admin pipeline [--step N]     Run full pipeline (or from step N)
+  playlist-admin podcast                 Run podcast pipeline
+  playlist-admin status                  Show status
+  playlist-admin spotube-download <name> Download one playlist
+  playlist-admin spotube-download-all    Download all playlists
+  playlist-admin spotube-move            Move M4A files
+  playlist-admin spotube-cleanup         Remove metadata-renamed mp3 duplicates
+  playlist-admin rag build [--reset]     Build podcast RAG vector DB
+  playlist-admin rag query "問題" [--topk N] [--show 節目] [--json]
 `;
 
-function getFlag(args, flag) {
-  const idx = args.indexOf(flag);
-  if (idx >= 0 && idx + 1 < args.length) {
-    return parseInt(args[idx + 1], 10) || 0;
+function projectRoot() {
+  if (process.env.PA_ROOT && fs.existsSync(process.env.PA_ROOT)) {
+    return process.env.PA_ROOT;
+  }
+  // Start from cwd and walk up to find the Flutter project (works whether
+  // invoked locally or from a globally installed npm package).
+  let dir = process.cwd();
+  while (true) {
+    if (fs.existsSync(path.join(dir, 'pubspec.yaml'))) return dir;
+    const parent = path.dirname(dir);
+    if (parent === dir) return null;
+    dir = parent;
   }
-  return 0;
 }
 
-const onProgress = () => {};
-const isCancelled = () => false;
+function exePath(root) {
+  const candidates = [
+    path.join(root, 'build', 'windows', 'x64', 'runner', 'Release', 'playlist_administrator.exe'),
+    path.join(root, 'build', 'windows', 'x64', 'runner', 'Debug', 'playlist_administrator.exe'),
+  ];
+  return candidates.find((c) => fs.existsSync(c)) || null;
+}
+
+function python() {
+  return process.env.PYTHON || 'python';
+}
+
+function forward(exe, args) {
+  // Deliver CLI args via PA_CLI_ARGS (JSON) — Flutter Dart exposes no
+  // command-line args in release on this SDK (see lib/main.dart).
+  const child = spawn(exe, [], {
+    stdio: 'inherit',
+    windowsHide: false,
+    env: { ...process.env, PA_CLI_ARGS: JSON.stringify(args) },
+  });
+  child.on('exit', (code) => process.exit(code ?? 1));
+  child.on('error', (e) => {
+    console.error(`無法啟動 ${exe}: ${e.message}`);
+    process.exit(1);
+  });
+}
+
+function forwardPy(args) {
+  // Python scripts take real argv.
+  const child = spawn(python(), args, { stdio: 'inherit', windowsHide: false });
+  child.on('exit', (code) => process.exit(code ?? 1));
+  child.on('error', (e) => {
+    console.error(`無法啟動 python: ${e.message}`);
+    process.exit(1);
+  });
+}
+
+function runRag(args) {
+  const root = projectRoot();
+  if (!root) {
+    console.error('找不到專案根目錄（pubspec.yaml）。請在專案內執行，或設定 PA_ROOT');
+    process.exit(1);
+  }
+  const sub = args[0];
+  const script = path.join(root, 'rag', sub === 'build' ? 'build_db.py' : 'query.py');
+  if (!fs.existsSync(script)) {
+    console.error(`找不到 ${script}`);
+    process.exit(1);
+  }
+  forwardPy([script, ...args.slice(1)]);
+}
 
 async function main() {
   const args = process.argv.slice(2);
@@ -35,81 +97,24 @@ async function main() {
     console.log(HELP);
     return;
   }
+  if (args[0] === 'rag') {
+    runRag(args.slice(1));
+    return;
+  }
 
-  const config = loadConfig();
-  const cmd = args[0];
-
-  switch (cmd) {
-    case 'pipeline': {
-      const fromStep = getFlag(args, '--step');
-      const orch = new PipelineOrchestrator({ config, onLog: (m) => console.log(m), onProgress, isCancelled });
-      await orch.run({ fromStep });
-      break;
-    }
-
-    case 'spotube-download': {
-      if (args.length < 2) {
-        console.log('Usage: playlist-admin spotube-download <playlist_name>');
-        return;
-      }
-      const ctrl = new SpotubeController({
-        libraryPath: config.libraryPath,
-        coords: config.spotubeCoords,
-      });
-      if (!(await ctrl.isRunning())) {
-        console.log('Spotube is not running');
-        return;
-      }
-      await ctrl.downloadPlaylist(args[1]);
-      break;
-    }
-
-    case 'spotube-download-all': {
-      const ctrl = new SpotubeController({
-        libraryPath: config.libraryPath,
-        coords: config.spotubeCoords,
-      });
-      if (!(await ctrl.isRunning())) {
-        console.log('Spotube is not running');
-        return;
-      }
-      const names = Object.values(config.urlNames);
-      for (const name of names) {
-        console.log(`Downloading: ${name}`);
-        await ctrl.downloadPlaylist(name);
-      }
-      break;
-    }
-
-    case 'spotube-move': {
-      const ctrl = new SpotubeController({ libraryPath: config.libraryPath });
-      const moved = await ctrl.moveDownloads(config.resolvedSpotubeDownloadPath);
-      console.log(`Moved ${moved} files`);
-      break;
-    }
-
-    case 'spotube-cleanup': {
-      await cleanupMp3(config.libraryPath);
-      break;
-    }
-
-    case 'podcast': {
-      const pipeline = new PodcastPipeline({ config, onLog: (m) => console.log(m), isCancelled });
-      await pipeline.run();
-      break;
-    }
-
-    case 'status': {
-      console.log(`Library: ${config.libraryPath}`);
-      console.log(`Playlists: ${Object.keys(config.urlNames).length}`);
-      console.log(`Downloaded: ${Object.keys(config.lastUpdated || {}).length}`);
-      break;
-    }
-
-    default:
-      console.log(`Unknown command: ${cmd}\n`);
-      console.log(HELP);
+  const root = projectRoot();
+  if (!root) {
+    console.error('找不到專案根目錄（pubspec.yaml）。請在專案內執行，或設定 PA_ROOT');
+    process.exit(1);
+  }
+  const exe = exePath(root);
+  if (!exe) {
+    console.error(`找不到 build 好的 exe。
+請先在專案內執行: flutter build windows --release
+或設定 PA_ROOT=<專案根目錄>`);
+    process.exit(1);
   }
+  forward(exe, args);
 }
 
 main().catch((e) => {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "playlist-admin",
-  "version": "2.6.81",
-  "description": "Playlist Administrator CLI - music library pipeline, podcast pipeline and Spotube automation (Node.js)",
+  "version": "2.6.82",
+  "description": "Playlist Administrator CLI - thin launcher for the Flutter app engine (pipeline, podcast, spotube) + podcast RAG query",
   "type": "module",
   "bin": {
     "playlist-admin": "./cli/index.js"
@@ -16,10 +16,8 @@
     "spotube-cleanup": "node cli/index.js spotube-cleanup"
   },
   "files": [
-    "cli/",
-    "assets/zhcdict.json",
-    "README.md",
-    "COMMANDS.md"
+    "cli/index.js",
+    "rag/"
   ],
   "keywords": [
     "playlist",

package/rag/README.md

@@ -0,0 +1,48 @@
+# Podcast RAG
+
+對 podcast 逐字稿做本地語意檢索（embeddings + ChromaDB）。
+
+## 前置需求
+
+- 安裝 Ollama：`https://ollama.com/download`
+  ```bash
+  ollama pull bge-m3          # embedding 模型
+  ollama pull qwen2.5:7b      # （選用）本機生成
+  ```
+- Python 套件：
+  ```bash
+  pip install chromadb requests
+  ```
+- 逐字稿：`podcast_downloads/<節目>/*.txt`（podcast pipeline 會自動產生 .txt）
+
+## 指令
+
+CLI（需已 build exe 或直接 python 執行）：
+
+```bash
+playlist-admin rag build            # 增量索引（沒被 index 過的 .txt）
+playlist-admin rag build --reset    # 全部重建
+playlist-admin rag query "2026年台股會一直漲嗎?" --topk 8
+playlist-admin rag query "問題" --json        # JSON 含全文
+playlist-admin rag query "問題" --show 游庭皓  # 只搜指定節目
+```
+
+直接用 Python（路徑從 config.json 自動解析）：
+
+```bash
+python rag/build_db.py
+python rag/query.py "問題"
+python rag/srt_to_text.py            # 把缺 .txt 的 .srt 補轉成 .txt
+```
+
+## Pipeline 整合
+
+podcast pipeline 產生/更新逐字稿後，跑 `playlist-admin rag build`
+即可把新集數加進向量庫（程式會自動跳過已索引的檔名）。
+
+## 檔案
+
+- `build_db.py` — 逐字稿切 chunk → Ollama embeddings → ChromaDB（cosine）
+- `query.py` — 問題 embedding → top-k 片段 → 輸出命中的節目/日期/全文偏移
+- `srt_to_text.py` — srt → 純文字
+- `_resolve.py` — 從 app config 解析 podcast_downloads 與 chroma_db 路徑

package/rag/_resolve.py

@@ -0,0 +1,35 @@
+"""Resolve podcast RAG paths from the app config (same as the CLI/GUI)."""
+import json
+import os
+from pathlib import Path
+
+
+def find_base() -> Path:
+    env_bp = os.environ.get("BASE_PATH")
+    if env_bp:
+        return Path(env_bp)
+    local = os.environ.get("LOCALAPPDATA")
+    pointer = None
+    if local:
+        pointer = Path(local) / "Playlist Administrator" / "data" / "config.json"
+    try:
+        if pointer and pointer.exists():
+            data = json.loads(pointer.read_text(encoding="utf-8"))
+            bp = data.get("base_path") or data.get("basePath")
+            if bp:
+                return Path(bp)
+    except Exception:
+        pass
+    return Path(r"C:\Users\CPXru\Music\Spotube")
+
+
+def podcast_downloads_dir() -> Path:
+    return find_base() / "podcast_downloads"
+
+
+def rag_data_dir() -> Path:
+    return find_base() / "podcast_downloads" / "podcast_rag"
+
+
+def chroma_db_dir() -> Path:
+    return rag_data_dir() / "chroma_db"

package/rag/build_db.py

@@ -0,0 +1,213 @@
+"""把 podcast 逐字稿 (.txt) 建立成 ChromaDB 向量資料庫
+用法: python rag/build_db.py [--data DIR] [--db DIR] [--model MODEL] [--reset] [--workers N]
+"""
+import argparse
+import os
+import re
+import sys
+import hashlib
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import chromadb
+import requests
+
+from _resolve import podcast_downloads_dir, chroma_db_dir
+
+OLLAMA_URL = "http://localhost:11434"
+
+
+def embed_one_batch(model: str, texts: list[str]) -> tuple[list[str], list[list[float]]]:
+    """單批 embedding；400 時逐個重試，回傳 (成功文字, 對應向量)"""
+    resp = requests.post(
+        f"{OLLAMA_URL}/api/embed",
+        json={"model": model, "input": texts},
+        timeout=600,
+    )
+    if resp.status_code == 400:
+        kept: list[str] = []
+        vectors: list[list[float]] = []
+        for t in texts:
+            r2 = requests.post(
+                f"{OLLAMA_URL}/api/embed",
+                json={"model": model, "input": [t]},
+                timeout=600,
+            )
+            if r2.status_code == 400:
+                print(f"!! 跳過問題 chunk (len={len(t)}): {t[:100]!r}")
+                continue
+            r2.raise_for_status()
+            kept.append(t)
+            vectors.append(r2.json()["embeddings"][0])
+        return kept, vectors
+    resp.raise_for_status()
+    return texts, resp.json()["embeddings"]
+
+
+def embed_texts(model: str, texts: list[str], batch: int = 32, workers: int = 6) -> tuple[list[str], list[list[float]]]:
+    """併發呼叫 Ollama /api/embed；回傳 (成功文字, 對應向量)"""
+    batches = [texts[i:i + batch] for i in range(0, len(texts), batch)]
+    if len(batches) <= 1:
+        if not batches:
+            return [], []
+        return embed_one_batch(model, batches[0])
+
+    kept_all: list[str] = []
+    vectors: list[list[float]] = []
+    results: list[tuple[list[str], list[list[float]]]] = [None] * len(batches)
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        fut = {ex.submit(embed_one_batch, model, b): i for i, b in enumerate(batches)}
+        for f in as_completed(fut):
+            results[fut[f]] = f.result()
+    for k, v in results:
+        if k is None:
+            continue
+        kept_all.extend(k)
+        vectors.extend(v)
+    return kept_all, vectors
+
+
+def read_text_file(path: Path) -> str:
+    """讀檔，自動嘗試 UTF-8 / Big5 / GBK"""
+    raw = path.read_bytes()
+    for enc in ("utf-8-sig", "utf-8", "big5", "gb18030"):
+        try:
+            return raw.decode(enc)
+        except UnicodeDecodeError:
+            continue
+    return raw.decode("utf-8", errors="replace")
+
+
+def split_sentences(text: str) -> list[str]:
+    """依中文句號斷句，過濾空白行"""
+    text = re.sub(r"\s+", "", text)
+    parts = re.split(r"(?<=[。！？!?；;])", text)
+    return [p for p in parts if len(p) >= 4]
+
+
+def make_chunks(sentences: list[str], max_len: int = 600, overlap: int = 60) -> list[str]:
+    """句子接成 chunk，每塊約 max_len 字元，前後重疊 overlap；超長句強制硬切"""
+    chunks: list[str] = []
+    cur = ""
+    for s in sentences:
+        while len(s) > max_len:
+            if cur:
+                chunks.append(cur)
+                cur = cur[-overlap:] if len(cur) > overlap else cur
+            chunks.append(s[:max_len])
+            s = s[max_len:]
+        if len(cur) + len(s) > max_len and cur:
+            chunks.append(cur)
+            tail = cur[-overlap:] if len(cur) > overlap else cur
+            cur = tail + s
+        else:
+            cur += s
+    if cur:
+        chunks.append(cur)
+    return chunks
+
+
+def detect_meta(path: Path) -> dict:
+    """從檔名/路徑猜節目與日期"""
+    parts = path.parts
+    show = parts[-2] if len(parts) >= 2 else "unknown"
+    name = path.stem
+    m = re.search(r"(20\d{2})[_\-_](\d{1,2})[_\-_](\d{1,2})", name)
+    date = f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}" if m else ""
+    return {"show": show, "file": name, "date": date}
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--data", default=str(podcast_downloads_dir()))
+    ap.add_argument("--db", default=str(chroma_db_dir()))
+    ap.add_argument("--model", default="bge-m3")
+    ap.add_argument("--reset", action="store_true", help="重建資料庫")
+    ap.add_argument("--limit", type=int, default=0, help="只索引前 N 篇 (測試用)")
+    ap.add_argument("--workers", type=int, default=6, help="併發 embedding 數")
+    args = ap.parse_args()
+
+    client = chromadb.PersistentClient(path=args.db)
+    if args.reset:
+        try:
+            client.delete_collection("podcasts")
+        except Exception:
+            pass
+    col = client.get_or_create_collection(
+        "podcasts",
+        metadata={"hnsw:space": "cosine"},
+    )
+
+    # 已索引的檔名 (resume 用) - 分段讀取避免 SQL 變數超限
+    done_files = set()
+    offset = 0
+    while True:
+        batch = col.get(limit=5000, offset=offset, include=["metadatas"])["metadatas"]
+        if not batch:
+            break
+        for m in batch:
+            if m:
+                done_files.add(m.get("file", ""))
+        offset += 5000
+        if offset > 1000000:
+            break
+
+    files = [f for f in sorted(Path(args.data).rglob("*.txt")) if f.stem not in done_files]
+    if args.limit:
+        files = files[: args.limit]
+    print(f"待索引 {len(files)} 篇 (已跳過 {len(done_files)} 篇)")
+
+    total_chunks = 0
+    t_start = time.time()
+    for i, f in enumerate(files, 1):
+        text = read_text_file(f)
+        chunks = make_chunks(split_sentences(text))
+        # token 保險: 單 chunk 超過 2800 字元就再切
+        final_chunks: list[str] = []
+        for ch in chunks:
+            final_chunks.extend(
+                [ch[i:i + 2800] for i in range(0, len(ch), 2800)] if len(ch) > 2800 else [ch]
+            )
+        chunks = final_chunks
+        if not chunks:
+            continue
+        meta = detect_meta(f)
+        try:
+            kept_chunks, vectors = embed_texts(args.model, chunks, workers=args.workers)
+        except requests.exceptions.HTTPError as e:
+            print(f"!! HTTP 錯誤 檔案={meta['file']} 前300字={chunks[0][:300] if chunks else ''}")
+            raise
+        except requests.exceptions.ConnectionError:
+            print("無法連線 Ollama，請先啟動 (ollama serve) 並 pull bge-m3")
+            sys.exit(1)
+        if not kept_chunks:
+            continue
+        ids = [
+            hashlib.md5(f"{meta['file']}|{j}".encode()).hexdigest()
+            for j in range(len(kept_chunks))
+        ]
+        metadatas = [
+            {**meta, "chunk": j, "chars": len(c), "path": str(f)}
+            for j, c in enumerate(kept_chunks)
+        ]
+        col.add(ids=ids, documents=kept_chunks, metadatas=metadatas, embeddings=vectors)
+        total_chunks += len(kept_chunks)
+        if i % 5 == 0 or i == len(files):
+            elapsed = time.time() - t_start
+            speed = i / elapsed
+            eta = (len(files) - i) / speed if speed > 0 else 0
+            print(
+                f"[{i}/{len(files)}] {i/len(files)*100:.1f}% | "
+                f"{speed:.2f}篇/秒 | 剩餘ETA {eta/60:.1f}分 | "
+                f"chunks累計 {total_chunks} | {meta['show'][:10]}/{meta['file'][:25]}",
+                flush=True,
+            )
+
+    print(f"完成! 共 {len(files)} 篇, {total_chunks} 個 chunk")
+
+
+if __name__ == "__main__":
+    main()

package/rag/query.py

@@ -0,0 +1,140 @@
+"""RAG 檢索: 問題 -> bge-m3 -> top-k 片段 -> 自動附加命中檔案的完整原文
+用法:
+  python rag/query.py "問題" --topk 8          # 文字輸出
+  python rag/query.py "問題" --json            # JSON 輸出 (含 full_text)
+  python rag/query.py "問題" --no-full         # 不回傳全文, 只回傳命中片段
+"""
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import chromadb
+import requests
+
+from build_db import read_text_file
+from _resolve import chroma_db_dir
+
+OLLAMA_URL = "http://localhost:11434"
+
+
+def embed_one(model: str, text: str) -> list[float]:
+    resp = requests.post(
+        f"{OLLAMA_URL}/api/embed", json={"model": model, "input": [text]}, timeout=120
+    )
+    resp.raise_for_status()
+    return resp.json()["embeddings"][0]
+
+
+def normalize(text: str) -> str:
+    return re.sub(r"\s+", "", text)
+
+
+def locate_chunk(full_norm: str, chunk: str, window: int = 100) -> int:
+    head = chunk[:window]
+    idx = full_norm.find(head)
+    if idx < 0:
+        head = chunk[:window // 2]
+        idx = full_norm.find(head)
+    return idx
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("question", nargs="?", help="要查的問題")
+    ap.add_argument("--db", default=str(chroma_db_dir()))
+    ap.add_argument("--embed", default="bge-m3")
+    ap.add_argument("--topk", type=int, default=8)
+    ap.add_argument("--json", action="store_true", help="輸出 JSON (含 full_text)")
+    ap.add_argument("--no-full", action="store_true", help="不回傳全文")
+    ap.add_argument("--out", default="", help="JSON 輸出檔路徑 (與 --json 併用)")
+    ap.add_argument("--show", default="", help="只搜指定節目")
+    args = ap.parse_args()
+
+    if not args.question:
+        args.question = input("問題: ").strip()
+    if not args.question:
+        sys.exit(1)
+
+    client = chromadb.PersistentClient(path=args.db)
+    col = client.get_collection("podcasts")
+    qvec = embed_one(args.embed, args.question)
+    fetch_n = max(args.topk * 5, 50) if args.show else args.topk
+    res = col.query(query_embeddings=[qvec], n_results=fetch_n)
+
+    raw_hits = []
+    for doc, meta, dist in zip(
+        res["documents"][0], res["metadatas"][0], res["distances"][0]
+    ):
+        if args.show and args.show not in meta.get("show", ""):
+            continue
+        raw_hits.append({
+            "similarity": round(1 - dist, 3),
+            "show": meta.get("show", "?"),
+            "date": meta.get("date", ""),
+            "file": meta.get("file", "?"),
+            "path": meta.get("path", ""),
+            "chunk": doc,
+        })
+
+    by_file: dict[str, dict] = {}
+    for h in raw_hits:
+        key = h["path"] or h["file"]
+        if key not in by_file:
+            e = {k: h[k] for k in ("show", "date", "file", "path")}
+            e["hits"] = []
+            e["best_similarity"] = 0.0
+            by_file[key] = e
+        by_file[key]["hits"].append(h)
+        by_file[key]["best_similarity"] = max(
+            by_file[key]["best_similarity"], h["similarity"]
+        )
+
+    results = []
+    for key, e in sorted(
+        by_file.items(), key=lambda kv: kv[1]["best_similarity"], reverse=True
+    ):
+        entry = {
+            "show": e["show"],
+            "date": e["date"],
+            "file": e["file"],
+            "best_similarity": e["best_similarity"],
+            "hits": [
+                {"similarity": h["similarity"], "chunk": h["chunk"]}
+                for h in e["hits"]
+            ],
+        }
+        if not args.no_full and e["path"]:
+            full = read_text_file(Path(e["path"]))
+            full_norm = normalize(full)
+            entry["full_text"] = full
+            entry["full_text_norm_len"] = len(full_norm)
+            entry["offsets"] = [
+                locate_chunk(full_norm, h["chunk"]) for h in e["hits"]
+            ]
+        results.append(entry)
+
+    if args.json:
+        payload = {"question": args.question, "results": results}
+        out = json.dumps(payload, ensure_ascii=False, indent=2)
+        if args.out:
+            Path(args.out).write_text(out, encoding="utf-8")
+            print(f"已寫入 {args.out}")
+        else:
+            print(out)
+        return
+
+    for r in results:
+        print(f"\n[相似度 {r['best_similarity']}] {r['show']} | {r['date']} | {r['file']}")
+        if "full_text" in r:
+            print(f"  (全文 {len(r['full_text'])} 字, 命中偏移 {r['offsets']})")
+        for h in r["hits"][:2]:
+            print(f"  > {h['chunk'][:120]}...")
+
+
+if __name__ == "__main__":
+    main()