playlist-admin 2.15.22 → 2.15.24

package/cli/index.js

@@ -1,7 +1,8 @@
 #!/usr/bin/env node
-import { spawn } from 'node:child_process';
+import { spawn, spawnSync } from 'node:child_process';
 import fs from 'node:fs';
 import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 
 // Thin launcher: the CLI engine lives in the Flutter app binary
 // (lib/cli_main.dart), so GUI and CLI share one implementation.
@@ -37,17 +38,39 @@ function projectRoot() {
   }
 }
 
+/// npm 包自身目錄（全域 npm i -g 時，cwd 向上找不到 pubspec，
+/// 但包內自帶 cli/ + rag/，RAG 命令靠它就能跑）。
+function packageRoot() {
+  try {
+    return path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+  } catch {
+    return null;
+  }
+}
+
 function exePath(root) {
   const candidates = [
     path.join(root, 'build', 'windows', 'x64', 'runner', 'Release', 'playlist-admin.exe'),
     path.join(root, 'build', 'windows', 'x64', 'runner', 'Debug', 'playlist-admin.exe'),
+    path.join(root, 'build', 'windows', 'arm64', 'runner', 'Release', 'playlist-admin.exe'),
+    path.join(root, 'build', 'windows', 'arm64', 'runner', 'Debug', 'playlist-admin.exe'),
     path.join(root, 'build', 'windows', 'x64', 'runner', 'Release', 'playlist_administrator.exe'),
   ];
   return candidates.find((c) => fs.existsSync(c)) || null;
 }
 
 function python() {
-  return process.env.PYTHON || 'python';
+  if (process.env.PYTHON) return process.env.PYTHON;
+  // Windows 常只有 py launcher 或 python：依序探測可用的。
+  for (const cmd of ['py', 'python', 'python3']) {
+    try {
+      const r = spawnSync(cmd, ['--version'], { stdio: 'ignore', windowsHide: true });
+      if (r.status === 0) return cmd;
+    } catch {
+      // 試下一個
+    }
+  }
+  return 'python';
 }
 
 function forward(exe, args) {
@@ -82,15 +105,17 @@ function forwardPy(args) {
 }
 
 function runRag(args) {
-  const root = projectRoot();
-  if (!root) {
-    console.error('找不到專案根目錄（pubspec.yaml）。請在專案內執行，或設定 PA_ROOT');
+  const sub = args[0];
+  if (sub !== 'build' && sub !== 'query') {
+    console.error(`未知 rag 子命令: ${sub ?? '(空)'}\n用法: playlist-admin rag build | playlist-admin rag query "問題"`);
     process.exit(1);
   }
-  const sub = args[0];
-  const script = path.join(root, 'rag', sub === 'build' ? 'build_db.py' : 'query.py');
-  if (!fs.existsSync(script)) {
-    console.error(`找不到 ${script}`);
+  // 優先專案內 rag/（開發時最新），全域安裝時退回包內自帶的 rag/。
+  const roots = [projectRoot(), packageRoot()].filter(Boolean);
+  const fname = sub === 'build' ? 'build_db.py' : 'query.py';
+  const script = roots.map((r) => path.join(r, 'rag', fname)).find((s) => fs.existsSync(s));
+  if (!script) {
+    console.error(`找不到 rag/${fname}。請在專案內執行，或設定 PA_ROOT`);
     process.exit(1);
   }
   forwardPy([script, ...args.slice(1)]);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "playlist-admin",
-  "version": "2.15.22",
+  "version": "2.15.24",
   "description": "playlist-admin CLI - thin launcher for the Flutter app engine (pipeline, podcast) + podcast RAG query",
   "type": "module",
   "bin": {

package/rag/_resolve.py

@@ -32,8 +32,22 @@ def find_base() -> Path:
             bp = _win_to_wsl(bp)
         return bp
 
-    # WSL: 直接用 /mnt/c/ 路徑
+    # WSL：優先 HOME/Music 推導，寫死的個人路徑只當最後 fallback。
+    # 多使用者機器：優先 USERNAME 對得上的那個。
     if _is_wsl():
+        home_music = Path.home() / "Music" / "playlist-admin"
+        if home_music.exists():
+            return home_music
+        try:
+            hits = sorted(Path("/mnt/c/Users").glob("*/Music/playlist-admin"))
+            if hits:
+                me = (os.environ.get("USERNAME") or os.environ.get("USER") or "").lower()
+                for h in hits:
+                    if me and h.parts[3].lower() == me:
+                        return h
+                return hits[0]
+        except Exception:
+            pass
         return Path("/mnt/c/Users/CPXru/Music/playlist-admin")
 
     # Windows: 讀 config.json
@@ -45,7 +59,8 @@ def find_base() -> Path:
             pointer = Path(local) / "Playlist Administrator" / "data" / "config.json"
     try:
         if pointer and pointer.exists():
-            data = json.loads(pointer.read_text(encoding="utf-8"))
+            # utf-8-sig：BOM 檔用 utf-8 讀會讓 json.loads 炸掉。
+            data = json.loads(pointer.read_text(encoding="utf-8-sig"))
             bp = data.get("base_path") or data.get("basePath")
             if bp:
                 return Path(bp)

package/rag/build_db.py

@@ -85,10 +85,12 @@ def read_text_file(path: Path) -> str:
 
 
 def split_sentences(text: str) -> list[str]:
-    """依中文句號斷句，過濾空白行"""
-    text = re.sub(r"\s+", "", text)
+    """依中文句號斷句，過濾空白行。
+    空白壓成單空格不斷字：舊寫法刪掉所有空白，英文全黏在一起傷 embedding。
+    （query.py 的 normalize 已同步改同一規則，否則 offset 對不上。）"""
+    text = re.sub(r"\s+", " ", text).strip()
     parts = re.split(r"(?<=[。！？!?；;])", text)
-    return [p for p in parts if len(p) >= 4]
+    return [p.strip() for p in parts if len(p.strip()) >= 4]
 
 
 def make_chunks(sentences: list[str], max_len: int = 600, overlap: int = 60) -> list[str]:
@@ -135,7 +137,41 @@ def main() -> None:
     ap.add_argument("--batch", type=int, default=64, help="每批 embedding 數量")
     args = ap.parse_args()
 
+    # 先拿鎖再開 PersistentClient：反過來雙進程仍會同時開 SQLite。
+    lock_file = Path(args.db) / ".build.lock"
+    if lock_file.exists():
+        # stale 鎖：mtime 超過 30 分鐘（kill -9 殘留）直接清掉重拿。
+        try:
+            age = time.time() - lock_file.stat().st_mtime
+        except Exception:
+            age = 0
+        if age < 1800:
+            print("另一個 build_db 正在寫入此 DB，本次跳過（避免 ChromaDB 寫入衝突）")
+            sys.exit(2)
+        try:
+            lock_file.unlink()
+            print("清掉過期 build lock，繼續執行")
+        except Exception:
+            pass
+    try:
+        fd = os.open(str(lock_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+        with os.fdopen(fd, "w") as lf:
+            lf.write(f"{os.getpid()}|{time.time()}")
+    except FileExistsError:
+        print("另一個 build_db 正在寫入此 DB，本次跳過（避免 ChromaDB 寫入衝突）")
+        sys.exit(2)
+
     client = chromadb.PersistentClient(path=args.db)
+    try:
+        _main_build(client, args)
+    finally:
+        try:
+            lock_file.unlink()
+        except Exception:
+            pass
+
+
+def _main_build(client, args) -> None:
     if args.reset:
         try:
             client.delete_collection("podcasts")
@@ -192,13 +228,16 @@ def main() -> None:
         return hashlib.md5(raw).hexdigest()
 
     migrated = 0
+    to_update_ids: list[str] = []
+    to_update_metas: list[dict] = []
     offset = 0
     while True:
-        batch = col.get(limit=5000, offset=offset,
-                        include=["metadatas"])["metadatas"]
+        got = col.get(limit=5000, offset=offset, include=["metadatas"])
+        batch = got["metadatas"]
+        ids = got["ids"]
         if not batch:
             break
-        for m in batch:
+        for rid, m in zip(ids, batch):
             if not m or m.get("sig"):
                 continue
             p = m.get("path")
@@ -206,13 +245,26 @@ def main() -> None:
                 continue
             try:
                 sig = file_sig(Path(p))
+                m["sig"] = sig
                 done_sigs.add(sig)
+                to_update_ids.append(rid)
+                to_update_metas.append(m)
                 migrated += 1
             except Exception:
                 pass
         offset += 5000
         if offset > 1000000:
             break
+    # 寫回 DB：舊寫法只加記憶體 set，從不寫回，每次啟動重掃且刪檔殘留。
+    for i in range(0, len(to_update_ids), 4500):
+        try:
+            col.update(
+                ids=to_update_ids[i:i + 4500],
+                metadatas=to_update_metas[i:i + 4500],
+            )
+        except Exception as e:
+            print(f"migration write-back failed: {e}")
+            break
     if migrated:
         print(f"migrated {migrated} legacy chunks to content-sig dedup")
 
@@ -231,9 +283,31 @@ def main() -> None:
     def has_content(path: Path) -> bool:
         return bool(make_chunks(split_sentences(read_text_file(path))))
 
+    # 過濾非 podcast 的 .txt：目錄黑名單 + 檔名/大小門檻
+    _SKIP_DIRS = {'podcast_rag', '__pycache__', 'chroma_db', 'node_modules', '.git'}
+    _SKIP_STEMS = {'README', 'changelog', 'license', 'requirements', 'setup', 'config'}
+    _MIN_TXT_BYTES = 100  # podcast 逐字稿至少 ~100 bytes
+
+    def _is_valid_podcast_txt(f: Path) -> bool:
+        # 跳過黑名單目錄下的所有檔案
+        for part in f.relative_to(Path(args.data)).parts[:-1]:
+            if part in _SKIP_DIRS:
+                return False
+        # 跳過已知非逐字稿檔名
+        if f.stem.lower() in _SKIP_STEMS:
+            return False
+        # 跳過太短的檔案（通常是設定檔或說明）
+        try:
+            if f.stat().st_size < _MIN_TXT_BYTES:
+                return False
+        except OSError:
+            return False
+        return True
+
     files = [
         f for f in sorted(Path(args.data).rglob("*.txt"))
-        if file_sig(f) not in done_sigs
+        if _is_valid_podcast_txt(f)
+        and file_sig(f) not in done_sigs
         and not (f.stem in skipped and not has_content(f))
     ]
     if args.limit:
@@ -266,23 +340,37 @@ def main() -> None:
         sig = file_sig(f)
         try:
             kept_chunks, vectors = embed_texts(args.model, chunks, batch=args.batch, workers=args.workers)
-        except requests.exceptions.HTTPError as e:
-            print(f"!! HTTP 錯誤 檔案={meta['file']} 前300字={chunks[0][:300] if chunks else ''}")
-            raise
         except requests.exceptions.ConnectionError:
             print("無法連線 Ollama，請先啟動 (ollama serve) 並 pull bge-m3")
             sys.exit(1)
+        except Exception as e:
+            # 單檔失敗（500/429/timeout）不可中斷整個索引：
+            # 跳過，下次增量 run 會自動補上（dedup：沒寫入就不在 done_sigs）。
+            print(f"!! 跳過 {meta['file']}: {e}")
+            continue
         if not kept_chunks:
             continue
+        # ID 混入 show+sig+檔名：同節目同內容不同檔也互不相撞；
+        # 寫入成功立刻登記 done_sigs（批次內第二檔直接跳過，不等下次 run）。
+        try:
+            col.delete(where={"path": str(f)})
+        except Exception as e:
+            print(f"!! 清舊 chunks 失敗 {meta['file']}: {e}")
         ids = [
-            hashlib.md5(f"{meta['file']}|{j}".encode()).hexdigest()
+            hashlib.md5(f"{meta['show']}|{meta['file']}|{sig}|{j}".encode()).hexdigest()
             for j in range(len(kept_chunks))
         ]
         metadatas = [
             {**meta, "sig": sig, "chunk": j, "chars": len(c), "path": str(f)}
             for j, c in enumerate(kept_chunks)
         ]
-        col.add(ids=ids, documents=kept_chunks, metadatas=metadatas, embeddings=vectors)
+        try:
+            col.add(ids=ids, documents=kept_chunks, metadatas=metadatas, embeddings=vectors)
+        except Exception as e:
+            # DuplicateID 等單檔失敗不可崩整個 build：下次增量補上。
+            print(f"!! 寫入失敗跳過 {meta['file']}: {e}")
+            continue
+        done_sigs.add(sig)
         total_chunks += len(kept_chunks)
         if i % 5 == 0 or i == len(files):
             elapsed = time.time() - t_start

package/rag/query.py

@@ -36,7 +36,9 @@ def embed_one(model: str, text: str) -> list[float]:
 
 
 def normalize(text: str) -> str:
-    return re.sub(r"\s+", "", text)
+    # 與 build_db.split_sentences 同規則（單空格），否則 chunk offset 對不上。
+    # 注意：此規則變更前的舊 chunks 是無空白版，offset 可能為 -1（僅顯示用，不影響檢索）。
+    return re.sub(r"\s+", " ", text).strip()
 
 
 def locate_chunk(full_norm: str, chunk: str, window: int = 100) -> int:
@@ -64,8 +66,9 @@ def pick_chat_models() -> list[str]:
     return [*local, *cloud]
 
 
-def generate_answer(question: str, hits: list[dict]) -> str:
-    """用 Ollama 生成基於檢索片段的回答；全部模型都失敗就回傳錯誤訊息"""
+def generate_answer(question: str, hits: list[dict], quiet: bool = False) -> str:
+    """用 Ollama 生成基於檢索片段的回答；全部模型都失敗就回傳錯誤訊息。
+    quiet=True 時不印 token（--json 模式：stdout 只能有 JSON，否則解析失敗）。"""
     context = "\n".join(
         f"[{h['show']} | {h['date']} | {h['file']}]\n{h['chunk']}"
         for h in hits
@@ -101,10 +104,12 @@ def generate_answer(question: str, hits: list[dict]) -> str:
                     token = chunk.get("message", {}).get("content", "")
                     if token:
                         answer.append(token)
-                        print(token, end="", flush=True)
+                        if not quiet:
+                            print(token, end="", flush=True)
                 except json.JSONDecodeError:
                     pass
-            print()  # 換行
+            if not quiet:
+                print()  # 換行
             return "".join(answer).strip()
         except requests.RequestException as e:
             errors.append(f"{model}: {e}")
@@ -139,9 +144,10 @@ def main() -> None:
     res = col.query(query_embeddings=[qvec], n_results=fetch_n)
 
     raw_hits = []
-    for doc, meta, dist in zip(
-        res["documents"][0], res["metadatas"][0], res["distances"][0]
-    ):
+    docs = res["documents"][0] if res.get("documents") else []
+    metas = res["metadatas"][0] if res.get("metadatas") else []
+    dists = res["distances"][0] if res.get("distances") else []
+    for doc, meta, dist in zip(docs, metas, dists):
         if args.show and args.show not in meta.get("show", ""):
             continue
         raw_hits.append({
@@ -181,20 +187,32 @@ def main() -> None:
             ],
         }
         if not args.no_full and e["path"]:
-            full = read_text_file(Path(e["path"]))
-            full_norm = normalize(full)
-            entry["full_text"] = full
-            entry["full_text_norm_len"] = len(full_norm)
-            entry["offsets"] = [
-                locate_chunk(full_norm, h["chunk"]) for h in e["hits"]
-            ]
+            # 檔案被搬走/刪除時跳過全文（舊寫法直接拋 FileNotFoundError 整個查詢崩潰）。
+            # 缺檔也要寫固定鍵：下游取 full_text/offsets 不可 KeyError。
+            try:
+                full = read_text_file(Path(e["path"]))
+            except (OSError, FileNotFoundError):
+                full = ""
+            if full:
+                full_norm = normalize(full)
+                entry["full_text"] = full
+                entry["full_text_norm_len"] = len(full_norm)
+                entry["offsets"] = [
+                    locate_chunk(full_norm, h["chunk"]) for h in e["hits"]
+                ]
+            else:
+                entry["full_text"] = ""
+                entry["full_text_norm_len"] = 0
+                entry["offsets"] = []
         results.append(entry)
 
     if args.json:
         payload = {"question": args.question, "results": results}
         if args.answer:
             try:
-                payload["answer"] = generate_answer(args.question, raw_hits[: max(args.topk_answer, 3)])
+                payload["answer"] = generate_answer(
+                    args.question, raw_hits[: max(args.topk_answer, 3)], quiet=True
+                )
             except Exception as e:
                 payload["answer_error"] = f"(Ollama 生成失敗: {e})"
         out = json.dumps(payload, ensure_ascii=False, indent=2)

package/rag/srt_to_text.py

@@ -11,8 +11,20 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 from _resolve import podcast_downloads_dir
 
 
+def read_srt(srt_path: Path) -> str:
+    """讀檔：utf-8-sig → utf-8 → big5 → gb18030，和 build_db 同順序。
+    舊寫法只試 utf-8-sig，Big5 繁中字幕直接亂碼進庫。"""
+    raw = srt_path.read_bytes()
+    for enc in ("utf-8-sig", "utf-8", "big5", "gb18030"):
+        try:
+            return raw.decode(enc)
+        except UnicodeDecodeError:
+            continue
+    return raw.decode("utf-8", errors="replace")
+
+
 def srt_to_text(srt_path: Path) -> str:
-    text = srt_path.read_text(encoding="utf-8-sig", errors="replace")
+    text = read_srt(srt_path)
     # 移除 srt 索引行 (純數字行)
     text = re.sub(r"^\d+\s*$", "", text, flags=re.MULTILINE)
     # 移除時間軸行 (00:00:00,000 --> 00:00:01,000)