playlist-admin 2.15.24 → 2.15.25

package/cli/index.js

@@ -21,6 +21,8 @@ Usage:
   playlist-admin favorite toggle <song>  Toggle favorite (我的最愛)
   playlist-admin rag build [--reset]     Build podcast RAG vector DB
   playlist-admin rag query "問題" [--topk N] [--show 節目] [--json]
+  playlist-admin study build [--reset]   Build study RAG (PDF + 課程)
+  playlist-admin study query "問題" [--topk N] [--category X] [--json]
 `;
 
 function projectRoot() {
@@ -121,6 +123,22 @@ function runRag(args) {
   forwardPy([script, ...args.slice(1)]);
 }
 
+function runStudy(args) {
+  const sub = args[0];
+  if (sub !== 'build' && sub !== 'query') {
+    console.error(`未知 study 子命令: ${sub ?? '(空)'}\n用法: playlist-admin study build | playlist-admin study query "問題"`);
+    process.exit(1);
+  }
+  const roots = [projectRoot(), packageRoot()].filter(Boolean);
+  const fname = sub === 'build' ? 'study_build.py' : 'study_query.py';
+  const script = roots.map((r) => path.join(r, 'rag', fname)).find((s) => fs.existsSync(s));
+  if (!script) {
+    console.error(`找不到 rag/${fname}。請在專案內執行，或設定 PA_ROOT`);
+    process.exit(1);
+  }
+  forwardPy([script, ...args.slice(1)]);
+}
+
 async function main() {
   const args = process.argv.slice(2);
   if (args.length === 0) {
@@ -131,6 +149,10 @@ async function main() {
     runRag(args.slice(1));
     return;
   }
+  if (args[0] === 'study') {
+    runStudy(args.slice(1));
+    return;
+  }
 
   const root = projectRoot();
   if (!root) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "playlist-admin",
-  "version": "2.15.24",
-  "description": "playlist-admin CLI - thin launcher for the Flutter app engine (pipeline, podcast) + podcast RAG query",
+  "version": "2.15.25",
+  "description": "playlist-admin CLI - pipeline, podcast, music + podcast RAG + study RAG (PDF 試題/課本)",
   "type": "module",
   "bin": {
     "playlist-admin": "./cli/index.js"

package/rag/study_build.py

@@ -0,0 +1,311 @@
+"""Build study RAG: PDF 試題/課本 + 課程逐字稿 → ChromaDB study collection
+用法:
+  python rag/study_build.py                          # 增量建庫
+  python rag/study_build.py --reset                  # 清空重建
+  python rag/study_build.py --pdf-root <DIR>         # 自訂 PDF 根目錄
+  python rag/study_build.py --limit 10               # 只建前 10 個 PDF（測試用）
+"""
+import argparse
+import hashlib
+import os
+import re
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import chromadb
+import requests
+
+from _resolve import find_base, chroma_db_dir
+
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8")
+
+COLLECTION = "study"
+OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434")
+EMBED_MODEL = os.environ.get("OLLAMA_EMBED_MODEL", "bge-m3")
+
+# ── Embedding ────────────────────────────────────────────────────────
+
+def embed_one_batch(model: str, texts: list[str]) -> tuple[list[str], list[list[float]]]:
+    resp = requests.post(
+        f"{OLLAMA_URL}/api/embed",
+        json={"model": model, "input": texts},
+        timeout=120,
+    )
+    if resp.status_code == 400:
+        ok, vecs = [], []
+        for t in texts:
+            try:
+                r2 = requests.post(
+                    f"{OLLAMA_URL}/api/embed",
+                    json={"model": model, "input": [t]}, timeout=120,
+                )
+                r2.raise_for_status()
+                ok.append(t)
+                vecs.append(r2.json()["embeddings"][0])
+            except Exception:
+                pass
+        return ok, vecs
+    resp.raise_for_status()
+    return texts, resp.json()["embeddings"]
+
+
+def embed_texts(model: str, texts: list[str], batch: int = 32, workers: int = 6):
+    batches = [texts[i:i + batch] for i in range(0, len(texts), batch)]
+    if len(batches) <= 1:
+        return embed_one_batch(model, batches[0]) if batches else ([], [])
+    kept, vecs = [], []
+    results = [None] * len(batches)
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        fut = {ex.submit(embed_one_batch, model, b): i for i, b in enumerate(batches)}
+        for f in as_completed(fut):
+            results[fut[f]] = f.result()
+    for k, v in results:
+        if k is None:
+            continue
+        kept.extend(k)
+        vecs.extend(v)
+    return kept, vecs
+
+
+# ── PDF extraction ───────────────────────────────────────────────────
+
+def extract_pdf_text(pdf_path: Path) -> str:
+    """用 PyMuPDF 提取 PDF 文字"""
+    try:
+        import fitz
+        doc = fitz.open(str(pdf_path))
+        parts = []
+        for page in doc:
+            parts.append(page.get_text())
+        doc.close()
+        return "\n".join(parts)
+    except Exception as e:
+        return f"[PDF extraction error: {e}]"
+
+
+# ── Chunking ─────────────────────────────────────────────────────────
+
+def split_sentences(text: str) -> list[str]:
+    text = re.sub(r"\s+", " ", text).strip()
+    parts = re.split(r"(?<=[。！？!?；;\n])", text)
+    return [p.strip() for p in parts if len(p.strip()) >= 4]
+
+
+def make_chunks(sentences: list[str], max_len: int = 800, overlap: int = 80) -> list[str]:
+    """句子接成 chunk，max_len 較大（試題需要更多上下文）"""
+    chunks: list[str] = []
+    cur = ""
+    for s in sentences:
+        if len(cur) + len(s) > max_len and cur:
+            chunks.append(cur.strip())
+            # overlap: 保留尾巴
+            overlap_text = cur[-overlap:] if len(cur) > overlap else cur
+            cur = overlap_text + " " + s
+        else:
+            cur = cur + " " + s if cur else s
+    if cur.strip():
+        chunks.append(cur.strip())
+    # 超長強制硬切
+    final = []
+    for ch in chunks:
+        if len(ch) > 2800:
+            final.extend(ch[i:i + 2800] for i in range(0, len(ch), 2800))
+        else:
+            final.append(ch)
+    return final
+
+
+def file_sig(path: Path) -> str:
+    return hashlib.md5(str(path.resolve()).encode()).hexdigest()
+
+
+def content_sig(text: str) -> str:
+    return hashlib.md5(text[:4096].encode()).hexdigest()
+
+
+# ── PDF discovery ────────────────────────────────────────────────────
+
+def discover_pdfs(root: Path) -> list[tuple[str, Path]]:
+    """回傳 (category, pdf_path) 列表"""
+    results = []
+    for dirpath, dirnames, filenames in os.walk(str(root)):
+        dirnames[:] = [d for d in dirnames if not d.startswith('.')]
+        dp = Path(dirpath)
+        rel = dp.relative_to(root)
+        category = str(rel).replace("\\", "/") if str(rel) != "." else root.name
+        for f in sorted(filenames):
+            if f.lower().endswith(".pdf"):
+                results.append((category, dp / f))
+    return results
+
+
+def discover_txt_transcripts(podcasts_dir: Path) -> list[tuple[str, Path]]:
+    """掃描 podcast 資料夾中的課程目錄（非 RSS）"""
+    results = []
+    # 課程目錄名單（有 .txt 逐字稿的非 RSS 資料夾）
+    course_dirs = [
+        'T夜工智CoCo英文S', 'T夜工智張瑀國文S', 'T夜工智胡傑數學S',
+        '機械群劉徹機械力學', '機械群劉徹機械力學S',
+        '機械群許凱機件原理', '機械群許凱機件原理S',
+        '機械群陳海程機械製圖', '機械群陳海程機械製圖S',
+        '機械群陳海程機械製造與實習', '機械群陳海程機械製造與實習S',
+    ]
+    for name in course_dirs:
+        d = podcasts_dir / name
+        if not d.is_dir():
+            continue
+        for f in sorted(d.glob("*.txt")):
+            results.append((f"課程/{name}", f))
+    return results
+
+
+# ── Main ─────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description="Build study RAG")
+    ap.add_argument("--reset", action="store_true", help="清空重建")
+    ap.add_argument("--pdf-root", default="", help="PDF 根目錄")
+    ap.add_argument("--limit", type=int, default=0, help="只建前 N 個 PDF（測試）")
+    ap.add_argument("--db", default=str(chroma_db_dir()))
+    ap.add_argument("--embed", default=EMBED_MODEL)
+    ap.add_argument("--workers", type=int, default=6)
+    args = ap.parse_args()
+
+    # Resolve PDF root
+    if args.pdf_root:
+        pdf_root = Path(args.pdf_root)
+    else:
+        # Auto-discover: check Desktop, then common locations
+        home = Path.home()
+        candidates = [
+            home / "Desktop" / "四技二專歷屆試題與解答",
+            find_base() / "study_materials",
+        ]
+        pdf_root = None
+        for c in candidates:
+            if c.exists():
+                pdf_root = c
+                break
+        if pdf_root is None:
+            print("找不到 PDF 目錄。請用 --pdf-root 指定。")
+            sys.exit(1)
+
+    print(f"PDF 根目錄: {pdf_root}")
+
+    # Connect ChromaDB
+    client = chromadb.PersistentClient(path=args.db)
+    if args.reset:
+        try:
+            client.delete_collection(COLLECTION)
+            print("已清空 study collection")
+        except Exception:
+            pass
+    col = client.get_or_create_collection(
+        COLLECTION,
+        metadata={"hnsw:space": "cosine"},
+    )
+
+    # Existing signatures (dedup)
+    existing = col.get(include=["metadatas"])
+    done_sigs: set[str] = set()
+    if existing["ids"]:
+        for meta in existing["metadatas"]:
+            sig = meta.get("sig", "")
+            if sig:
+                done_sigs.add(sig)
+    print(f"已存在 {len(existing['ids'])} 筆, 已知 sig {len(done_sigs)} 個")
+
+    # Discover files
+    pdfs = discover_pdfs(pdf_root)
+    transcripts = discover_txt_transcripts(find_base() / "podcasts")
+    all_files = [(cat, p, "pdf") for cat, p in pdfs] + [(cat, p, "txt") for cat, p in transcripts]
+    print(f"發現 {len(pdfs)} 個 PDF, {len(transcripts)} 個逐字稿")
+
+    if args.limit:
+        all_files = all_files[:args.limit]
+        print(f"測試模式: 只建前 {args.limit} 個")
+
+    # Extract → chunk → embed
+    total_chunks = 0
+    t_start = time.time()
+
+    for fi, (category, fpath, ftype) in enumerate(all_files, 1):
+        if ftype == "pdf":
+            sig = file_sig(fpath)
+            if sig in done_sigs:
+                continue
+            text = extract_pdf_text(fpath)
+        else:
+            sig = file_sig(fpath)
+            if sig in done_sigs:
+                continue
+            text = fpath.read_text(encoding="utf-8-sig", errors="replace")
+
+        if not text or len(text.strip()) < 20:
+            continue
+
+        chunks = make_chunks(split_sentences(text))
+        if not chunks:
+            continue
+
+        # Token safety: 2800 chars per chunk
+        final_chunks = []
+        for ch in chunks:
+            if len(ch) > 2800:
+                final_chunks.extend(ch[i:i + 2800] for i in range(0, len(ch), 2800))
+            else:
+                final_chunks.append(ch)
+        chunks = final_chunks
+
+        # Embed
+        kept, vectors = embed_texts(args.embed, chunks, workers=args.workers)
+        if not kept:
+            continue
+
+        # Write to ChromaDB
+        prefix = f"study_{sig[:8]}"
+        ids = [f"{prefix}_{i}" for i in range(len(kept))]
+        metas = [
+            {
+                "category": category,
+                "file": fpath.name,
+                "path": str(fpath),
+                "sig": sig if i == 0 else "",  # only first chunk carries sig
+                "chunk_index": i,
+                "total_chunks": len(kept),
+                "source_type": ftype,
+            }
+            for i in range(len(kept))
+        ]
+
+        batch_size = 4500
+        for bi in range(0, len(ids), batch_size):
+            col.add(
+                ids=ids[bi:bi + batch_size],
+                documents=kept[bi:bi + batch_size],
+                metadatas=metas[bi:bi + batch_size],
+                embeddings=vectors[bi:bi + batch_size],
+            )
+
+        total_chunks += len(kept)
+        elapsed = time.time() - t_start
+        pct = fi / len(all_files) * 100
+        rate = fi / elapsed if elapsed > 0 else 0
+        eta = (len(all_files) - fi) / rate if rate > 0 else 0
+        print(
+            f"\r  [{fi}/{len(all_files)}] {pct:.1f}% "
+            f"| {rate:.2f} files/s | ETA {eta / 60:.1f}min "
+            f"| chunks累計 {total_chunks} | {category}/{fpath.name[:30]}",
+            end="", flush=True,
+        )
+
+    print(f"\n完成! 共 {total_chunks} 個新 chunk, 總計 {col.count()} 筆")
+
+
+if __name__ == "__main__":
+    main()

package/rag/study_query.py

@@ -0,0 +1,180 @@
+"""Study RAG 查詢: 問題 → bge-m3 → top-k 片段
+用法:
+  python rag/study_query.py "問題"                    # 文字輸出
+  python rag/study_query.py "問題" --json             # JSON 輸出
+  python rag/study_query.py "問題" --topk 5           # 前 5 個結果
+  python rag/study_query.py "問題" --category 機械    # 只搜某分類
+  python rag/study_query.py "問題" --answer           # Ollama 生成回答
+"""
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import chromadb
+import requests
+
+from _resolve import chroma_db_dir
+
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8")
+
+OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434")
+CHAT_MODEL = os.environ.get("OLLAMA_CHAT_MODEL", "qwen2.5:7b")
+COLLECTION = "study"
+
+
+def embed_one(model: str, text: str) -> list[float]:
+    resp = requests.post(
+        f"{OLLAMA_URL}/api/embed", json={"model": model, "input": [text]}, timeout=120
+    )
+    resp.raise_for_status()
+    return resp.json()["embeddings"][0]
+
+
+def read_file(path_str: str) -> str:
+    p = Path(path_str)
+    if not p.exists():
+        return ""
+    for enc in ("utf-8-sig", "utf-8", "big5", "gb18030"):
+        try:
+            return p.read_text(encoding=enc)
+        except (UnicodeDecodeError, UnicodeError):
+            continue
+    return p.read_text(encoding="utf-8", errors="replace")
+
+
+def pick_chat_models() -> list[str]:
+    forced = os.environ.get("OLLAMA_CHAT_MODEL", "").strip()
+    try:
+        tags = requests.get(f"{OLLAMA_URL}/api/tags", timeout=30).json().get("models", [])
+    except Exception:
+        return [forced] if forced else []
+    local = [m["name"] for m in tags if m.get("size", 0) > 0]
+    cloud = [m["name"] for m in tags if m.get("size", 0) == 0]
+    banned = {"bge-m3", "bge-m3:latest"}
+    local = [m for m in local if m not in banned]
+    if forced:
+        return [forced, *local, *cloud]
+    return [*local, *cloud]
+
+
+def generate_answer(question: str, hits: list[dict], quiet: bool = False) -> str:
+    context = "\n".join(
+        f"[{h['category']} | {h['file']}]\n{h['chunk']}" for h in hits
+    )
+    prompt = (
+        "你是學習助理，專精四技二專統一入學測驗與機械群專業科目。"
+        "請只根據以下學習資料片段回答使用者的問題，"
+        "回答用繁體中文，簡潔、分點陳述，並在結尾列出引用來源。\n\n"
+        f"片段資料：\n{context}\n\n問題：{question}"
+    )
+    errors = []
+    for model in pick_chat_models():
+        try:
+            resp = requests.post(
+                f"{OLLAMA_URL}/api/chat",
+                json={"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True},
+                timeout=600, stream=True,
+            )
+            if resp.status_code in (400, 401, 404):
+                errors.append(f"{model}: HTTP {resp.status_code}")
+                continue
+            resp.raise_for_status()
+            answer = []
+            for line in resp.iter_lines():
+                if not line:
+                    continue
+                try:
+                    chunk = json.loads(line)
+                    token = chunk.get("message", {}).get("content", "")
+                    if token:
+                        answer.append(token)
+                        if not quiet:
+                            print(token, end="", flush=True)
+                except json.JSONDecodeError:
+                    pass
+            if not quiet:
+                print()
+            return "".join(answer).strip()
+        except requests.RequestException as e:
+            errors.append(f"{model}: {e}")
+    if errors:
+        raise RuntimeError(" | ".join(errors))
+    return ""
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Study RAG query")
+    ap.add_argument("question", nargs="?", help="要查的問題")
+    ap.add_argument("--db", default=str(chroma_db_dir()))
+    ap.add_argument("--embed", default="bge-m3")
+    ap.add_argument("--topk", type=int, default=5)
+    ap.add_argument("--json", action="store_true")
+    ap.add_argument("--category", default="", help="只搜某分類")
+    ap.add_argument("--answer", action="store_true", help="用 Ollama 生成回答")
+    ap.add_argument("--topk-answer", type=int, default=4)
+    args = ap.parse_args()
+
+    if not args.question:
+        args.question = input("問題: ").strip()
+    if not args.question:
+        sys.exit(1)
+
+    client = chromadb.PersistentClient(path=args.db)
+    try:
+        col = client.get_collection(COLLECTION)
+    except Exception:
+        print(f"找不到 {COLLECTION} collection。請先執行: python rag/study_build.py")
+        sys.exit(1)
+
+    qvec = embed_one(args.embed, args.question)
+    fetch_n = max(args.topk * 5, 30) if args.category else args.topk
+    where_filter = {"category": {"$contains": args.category}} if args.category else None
+    res = col.query(
+        query_embeddings=[qvec],
+        n_results=fetch_n,
+        where=where_filter,
+    )
+
+    raw_hits = []
+    docs = res["documents"][0] if res.get("documents") else []
+    metas = res["metadatas"][0] if res.get("metadatas") else []
+    dists = res["distances"][0] if res.get("distances") else []
+    for doc, meta, dist in zip(docs, metas, dists):
+        if args.category and args.category not in meta.get("category", ""):
+            continue
+        raw_hits.append({
+            "similarity": round(1 - dist, 3),
+            "category": meta.get("category", "?"),
+            "file": meta.get("file", "?"),
+            "path": meta.get("path", ""),
+            "source_type": meta.get("source_type", "?"),
+            "chunk": doc,
+        })
+
+    if args.json:
+        payload = {"question": args.question, "results": raw_hits[:args.topk]}
+        if args.answer:
+            try:
+                payload["answer"] = generate_answer(
+                    args.question, raw_hits[:args.topk_answer], quiet=True
+                )
+            except Exception as e:
+                payload["answer_error"] = str(e)
+        print(json.dumps(payload, ensure_ascii=False, indent=2))
+        return
+
+    print(f"\n🔍 {args.question}\n")
+    for i, h in enumerate(raw_hits[:args.topk], 1):
+        print(f"[{i}] 類似度 {h['similarity']} | {h['source_type']} | {h['category']}/{h['file']}")
+        print(f"    {h['chunk'][:150]}...")
+        print()
+
+
+if __name__ == "__main__":
+    main()