videonote 0.1.7__py3-none-any.whl

app/__init__.py

@@ -0,0 +1,2 @@
+# vendored 自 BiliNote 的核心流水线包。
+# 上游 app/__init__.py 创建 FastAPI app（Web 层），本仓库剥离后留空作为包标记。

app/db/builtin_providers.json

@@ -0,0 +1,68 @@
+[
+  {
+    "id": "openai",
+    "name": "OpenAI",
+    "type": "built-in",
+    "logo": "OpenAI",
+    "api_key": "",
+    "base_url": "https://api.openai.com/v1"
+  },
+  {
+    "id": "deepseek",
+    "name": "DeepSeek",
+    "type": "built-in",
+    "logo": "DeepSeek",
+    "api_key": "",
+    "base_url": "https://api.deepseek.com",
+    "models": [
+      "deepseek-chat",
+      "deepseek-reasoner"
+    ]
+  },
+  {
+    "id": "qwen",
+    "name": "Qwen",
+    "type": "built-in",
+    "logo": "Qwen",
+    "api_key": "",
+    "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+    "models": [
+      "qwen-plus",
+      "qwen-turbo",
+      "qwen-max",
+      "qwen-long"
+    ]
+  },
+  {
+    "id": "Claude",
+    "name": "Claude",
+    "type": "built-in",
+    "logo": "Claude",
+    "api_key": "",
+    "base_url": "https://api.anthropic.com/v1/"
+  },
+  {
+    "id": "gemini",
+    "name": "Gemini",
+    "type": "built-in",
+    "logo": "Gemini",
+    "api_key": "",
+    "base_url": "https://generativelanguage.googleapis.com/v1beta/openai/"
+  },
+  {
+    "id": "groq",
+    "name": "Groq",
+    "type": "built-in",
+    "logo": "Groq",
+    "api_key": "",
+    "base_url": "https://api.groq.com/openai/v1"
+  },
+  {
+    "id": "ollama",
+    "name": "ollama",
+    "type": "built-in",
+    "logo": "Ollama",
+    "api_key": "",
+    "base_url": "http://127.0.0.1:11434/v1"
+  }
+]

app/db/engine.py

@@ -0,0 +1,71 @@
+import os
+
+from dotenv import load_dotenv
+from sqlalchemy import create_engine, event
+from sqlalchemy.orm import declarative_base, sessionmaker
+
+from app.utils.path_helper import get_data_dir
+
+if not os.environ.get("VIDEONOTE_DATA_DIR"):
+    load_dotenv()
+
+# 默认 SQLite，如果想换 PostgreSQL 或 MySQL，可以直接改 .env。
+# 默认路径固定到稳定数据目录（与 videonote_mcp.config 的 setdefault 同值）——
+# 相对路径 `sqlite:///video_note.db` 会随进程 CWD 漂移，在仓库根/其它目录
+# 跑脚本或测试会分裂出多个互不相通的 DB（本仓库根目录曾泄漏出 video_note.db）。
+DATABASE_URL = os.getenv("DATABASE_URL") or f"sqlite:///{os.path.join(get_data_dir(), 'video_note.db')}"
+
+# SQLite 需要特定连接参数，其他数据库不需要
+engine_args = {}
+if DATABASE_URL.startswith("sqlite"):
+    engine_args["connect_args"] = {
+        "check_same_thread": False,
+        # 连接级 busy timeout（秒）：MCP 多线程并发写（任务索引/provider 更新）
+        # 时避免立刻抛 "database is locked"
+        "timeout": 30,
+    }
+
+_pool_args = {}
+if not DATABASE_URL.startswith("sqlite"):
+    _pool_args = {
+        "pool_size": int(os.getenv("DB_POOL_SIZE", "10")),
+        "max_overflow": int(os.getenv("DB_MAX_OVERFLOW", "20")),
+        "pool_pre_ping": True,
+    }
+
+engine = create_engine(
+    DATABASE_URL,
+    echo=os.getenv("SQLALCHEMY_ECHO", "false").lower() == "true",
+    **engine_args,
+    **_pool_args,
+)
+
+if DATABASE_URL.startswith("sqlite"):
+    @event.listens_for(engine, "connect")
+    def _set_sqlite_pragma(dbapi_conn, connection_record):
+        """SQLite 并发写优化：WAL + busy_timeout + NORMAL 同步。
+
+        否则两个线程并发写会互锁抛 OperationalError: database is locked；
+        WAL 让读写不互斥，busy_timeout 让短竞争自旋而非立刻失败。
+        """
+        cur = dbapi_conn.cursor()
+        cur.execute("PRAGMA journal_mode=WAL")
+        cur.execute("PRAGMA busy_timeout=30000")
+        cur.execute("PRAGMA synchronous=NORMAL")
+        cur.close()
+
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+Base = declarative_base()
+
+
+def get_engine():
+    return engine
+
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

app/db/init_db.py

@@ -0,0 +1,60 @@
+import logging
+
+from app.db.engine import Base, get_engine
+
+logger = logging.getLogger(__name__)
+
+# video_tasks 表在数据层重构中新增的列（SQLite ALTER 幂等迁移用）
+_VIDEO_TASK_MIGRATIONS = [
+    ("title", "VARCHAR"),
+    ("status", "VARCHAR"),
+    ("summary", "VARCHAR"),
+    ("note_dir", "VARCHAR"),
+]
+
+
+def _migrate_video_tasks(engine):
+    """给已存在的 video_tasks 表补新增列（create_all 不会改已有表）。
+
+    SQLite 的 ALTER TABLE ADD COLUMN 是幂等的；用 PRAGMA table_info 检查列，
+    缺失才补，避免重复 ADD 报错。
+    """
+
+    with engine.connect() as conn:
+        cols = {row[1] for row in conn.exec_driver_sql("PRAGMA table_info(video_tasks)")}
+        for name, typ in _VIDEO_TASK_MIGRATIONS:
+            if name not in cols:
+                conn.exec_driver_sql(
+                    f"ALTER TABLE video_tasks ADD COLUMN {name} {typ}"
+                )
+
+
+def init_db():
+    engine = get_engine()
+
+    Base.metadata.create_all(bind=engine)
+    try:
+        _migrate_video_tasks(engine)
+    except Exception:
+        # 迁移失败不致命（表可能不存在 / 已是最新），保底 create_all 已建表
+        pass
+    _create_indexes(engine)
+
+
+# 常用查询列索引（docs/05 第 16 轮 C8）：任务量上万后 get_task_by_video /
+# get_provider_by_name / get_models_by_provider 从全表扫描降为索引查找。
+_INDEXES = [
+    "CREATE INDEX IF NOT EXISTS ix_video_tasks_video ON video_tasks(video_id, platform)",
+    "CREATE INDEX IF NOT EXISTS ix_providers_name ON providers(name)",
+    "CREATE INDEX IF NOT EXISTS ix_models_provider ON models(provider_id)",
+]
+
+
+def _create_indexes(engine) -> None:
+    with engine.connect() as conn:
+        for ddl in _INDEXES:
+            try:
+                conn.exec_driver_sql(ddl)
+            except Exception:  # noqa: BLE001 —— 表尚未创建（时序/旧库）幂等跳过，下次 init_db 补建
+                logger.warning("建索引失败（表可能尚未创建，跳过）: %s", ddl)
+        conn.commit()

app/db/model_dao.py

@@ -0,0 +1,55 @@
+from app.db.engine import get_db
+from app.db.models.models import Model
+
+
+def get_model_by_provider_and_name(provider_id: int, model_name: str):
+    db = next(get_db())
+    try:
+        model = db.query(Model).filter_by(provider_id=provider_id, model_name=model_name).first()
+        if model:
+            return {
+                "id": model.id,
+                "provider_id": model.provider_id,
+                "model_name": model.model_name,
+                "created_at": model.created_at,
+            }
+        return None
+    finally:
+        db.close()
+
+
+def insert_model(provider_id: int, model_name: str):
+    db = next(get_db())
+    try:
+        model = Model(provider_id=provider_id, model_name=model_name)
+        db.add(model)
+        db.commit()
+        db.refresh(model)
+        return {
+            "id": model.id,
+            "provider_id": model.provider_id,
+            "model_name": model.model_name,
+            "created_at": model.created_at,
+        }
+    finally:
+        db.close()
+
+
+def get_models_by_provider(provider_id: int):
+    db = next(get_db())
+    try:
+        models = db.query(Model).filter_by(provider_id=provider_id).all()
+        return [{"id": m.id, "model_name": m.model_name} for m in models]
+    finally:
+        db.close()
+
+
+def delete_model(model_id: int):
+    db = next(get_db())
+    try:
+        model = db.query(Model).filter_by(id=model_id).first()
+        if model:
+            db.delete(model)
+            db.commit()
+    finally:
+        db.close()

app/db/models/models.py

@@ -0,0 +1,12 @@
+from sqlalchemy import Column, DateTime, Integer, String, func
+
+from app.db.engine import Base
+
+
+class Model(Base):
+    __tablename__ = "models"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    provider_id = Column(Integer, nullable=False)
+    model_name = Column(String, nullable=False)
+    created_at = Column(DateTime, server_default=func.now())

app/db/models/providers.py

@@ -0,0 +1,16 @@
+from sqlalchemy import Column, DateTime, Integer, String, func
+
+from app.db.engine import Base
+
+
+class Provider(Base):
+    __tablename__ = "providers"
+
+    id = Column(String, primary_key=True)
+    name = Column(String, nullable=False)
+    logo = Column(String, nullable=False)
+    type = Column(String, nullable=False)
+    api_key = Column(String, nullable=False)
+    base_url = Column(String, nullable=False)
+    enabled = Column(Integer, default=1)
+    created_at = Column(DateTime, server_default=func.now())

app/db/models/video_tasks.py

@@ -0,0 +1,18 @@
+from sqlalchemy import Column, DateTime, Integer, String, func
+
+from app.db.engine import Base
+
+
+class VideoTask(Base):
+    __tablename__ = "video_tasks"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    video_id = Column(String, nullable=False)
+    platform = Column(String, nullable=False)
+    task_id = Column(String, unique=True, nullable=False)
+    created_at = Column(DateTime, server_default=func.now())
+    # —— 数据层重构新增：全局任务索引的语义元数据 ——
+    title = Column(String, default="", nullable=True)        # 语义标题（视频标题 / LLM 标题）
+    status = Column(String, default="", nullable=True)       # 最新状态（SUCCESS/FAILED/CANCELLED…）
+    summary = Column(String, default="", nullable=True)      # 语义简介（转写前若干字）
+    note_dir = Column(String, nullable=True)                 # 任务文件夹路径（note_results/{task_id}）

app/db/provider_dao.py

@@ -0,0 +1,147 @@
+import json
+import os
+import sys
+
+from app.db.engine import get_db
+from app.db.models.providers import Provider
+from app.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def get_builtin_providers_path():
+    if getattr(sys, 'frozen', False):
+        base_path = sys._MEIPASS
+    else:
+        base_path = os.path.dirname(__file__)
+    return os.path.join(base_path, 'builtin_providers.json')
+
+
+def seed_default_providers():
+    db = next(get_db())
+    try:
+        if db.query(Provider).count() > 0:
+            logger.info("Providers already exist, skipping seed.")
+            return
+
+        json_path = get_builtin_providers_path()
+        try:
+            with open(json_path, 'r', encoding='utf-8') as f:
+                providers = json.load(f)
+        except Exception as e:
+            logger.error(f"Failed to read builtin_providers.json: {e}")
+            return
+
+        for p in providers:
+            db.add(Provider(
+                id=p['id'],
+                name=p['name'],
+                api_key=p['api_key'],
+                base_url=p['base_url'],
+                logo=p['logo'],
+                type=p['type'],
+                enabled=p.get('enabled', 1)
+            ))
+        db.commit()
+        logger.info("Default providers seeded successfully.")
+    except Exception as e:
+        logger.error(f"Failed to seed default providers: {e}")
+    finally:
+        db.close()
+
+
+def insert_provider(id: str, name: str, api_key: str, base_url: str, logo: str, type_: str, enabled: int = 1):
+    db = next(get_db())
+    try:
+        # 落盘加密（docs/05 #29）；明文兼容：加密失败自动回退明文写入。
+        # 惰性 import：app/ 是 vendored 层，不强制依赖 videonote_mcp（上游可独立跑）
+        from videonote_mcp.crypto import encrypt_value
+
+        provider = Provider(
+            id=id, name=name, api_key=encrypt_value(api_key), base_url=base_url,
+            logo=logo, type=type_, enabled=enabled,
+        )
+        db.add(provider)
+        db.commit()
+        logger.info(f"Provider inserted successfully. id: {id}, name: {name}, type: {type_}")
+        return id
+    except Exception as e:
+        logger.error(f"Failed to insert provider: {e}")
+    finally:
+        db.close()
+
+
+def get_provider_by_name(name: str):
+    db = next(get_db())
+    try:
+        return db.query(Provider).filter_by(name=name).first()
+    finally:
+        db.close()
+
+
+def get_provider_by_id(id: str):
+    db = next(get_db())
+    try:
+        return db.query(Provider).filter_by(id=id).first()
+    finally:
+        db.close()
+
+
+def get_all_providers():
+    db = next(get_db())
+    try:
+        return db.query(Provider).all()
+    finally:
+        db.close()
+
+
+def update_provider(id: str, **kwargs):
+    db = next(get_db())
+    try:
+        provider = db.query(Provider).filter_by(id=id).first()
+        if not provider:
+            logger.warning(f"Provider {id} not found for update.")
+            return
+
+        for key, value in kwargs.items():
+            if hasattr(provider, key):
+                if key == "api_key":
+                    # 落盘加密（docs/05 #29）。update 通常先读后写（已解密），
+                    # 但 enc: 前缀值会二次加密——先解密再加密保持幂等。
+                    # 解密失败（key 缺失/不匹配）时跳过写入而非二次加密：
+                    # 二次加密会把 enc: 串再包一层，产生永远解不出的数据（docs 审计 G1）
+                    from videonote_mcp.crypto import decrypt_value, encrypt_value
+
+                    if value and str(value).startswith("enc:"):
+                        decrypted = decrypt_value(value)
+                        if decrypted is None:
+                            logger.warning(
+                                f"Provider {id} 的 api_key 无法解密（fernet.key 缺失/不匹配），"
+                                f"跳过该字段更新"
+                            )
+                            continue
+                        value = encrypt_value(decrypted)
+                    else:
+                        value = encrypt_value(value)
+                setattr(provider, key, value)
+
+        db.commit()
+        logger.info(f"Provider updated successfully. id: {id}, updated_fields: {list(kwargs.keys())}")
+    except Exception as e:
+        logger.error(f"Failed to update provider: {e}")
+    finally:
+        db.close()
+
+
+def delete_provider(id: str):
+    db = next(get_db())
+    try:
+        provider = db.query(Provider).filter_by(id=id).first()
+        if provider:
+            db.delete(provider)
+            db.commit()
+            logger.info(f"Provider deleted successfully. id: {id}")
+    except Exception as e:
+        logger.error(f"Failed to delete provider: {e}")
+    finally:
+        db.close()

app/db/video_task_dao.py

@@ -0,0 +1,159 @@
+from typing import Optional
+
+from app.db.engine import get_db
+from app.db.models.video_tasks import VideoTask
+from app.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+# 插入/更新任务（task_id 冲突时更新语义元数据，而非抛错）
+def insert_video_task(
+    video_id: str,
+    platform: str,
+    task_id: str,
+    title: str = "",
+    status: str = "",
+    summary: str = "",
+    note_dir: str = None,
+):
+    db = next(get_db())
+    try:
+        existing = db.query(VideoTask).filter_by(task_id=task_id).first()
+        if existing:
+            existing.video_id = video_id
+            existing.platform = platform
+            if title:
+                existing.title = title
+            if status:
+                existing.status = status
+            if summary:
+                existing.summary = summary
+            if note_dir is not None:
+                existing.note_dir = note_dir
+        else:
+            db.add(
+                VideoTask(
+                    video_id=video_id,
+                    platform=platform,
+                    task_id=task_id,
+                    title=title,
+                    status=status,
+                    summary=summary,
+                    note_dir=note_dir,
+                )
+            )
+        db.commit()
+        # title 可能是 None（无标题视频/平台），f-string 里切片会 TypeError——
+        # 行已入库却被记成「插入失败」，排查指向错误方向（#126 B8）
+        logger.info(f"Video task saved. task_id={task_id}, title={(title or '')[:40]!r}")
+    except Exception as e:
+        logger.error(f"Failed to insert video task: {e}")
+    finally:
+        db.close()
+
+
+# 更新任务状态（每次 _update_status 时同步到全局索引）
+def update_task_status(task_id: str, status: str, message: str = ""):
+    db = next(get_db())
+    try:
+        task = db.query(VideoTask).filter_by(task_id=task_id).first()
+        if task:
+            task.status = status
+            # 占位文案（"任务排队中"）只作状态消息、不落 summary（docs/05 第 16 轮 B7）：
+            # 步骤任务（transcribe_media 等）成功后 summary 曾恒为"任务排队中"误导 Agent。
+            # 语义简介由调用方显式写（_save_metadata / insert_video_task）。
+            if message and not task.summary and message != "任务排队中":
+                task.summary = message
+            if status == "SUCCESS" and task.summary == "任务排队中":
+                task.summary = ""
+        else:
+            logger.warning(f"update_task_status: task {task_id} 不在全局索引，跳过")
+        db.commit()
+    except Exception as e:
+        logger.error(f"Failed to update task status: {e}")
+    finally:
+        db.close()
+
+
+# 列出全部任务（全局索引），供 list_tasks 工具 / setup 数据管理
+def list_tasks(limit: Optional[int] = None, offset: int = 0) -> list:
+    db = next(get_db())
+    try:
+        # 分页下推到 SQL（LIMIT/OFFSET），任务量大的会话不再全表拉回 Python 层切片
+        query = db.query(VideoTask).order_by(VideoTask.created_at.desc()).offset(max(0, int(offset or 0)))
+        if limit is not None:
+            query = query.limit(max(1, int(limit)))
+        rows = query.all()
+        return [
+            {
+                "task_id": r.task_id,
+                "video_id": r.video_id,
+                "platform": r.platform,
+                "title": r.title or "",
+                "status": r.status or "",
+                "summary": r.summary or "",
+                "note_dir": r.note_dir,
+                "created_at": r.created_at.isoformat() if r.created_at else None,
+            }
+            for r in rows
+        ]
+    except Exception as e:
+        logger.error(f"Failed to list tasks: {e}")
+        return []
+    finally:
+        db.close()
+
+
+# 清空全局索引（cleanup_all 用）：单条 DELETE 替代 list + N 条单删（#125 B12）。
+# 返回受影响行数；失败时抛给调用方显式处理（空表不是失败）。调用方保证只在
+# 显式 cleanup_all 语义下调用，绝不因卸载/升级触发。
+def delete_all_tasks() -> int:
+    db = next(get_db())
+    try:
+        deleted = db.query(VideoTask).delete(synchronize_session=False)
+        db.commit()
+        logger.info(f"Cleared task index: {deleted} rows")
+        return deleted
+    except Exception as e:
+        logger.error(f"Failed to clear task index: {e}")
+        raise
+    finally:
+        db.close()
+
+
+# 按 task_id 删一条全局索引（清理任务时调用）
+def delete_task(task_id: str):
+    db = next(get_db())
+    try:
+        task = db.query(VideoTask).filter_by(task_id=task_id).first()
+        if task:
+            db.delete(task)
+            db.commit()
+            logger.info(f"Task deleted from index: {task_id}")
+    except Exception as e:
+        logger.error(f"Failed to delete task: {e}")
+    finally:
+        db.close()
+
+
+# 查询任务（最新一条）
+def get_task_by_video(video_id: str, platform: str):
+    db = next(get_db())
+    try:
+        task = (
+            db.query(VideoTask)
+            .filter_by(video_id=video_id, platform=platform)
+            .order_by(VideoTask.created_at.desc())
+            .first()
+        )
+        if task:
+            logger.info(f"Task found for video_id: {video_id} and platform: {platform}")
+            return task.task_id
+        else:
+            logger.info(f"No task found for video_id: {video_id} and platform: {platform}")
+            return None
+    except Exception as e:
+        logger.error(f"Failed to get task by video: {e}")
+    finally:
+        db.close()

app/decorators/timeit.py

@@ -0,0 +1,20 @@
+import functools
+import time
+
+from app.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def timeit(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        start = time.perf_counter()
+        result = func(*args, **kwargs)
+        end = time.perf_counter()
+        duration = end - start
+        # 度量走 logger 而非 print：print 在 MCP 下靠 stderr 重定向兜底，
+        # 裸脚本 / pytest 直跑会打进 stdout 污染输出（#127 B7）
+        logger.info(f"{func.__name__} executed in {duration:.4f} seconds")
+        return result
+    return wrapper