deepresearch-flow 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

deepresearch_flow/paper/db.py

@@ -424,6 +424,87 @@ def register_db_commands(db_group: click.Group) -> None:
         click.echo(f"Wrote snapshot DB: {opts.output_db}")
         click.echo(f"Wrote static export: {opts.static_export_dir}")
 
+    @snapshot_group.group("unpack")
+    def snapshot_unpack_group() -> None:
+        """Unpack snapshot artifacts."""
+
+    @snapshot_unpack_group.command("md")
+    @click.option("--snapshot-db", "snapshot_db", required=True, help="Path to snapshot database")
+    @click.option(
+        "--static-export-dir",
+        "static_export_dir",
+        required=True,
+        help="Path to static export directory",
+    )
+    @click.option(
+        "--pdf-root",
+        "pdf_roots",
+        multiple=True,
+        required=True,
+        help="PDF root directories for name alignment (repeatable)",
+    )
+    @click.option("--md-output-dir", "md_output_dir", required=True, help="Output directory for Markdown")
+    @click.option(
+        "--md-translated-output-dir",
+        "md_translated_output_dir",
+        required=True,
+        help="Output directory for translated Markdown",
+    )
+    def snapshot_unpack_md(
+        snapshot_db: str,
+        static_export_dir: str,
+        pdf_roots: tuple[str, ...],
+        md_output_dir: str,
+        md_translated_output_dir: str,
+    ) -> None:
+        """Unpack source/translated markdown and align filenames to PDFs."""
+        from deepresearch_flow.paper.snapshot.unpacker import SnapshotUnpackMdOptions, unpack_md
+
+        opts = SnapshotUnpackMdOptions(
+            snapshot_db=Path(snapshot_db),
+            static_export_dir=Path(static_export_dir),
+            pdf_roots=[Path(path) for path in pdf_roots],
+            md_output_dir=Path(md_output_dir),
+            md_translated_output_dir=Path(md_translated_output_dir),
+        )
+        unpack_md(opts)
+
+    @snapshot_unpack_group.command("info")
+    @click.option("--snapshot-db", "snapshot_db", required=True, help="Path to snapshot database")
+    @click.option(
+        "--static-export-dir",
+        "static_export_dir",
+        required=True,
+        help="Path to static export directory",
+    )
+    @click.option(
+        "--pdf-root",
+        "pdf_roots",
+        multiple=True,
+        required=True,
+        help="PDF root directories for name alignment (repeatable)",
+    )
+    @click.option("--template", "template", required=True, help="Summary template tag")
+    @click.option("--output-json", "output_json", required=True, help="Output JSON file path")
+    def snapshot_unpack_info(
+        snapshot_db: str,
+        static_export_dir: str,
+        pdf_roots: tuple[str, ...],
+        template: str,
+        output_json: str,
+    ) -> None:
+        """Unpack aggregated paper_infos.json from snapshot summaries."""
+        from deepresearch_flow.paper.snapshot.unpacker import SnapshotUnpackInfoOptions, unpack_info
+
+        opts = SnapshotUnpackInfoOptions(
+            snapshot_db=Path(snapshot_db),
+            static_export_dir=Path(static_export_dir),
+            pdf_roots=[Path(path) for path in pdf_roots],
+            template=template,
+            output_json=Path(output_json),
+        )
+        unpack_info(opts)
+
     @db_group.group("api")
     def api_group() -> None:
         """Read-only JSON API server backed by a snapshot DB."""

deepresearch_flow/paper/snapshot/api.py

@@ -11,8 +11,9 @@ from starlette.applications import Starlette
 from starlette.middleware.cors import CORSMiddleware
 from starlette.requests import Request
 from starlette.responses import JSONResponse, Response
-from starlette.routing import Route
+from starlette.routing import Mount, Route
 
+from deepresearch_flow.paper.snapshot.common import ApiLimits, _open_ro_conn
 from deepresearch_flow.paper.snapshot.text import merge_adjacent_markers, remove_cjk_spaces, rewrite_search_query
 
 _WHITESPACE_RE = re.compile(r"\s+")
@@ -87,13 +88,6 @@ _FACET_TYPE_TO_KEY = {
 }
 
 
-@dataclass(frozen=True)
-class ApiLimits:
-    max_query_length: int = 500
-    max_page_size: int = 100
-    max_pagination_offset: int = 10_000  # page * page_size
-
-
 @dataclass(frozen=True)
 class SnapshotApiConfig:
     snapshot_db: Path
@@ -110,12 +104,6 @@ def _json_error(status_code: int, *, error: str, detail: str) -> JSONResponse:
     return JSONResponse({"error": error, "detail": detail}, status_code=status_code)
 
 
-def _open_ro_conn(db_path: Path) -> sqlite3.Connection:
-    uri = f"file:{db_path.as_posix()}?mode=ro"
-    conn = sqlite3.connect(uri, uri=True)
-    conn.row_factory = sqlite3.Row
-    conn.execute("PRAGMA query_only=ON;")
-    return conn
 
 
 def _snapshot_build_id(conn: sqlite3.Connection) -> str:
@@ -917,6 +905,22 @@ def create_app(
         limits=limits or ApiLimits(),
     )
 
+    # Lazy import to avoid circular dependency
+    from deepresearch_flow.paper.snapshot.mcp_server import (
+        McpSnapshotConfig,
+        create_mcp_app,
+        resolve_static_export_dir,
+    )
+
+    mcp_config = McpSnapshotConfig(
+        snapshot_db=snapshot_db,
+        static_base_url=_normalize_base_url(static_base_url),
+        static_export_dir=resolve_static_export_dir(),
+        limits=limits or ApiLimits(),
+        origin_allowlist=cors_allowed_origins or ["*"],
+    )
+    mcp_app, mcp_lifespan = create_mcp_app(mcp_config)
+
     routes = [
         Route("/api/v1/config", _api_config, methods=["GET"]),
         Route("/api/v1/search", _api_search, methods=["GET"]),
@@ -927,9 +931,15 @@ def create_app(
         Route("/api/v1/facets/{facet:str}/{facet_id:str}/stats", _api_facet_stats, methods=["GET"]),
         Route("/api/v1/facets/{facet:str}/by-value/{value:str}/papers", _api_facet_by_value_papers, methods=["GET"]),
         Route("/api/v1/facets/{facet:str}/by-value/{value:str}/stats", _api_facet_by_value_stats, methods=["GET"]),
+        Mount("/mcp", app=mcp_app),
     ]
 
-    app = Starlette(routes=routes)
+    # Pass MCP lifespan to ensure session manager initializes properly
+    # https://gofastmcp.com/deployment/http#mounting-in-starlette
+    app = Starlette(
+        routes=routes,
+        lifespan=mcp_lifespan,
+    )
     if cfg.cors_allowed_origins:
         app.add_middleware(
             CORSMiddleware,

deepresearch_flow/paper/snapshot/common.py

@@ -0,0 +1,34 @@
+"""Shared utilities for snapshot API and MCP server.
+
+This module contains common types, configuration, and utilities used by both
+the snapshot REST API and the MCP server to avoid circular imports.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+import sqlite3
+
+
+@dataclass(frozen=True)
+class ApiLimits:
+    """API rate and size limits."""
+
+    max_query_length: int = 500
+    max_page_size: int = 100
+    max_pagination_offset: int = 10_000
+
+
+def _open_ro_conn(db_path: Path) -> sqlite3.Connection:
+    """Open a read-only SQLite connection with Row factory.
+
+    Args:
+        db_path: Path to the SQLite database file.
+
+    Returns:
+        sqlite3.Connection: A read-only connection with row_factory set to Row.
+    """
+    conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, check_same_thread=False)
+    conn.row_factory = sqlite3.Row
+    return conn

deepresearch_flow/paper/snapshot/mcp_server.py

@@ -0,0 +1,686 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import json
+import os
+from pathlib import Path
+import re
+from typing import Any
+
+import httpx
+from starlette.applications import Starlette
+from starlette.middleware import Middleware
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.routing import Mount
+
+from fastmcp import FastMCP
+
+from deepresearch_flow.paper.snapshot.common import ApiLimits, _open_ro_conn
+from deepresearch_flow.paper.snapshot.text import merge_adjacent_markers, remove_cjk_spaces, rewrite_search_query
+
+_SUPPORTED_PROTOCOL_VERSIONS = {"2025-03-26", "2025-06-18"}
+_DEFAULT_MAX_CHARS = 50_000
+_DEFAULT_TIMEOUT = 10.0
+_PAPER_ID_PATTERN = re.compile(r'^[a-zA-Z0-9_-]+$')
+
+
+class McpToolError(Exception):
+    """MCP tool exception for standardized error handling.
+    
+    FastMCP will catch this exception and convert it to a proper
+    JSON-RPC error response that the client can understand.
+    """
+    
+    def __init__(self, code: str, message: str, **details):
+        self.code = code
+        self.message = message
+        self.details = details
+        super().__init__(message)
+    
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to error dictionary format."""
+        return {"error": self.code, "message": self.message, **self.details}
+
+
+@dataclass(frozen=True)
+class McpSnapshotConfig:
+    snapshot_db: Path
+    static_base_url: str
+    static_export_dir: Path | None
+    limits: ApiLimits
+    origin_allowlist: list[str]
+    max_chars_default: int = _DEFAULT_MAX_CHARS
+    http_timeout: float = _DEFAULT_TIMEOUT
+    max_paper_id_length: int = 64
+    # HTTP client stored in object __dict__ to avoid dataclass frozen restriction
+    _http_client: httpx.Client | None = field(default=None, repr=False, compare=False)
+    
+    def get_http_client(self) -> httpx.Client:
+        """Get or create a shared HTTP client with connection pooling."""
+        if self._http_client is None:
+            object.__setattr__(
+                self,
+                '_http_client',
+                httpx.Client(
+                    timeout=self.http_timeout,
+                    follow_redirects=True,
+                    limits=httpx.Limits(
+                        max_keepalive_connections=10,
+                        max_connections=20
+                    )
+                )
+            )
+        return self._http_client
+
+
+class McpRequestGuardMiddleware(BaseHTTPMiddleware):
+    def __init__(self, app, *, origin_allowlist: list[str]) -> None:
+        super().__init__(app)
+        self._allowlist = [origin.lower() for origin in origin_allowlist]
+
+    async def dispatch(self, request: Request, call_next):  # type: ignore[override]
+        if request.method == "GET":
+            return Response("Method Not Allowed", status_code=405)
+        if request.method not in {"POST", "OPTIONS"}:
+            return Response("Method Not Allowed", status_code=405)
+        origin = request.headers.get("origin")
+        if origin and not self._is_allowed_origin(origin):
+            return Response("Forbidden", status_code=403)
+        protocol = request.headers.get("mcp-protocol-version")
+        if protocol and protocol not in _SUPPORTED_PROTOCOL_VERSIONS:
+            return Response("Bad Request", status_code=400)
+        return await call_next(request)
+
+    def _is_allowed_origin(self, origin: str) -> bool:
+        if not self._allowlist or "*" in self._allowlist:
+            return True
+        return origin.lower() in self._allowlist
+
+
+_CONFIG: McpSnapshotConfig | None = None
+mcp = FastMCP("Paper DB MCP")
+
+
+def configure(config: McpSnapshotConfig) -> None:
+    global _CONFIG
+    _CONFIG = config
+
+
+def create_mcp_app(config: McpSnapshotConfig) -> tuple[Starlette, Any]:
+    """Create MCP app with middleware and return it along with lifespan.
+    
+    Returns:
+        Tuple of (wrapped_app, lifespan_context) for use by parent Starlette.
+    """
+    configure(config)
+    mcp_app = mcp.http_app(path="/", stateless_http=True)
+    wrapped = Starlette(
+        routes=[Mount("/", app=mcp_app)],
+        middleware=[
+            Middleware(McpRequestGuardMiddleware, origin_allowlist=config.origin_allowlist),
+        ],
+    )
+    return wrapped, mcp_app.lifespan
+
+
+def _get_config() -> McpSnapshotConfig:
+    if _CONFIG is None:
+        raise RuntimeError("MCP server not configured")
+    return _CONFIG
+
+
+def _validate_query(query: str, cfg: McpSnapshotConfig) -> str:
+    """Validate search query string.
+    
+    Raises:
+        McpToolError: If query is invalid or too long.
+    """
+    if not query or not query.strip():
+        raise McpToolError("invalid_query", "Query cannot be empty")
+    if len(query) > cfg.limits.max_query_length:
+        raise McpToolError(
+            "query_too_long",
+            f"Query exceeds maximum length of {cfg.limits.max_query_length}",
+            length=len(query),
+            max_length=cfg.limits.max_query_length
+        )
+    return query.strip()
+
+
+def _validate_paper_id(paper_id: str, cfg: McpSnapshotConfig) -> str:
+    """Validate paper ID format.
+    
+    Raises:
+        McpToolError: If paper_id is invalid.
+    """
+    if not paper_id:
+        raise McpToolError("invalid_paper_id", "Paper ID cannot be empty")
+    if len(paper_id) > cfg.max_paper_id_length:
+        raise McpToolError(
+            "paper_id_too_long",
+            f"Paper ID exceeds maximum length of {cfg.max_paper_id_length}",
+            length=len(paper_id),
+            max_length=cfg.max_paper_id_length
+        )
+    if not _PAPER_ID_PATTERN.match(paper_id):
+        raise McpToolError(
+            "invalid_paper_id_format",
+            "Paper ID must contain only alphanumeric characters, hyphens, and underscores",
+            paper_id=paper_id
+        )
+    return paper_id
+
+
+def _truncate(text: str, max_chars: int | None) -> str:
+    """Truncate text with marker."""
+    if max_chars is None or max_chars <= 0:
+        return text
+    if len(text) <= max_chars:
+        return text
+    remaining = len(text) - max_chars
+    return f"{text[:max_chars]}\n[truncated: {remaining} more chars]"
+
+
+def _read_static_text(rel_path: str) -> str | None:
+    """Read static text from local export directory if available."""
+    cfg = _get_config()
+    if cfg.static_export_dir:
+        path = cfg.static_export_dir / rel_path
+        if path.exists():
+            return path.read_text(encoding="utf-8")
+    return None
+
+
+def _fetch_static_text(rel_path: str) -> str:
+    """Fetch static text from HTTP remote."""
+    cfg = _get_config()
+    if cfg.static_base_url:
+        base = cfg.static_base_url.rstrip("/")
+        url = f"{base}/{rel_path.lstrip('/')}"
+        client = cfg.get_http_client()
+        response = client.get(url)
+        response.raise_for_status()
+        return response.text
+    raise FileNotFoundError("static_base_url not configured")
+
+
+def _load_static_text(rel_path: str) -> str:
+    """Load static text with fallback: local first, then HTTP."""
+    try:
+        text = _read_static_text(rel_path)
+        if text is not None:
+            return text
+        return _fetch_static_text(rel_path)
+    except httpx.HTTPStatusError as exc:
+        raise RuntimeError(f"asset_fetch_failed:{exc.response.status_code}") from exc
+    except httpx.RequestError as exc:
+        raise RuntimeError("asset_fetch_failed:request_error") from exc
+    except FileNotFoundError as exc:
+        raise RuntimeError("asset_fetch_failed:not_configured") from exc
+
+
+def _load_summary_json(paper_id: str, template: str | None) -> tuple[str | None, list[str] | None]:
+    """Load summary JSON content and return available templates list."""
+    cfg = _get_config()
+    conn = _open_ro_conn(cfg.snapshot_db)
+    try:
+        row = conn.execute(
+            "SELECT preferred_summary_template, summary_asset_paths_json FROM paper WHERE paper_id = ?",
+            (paper_id,),
+        ).fetchone()
+        if not row:
+            return None, None
+        preferred = row["preferred_summary_template"]
+        asset_paths = json.loads(row["summary_asset_paths_json"] or "{}")
+        available = sorted(asset_paths.keys())
+        selected = template if template else preferred
+        if not selected or selected not in asset_paths:
+            return None, available
+        rel_path = asset_paths[selected]
+        return _load_static_text(rel_path), available
+    finally:
+        conn.close()
+
+
+def _load_source_markdown(paper_id: str) -> str | None:
+    """Load source markdown for paper."""
+    cfg = _get_config()
+    conn = _open_ro_conn(cfg.snapshot_db)
+    try:
+        row = conn.execute(
+            "SELECT source_md_content_hash FROM paper WHERE paper_id = ?",
+            (paper_id,),
+        ).fetchone()
+        if not row or not row["source_md_content_hash"]:
+            return None
+        rel_path = f"md/{row['source_md_content_hash']}.md"
+        return _load_static_text(rel_path)
+    finally:
+        conn.close()
+
+
+def _load_translation_markdown(paper_id: str, lang: str) -> str | None:
+    """Load translation markdown for paper and language."""
+    cfg = _get_config()
+    conn = _open_ro_conn(cfg.snapshot_db)
+    try:
+        row = conn.execute(
+            "SELECT translations_json FROM paper WHERE paper_id = ?",
+            (paper_id,),
+        ).fetchone()
+        if not row or not row["translations_json"]:
+            return None
+        translations = json.loads(row["translations_json"])
+        rel_path = translations.get(lang)
+        if not rel_path:
+            return None
+        return _load_static_text(rel_path)
+    finally:
+        conn.close()
+
+
+# ==================== MCP Tools ====================
+
+@mcp.tool()
+def search_papers(query: str, limit: int = 10) -> list[dict[str, Any]]:
+    """Full-text search for papers (relevance-ranked).
+    
+    Use when you only have topic keywords.
+    Returns paper_id, title, year, venue, snippet_markdown.
+    """
+    cfg = _get_config()
+    query = _validate_query(query, cfg)
+    limit = min(max(1, int(limit)), cfg.limits.max_page_size)
+    
+    conn = _open_ro_conn(cfg.snapshot_db)
+    try:
+        cur = conn.execute(
+            """
+            SELECT paper_id, title, year, venue, abstract
+            FROM paper_search
+            WHERE paper_search MATCH ?
+            ORDER BY rank
+            LIMIT ?
+            """,
+            (rewrite_search_query(query), limit),
+        )
+        rows = cur.fetchall()
+        results: list[dict[str, Any]] = []
+        for row in rows:
+            snippet = str(row["abstract"] or "")
+            snippet = remove_cjk_spaces(snippet)
+            snippet, markers = merge_adjacent_markers(snippet)
+            results.append({
+                "paper_id": str(row["paper_id"]),
+                "title": str(row["title"]),
+                "year": str(row["year"]),
+                "venue": str(row["venue"]),
+                "snippet_markdown": snippet,
+            })
+        return results
+    finally:
+        conn.close()
+
+
+@mcp.tool()
+def search_papers_by_keyword(keyword: str, limit: int = 10) -> list[dict[str, Any]]:
+    """Search papers by keyword/tag (exact match).
+    
+    Use when you know specific keywords or tags.
+    """
+    cfg = _get_config()
+    limit = min(max(1, int(limit)), cfg.limits.max_page_size)
+    
+    conn = _open_ro_conn(cfg.snapshot_db)
+    try:
+        rows = conn.execute(
+            """
+            SELECT DISTINCT p.paper_id, p.title, p.year, p.venue, p.abstract
+            FROM paper p
+            JOIN paper_keyword pk ON pk.paper_id = p.paper_id
+            JOIN keyword k ON k.keyword_id = pk.keyword_id
+            WHERE k.value LIKE ?
+            ORDER BY p.year DESC, p.title ASC
+            LIMIT ?
+            """,
+            (f"%{keyword}%", limit),
+        ).fetchall()
+        results: list[dict[str, Any]] = []
+        for row in rows:
+            snippet = str(row["abstract"] or "")
+            snippet = remove_cjk_spaces(snippet)
+            snippet, markers = merge_adjacent_markers(snippet)
+            results.append({
+                "paper_id": str(row["paper_id"]),
+                "title": str(row["title"]),
+                "year": str(row["year"]),
+                "venue": str(row["venue"]),
+                "snippet_markdown": snippet,
+            })
+        return results
+    finally:
+        conn.close()
+
+
+@mcp.tool()
+def get_paper_metadata(paper_id: str) -> dict[str, Any]:
+    """Get paper metadata and available summary templates.
+    
+    Call this first before requesting a summary to discover available templates.
+    """
+    cfg = _get_config()
+    paper_id = _validate_paper_id(paper_id, cfg)
+    
+    conn = _open_ro_conn(cfg.snapshot_db)
+    try:
+        row = conn.execute(
+            """
+            SELECT paper_id, title, year, venue, doi, arxiv_id, openreview_id, paper_pw_url,
+                   preferred_summary_template, summary_asset_paths_json
+            FROM paper WHERE paper_id = ?
+            """,
+            (paper_id,),
+        ).fetchone()
+        if not row:
+            raise McpToolError("not_found", "paper not found", paper_id=paper_id)
+        
+        asset_paths = json.loads(row["summary_asset_paths_json"] or "{}")
+        available = sorted(asset_paths.keys())
+        return {
+            "paper_id": str(row["paper_id"]),
+            "title": str(row["title"]),
+            "year": str(row["year"]),
+            "venue": str(row["venue"]),
+            "doi": row["doi"],
+            "arxiv_id": row["arxiv_id"],
+            "openreview_id": row["openreview_id"],
+            "paper_pw_url": row["paper_pw_url"],
+            "preferred_summary_template": row["preferred_summary_template"],
+            "available_summary_templates": available,
+        }
+    finally:
+        conn.close()
+
+
+@mcp.tool()
+def get_paper_summary(paper_id: str, template: str | None = None, max_chars: int | None = None) -> str:
+    """Get summary JSON as raw string.
+    
+    Uses preferred template if template is not specified.
+    Returns the full JSON content (not a URL).
+    """
+    cfg = _get_config()
+    paper_id = _validate_paper_id(paper_id, cfg)
+    max_chars = max_chars if max_chars is not None else cfg.max_chars_default
+    
+    try:
+        payload, available = _load_summary_json(paper_id, template)
+    except RuntimeError as exc:
+        raise McpToolError(
+            "asset_fetch_failed",
+            "Failed to fetch summary asset",
+            paper_id=paper_id,
+            template=template,
+            detail=str(exc),
+        ) from exc
+    
+    if payload is None:
+        raise McpToolError(
+            "template_not_available",
+            "Template not available",
+            paper_id=paper_id,
+            template=template,
+            available_summary_templates=available,
+        )
+    
+    return _truncate(payload, max_chars)
+
+
+@mcp.tool()
+def get_paper_source(paper_id: str, max_chars: int | None = None) -> str:
+    """Get source markdown text.
+    
+    Content may be large; use max_chars to limit size.
+    """
+    cfg = _get_config()
+    paper_id = _validate_paper_id(paper_id, cfg)
+    max_chars = max_chars if max_chars is not None else cfg.max_chars_default
+    
+    try:
+        content = _load_source_markdown(paper_id)
+    except RuntimeError as exc:
+        raise McpToolError(
+            "asset_fetch_failed",
+            "Failed to fetch source asset",
+            paper_id=paper_id,
+            detail=str(exc),
+        ) from exc
+    
+    if content is None:
+        raise McpToolError(
+            "source_not_available",
+            "Source markdown not available",
+            paper_id=paper_id
+        )
+    
+    return _truncate(content, max_chars)
+
+
+@mcp.tool()
+def get_database_stats() -> dict[str, Any]:
+    """Get database statistics.
+    
+    Returns totals, year/month distributions, and top facets
+    (authors, venues, keywords, institutions, tags).
+    """
+    cfg = _get_config()
+    conn = _open_ro_conn(cfg.snapshot_db)
+    try:
+        total_row = conn.execute("SELECT COUNT(*) AS c FROM paper").fetchone()
+        total = int(total_row["c"]) if total_row else 0
+        
+        def top(table: str, limit: int = 20) -> list[dict[str, Any]]:
+            rows = conn.execute(
+                f"SELECT value, paper_count FROM {table} ORDER BY paper_count DESC, value ASC LIMIT ?",
+                (limit,),
+            ).fetchall()
+            return [{"value": str(r["value"]), "paper_count": int(r["paper_count"])} for r in rows]
+        
+        years = conn.execute(
+            """
+            SELECT year AS value, paper_count
+            FROM year_count
+            ORDER BY CASE WHEN year GLOB '[0-9][0-9][0-9][0-9]' THEN 0 ELSE 1 END,
+                     CAST(year AS INT) DESC, year ASC
+            LIMIT 50
+            """,
+        ).fetchall()
+        months = conn.execute(
+            """
+            SELECT month AS value, paper_count
+            FROM month_count
+            ORDER BY CASE WHEN month GLOB '[0-1][0-9]' THEN 0 ELSE 1 END,
+                     CAST(month AS INT) ASC, month ASC
+            """,
+        ).fetchall()
+        
+        return {
+            "total": total,
+            "years": [{"value": str(r["value"]), "paper_count": int(r["paper_count"])} for r in years],
+            "months": [{"value": str(r["value"]), "paper_count": int(r["paper_count"])} for r in months],
+            "authors": top("author"),
+            "venues": top("venue"),
+            "institutions": top("institution"),
+            "keywords": top("keyword"),
+            "tags": top("tag"),
+        }
+    finally:
+        conn.close()
+
+
+@mcp.tool()
+def list_top_facets(category: str, limit: int = 20) -> list[dict[str, Any]]:
+    """List top facet values.
+    
+    Category: author | venue | keyword | institution | tag
+    """
+    table_map = {
+        "author": "author",
+        "venue": "venue",
+        "keyword": "keyword",
+        "institution": "institution",
+        "tag": "tag",
+    }
+    table = table_map.get((category or "").strip().lower())
+    if not table:
+        raise McpToolError(
+            "invalid_category",
+            f"Invalid category: {category}. Must be one of: {', '.join(table_map.keys())}",
+            category=category
+        )
+    
+    limit = max(1, int(limit))
+    cfg = _get_config()
+    conn = _open_ro_conn(cfg.snapshot_db)
+    try:
+        rows = conn.execute(
+            f"SELECT value, paper_count FROM {table} ORDER BY paper_count DESC, value ASC LIMIT ?",
+            (limit,),
+        ).fetchall()
+        return [{"value": str(r["value"]), "paper_count": int(r["paper_count"])} for r in rows]
+    finally:
+        conn.close()
+
+
+@mcp.tool()
+def filter_papers(
+    author: str | None = None,
+    venue: str | None = None,
+    year: str | None = None,
+    keyword: str | None = None,
+    tag: str | None = None,
+    limit: int = 10,
+) -> list[dict[str, Any]]:
+    """Filter papers by structured fields.
+    
+    Use for precise filtering by author, venue, year, keyword, or tag.
+    """
+    cfg = _get_config()
+    limit = min(max(1, int(limit)), cfg.limits.max_page_size)
+    
+    query = "SELECT DISTINCT p.paper_id, p.title, p.year, p.venue FROM paper p"
+    joins: list[str] = []
+    conditions: list[str] = []
+    params: list[Any] = []
+    
+    if author:
+        joins.append("JOIN paper_author pa ON pa.paper_id = p.paper_id")
+        joins.append("JOIN author a ON a.author_id = pa.author_id")
+        conditions.append("a.value LIKE ?")
+        params.append(f"%{author}%")
+    if keyword:
+        joins.append("JOIN paper_keyword pk ON pk.paper_id = p.paper_id")
+        joins.append("JOIN keyword k ON k.keyword_id = pk.keyword_id")
+        conditions.append("k.value LIKE ?")
+        params.append(f"%{keyword}%")
+    if tag:
+        joins.append("JOIN paper_tag pt ON pt.paper_id = p.paper_id")
+        joins.append("JOIN tag t ON t.tag_id = pt.tag_id")
+        conditions.append("t.value LIKE ?")
+        params.append(f"%{tag}%")
+    if venue:
+        conditions.append("p.venue LIKE ?")
+        params.append(f"%{venue}%")
+    if year:
+        conditions.append("p.year = ?")
+        params.append(str(year))
+    
+    if joins:
+        query += " " + " ".join(joins)
+    if conditions:
+        query += " WHERE " + " AND ".join(conditions)
+    query += " ORDER BY p.year DESC, p.title ASC LIMIT ?"
+    params.append(limit)
+    
+    conn = _open_ro_conn(cfg.snapshot_db)
+    try:
+        rows = conn.execute(query, tuple(params)).fetchall()
+        return [
+            {
+                "paper_id": str(row["paper_id"]),
+                "title": str(row["title"]),
+                "year": str(row["year"]),
+                "venue": str(row["venue"]),
+            }
+            for row in rows
+        ]
+    finally:
+        conn.close()
+
+
+# ==================== MCP Resources ====================
+
+@mcp.resource("paper://{paper_id}/metadata")
+def resource_metadata(paper_id: str) -> str:
+    """Resource: metadata as JSON string."""
+    payload = get_paper_metadata(paper_id)
+    return json.dumps(payload, ensure_ascii=False)
+
+
+@mcp.resource("paper://{paper_id}/summary")
+def resource_summary_default(paper_id: str) -> str:
+    """Resource: preferred summary JSON string."""
+    payload = get_paper_summary(paper_id)
+    return payload  # Already a JSON string
+
+
+@mcp.resource("paper://{paper_id}/summary/{template}")
+def resource_summary_template(paper_id: str, template: str) -> str:
+    """Resource: summary JSON string for a specific template."""
+    payload = get_paper_summary(paper_id, template=template)
+    return payload  # Already a JSON string
+
+
+@mcp.resource("paper://{paper_id}/source")
+def resource_source(paper_id: str) -> str:
+    """Resource: source markdown text."""
+    payload = get_paper_source(paper_id)
+    return payload
+
+
+@mcp.resource("paper://{paper_id}/translation/{lang}")
+def resource_translation(paper_id: str, lang: str) -> str:
+    """Resource: translated markdown text."""
+    cfg = _get_config()
+    paper_id = _validate_paper_id(paper_id, cfg)
+    
+    try:
+        content = _load_translation_markdown(paper_id, lang.lower())
+    except RuntimeError as exc:
+        raise McpToolError(
+            "asset_fetch_failed",
+            "Failed to fetch translation asset",
+            paper_id=paper_id,
+            lang=lang,
+            detail=str(exc),
+        ) from exc
+    
+    if content is None:
+        raise McpToolError(
+            "translation_not_available",
+            "Translation not available",
+            paper_id=paper_id,
+            lang=lang,
+        )
+    
+    return _truncate(content, cfg.max_chars_default)
+
+
+def resolve_static_export_dir() -> Path | None:
+    """Resolve static export directory from environment variable."""
+    value = os.getenv("PAPER_DB_STATIC_EXPORT_DIR")
+    if not value:
+        return None
+    return Path(value)

deepresearch_flow/paper/snapshot/unpacker.py

@@ -0,0 +1,259 @@
+"""Unpack snapshot to recover original files with readable names.
+
+This is the reverse operation of builder.build_snapshot().
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import hashlib
+import json
+from pathlib import Path
+import re
+import sqlite3
+from typing import Any, Iterable
+
+from rich.console import Console
+from rich.table import Table
+
+
+@dataclass(frozen=True)
+class SnapshotUnpackBaseOptions:
+    snapshot_db: Path
+    static_export_dir: Path
+    pdf_roots: list[Path]
+
+
+@dataclass(frozen=True)
+class SnapshotUnpackMdOptions(SnapshotUnpackBaseOptions):
+    md_output_dir: Path
+    md_translated_output_dir: Path
+
+
+@dataclass(frozen=True)
+class SnapshotUnpackInfoOptions(SnapshotUnpackBaseOptions):
+    template: str
+    output_json: Path
+
+
+@dataclass
+class UnpackCounts:
+    total: int = 0
+    succeeded: int = 0
+    failed: int = 0
+    missing_pdf: int = 0
+    translated_succeeded: int = 0
+    translated_failed: int = 0
+
+
+def _sanitize_filename(title: str) -> str:
+    """Convert title to safe filename."""
+    sanitized = re.sub(r'[<>:"/\\|?*]', "_", title)
+    if len(sanitized) > 200:
+        sanitized = sanitized[:200]
+    sanitized = sanitized.strip()
+    if not sanitized:
+        sanitized = "untitled"
+    return sanitized
+
+
+def _hash_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _build_pdf_hash_index(pdf_roots: Iterable[Path]) -> dict[str, Path]:
+    index: dict[str, Path] = {}
+    for root in pdf_roots:
+        if root.is_file() and root.suffix.lower() == ".pdf":
+            pdf_hash = _hash_file(root)
+            index.setdefault(pdf_hash, root)
+            continue
+        if not root.is_dir():
+            continue
+        for path in root.rglob("*.pdf"):
+            if not path.is_file():
+                continue
+            pdf_hash = _hash_file(path)
+            index.setdefault(pdf_hash, path)
+    return index
+
+
+def _unique_base_name(base: str, paper_id: str, used: set[str]) -> str:
+    candidate = base
+    if candidate in used:
+        candidate = f"{base}_{paper_id}"
+    counter = 1
+    while candidate in used:
+        candidate = f"{base}_{paper_id}_{counter}"
+        counter += 1
+    used.add(candidate)
+    return candidate
+
+
+def _open_snapshot_db(path: Path) -> sqlite3.Connection:
+    conn = sqlite3.connect(path)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def _print_summary(title: str, counts: UnpackCounts) -> None:
+    table = Table(title=title, header_style="bold cyan", title_style="bold magenta")
+    table.add_column("Metric", style="cyan", no_wrap=True)
+    table.add_column("Value", style="white", overflow="fold")
+    table.add_row("Total", str(counts.total))
+    table.add_row("Succeeded", str(counts.succeeded))
+    table.add_row("Failed", str(counts.failed))
+    table.add_row("Missing PDF", str(counts.missing_pdf))
+    if counts.translated_succeeded or counts.translated_failed:
+        table.add_row("Translated succeeded", str(counts.translated_succeeded))
+        table.add_row("Translated failed", str(counts.translated_failed))
+    Console().print(table)
+
+
+def unpack_md(opts: SnapshotUnpackMdOptions) -> None:
+    """Unpack source/translated markdown and align filenames to PDFs."""
+    opts.md_output_dir.mkdir(parents=True, exist_ok=True)
+    opts.md_translated_output_dir.mkdir(parents=True, exist_ok=True)
+
+    pdf_index = _build_pdf_hash_index(opts.pdf_roots)
+    used_names: set[str] = set()
+    counts = UnpackCounts()
+
+    conn = _open_snapshot_db(opts.snapshot_db)
+    try:
+        cursor = conn.execute(
+            """
+            SELECT
+                paper_id,
+                title,
+                source_hash,
+                pdf_content_hash,
+                source_md_content_hash
+            FROM paper
+            ORDER BY paper_index, title
+            """
+        )
+        for row in cursor.fetchall():
+            counts.total += 1
+            paper_id = str(row["paper_id"])
+            title = str(row["title"] or "")
+            pdf_hash = row["pdf_content_hash"]
+            md_hash = row["source_md_content_hash"]
+
+            base = ""
+            if pdf_hash and pdf_hash in pdf_index:
+                base = pdf_index[pdf_hash].stem
+            else:
+                counts.missing_pdf += 1
+                base = _sanitize_filename(title)
+            base = _unique_base_name(base, paper_id, used_names)
+
+            if md_hash:
+                src_md = opts.static_export_dir / "md" / f"{md_hash}.md"
+                if src_md.exists():
+                    dst_md = opts.md_output_dir / f"{base}.md"
+                    try:
+                        dst_md.write_text(src_md.read_text(encoding="utf-8"), encoding="utf-8")
+                        counts.succeeded += 1
+                    except OSError:
+                        counts.failed += 1
+                else:
+                    counts.failed += 1
+            else:
+                counts.failed += 1
+
+
+            for tr_row in conn.execute(
+                "SELECT lang, md_content_hash FROM paper_translation WHERE paper_id = ?",
+                (paper_id,),
+            ):
+                lang = str(tr_row["lang"] or "").lower()
+                tr_hash = tr_row["md_content_hash"]
+                if not lang or not tr_hash:
+                    counts.translated_failed += 1
+                    continue
+                src_tr = opts.static_export_dir / "md_translate" / lang / f"{tr_hash}.md"
+                if not src_tr.exists():
+                    counts.translated_failed += 1
+                    continue
+                dst_tr = opts.md_translated_output_dir / f"{base}.{lang}.md"
+                try:
+                    dst_tr.write_text(src_tr.read_text(encoding="utf-8"), encoding="utf-8")
+                    counts.translated_succeeded += 1
+                except OSError:
+                    counts.translated_failed += 1
+    finally:
+        conn.close()
+
+    _print_summary("snapshot unpack md summary", counts)
+
+
+def unpack_info(opts: SnapshotUnpackInfoOptions) -> None:
+    """Unpack aggregated paper_infos.json from snapshot summaries."""
+    pdf_index = _build_pdf_hash_index(opts.pdf_roots)
+    counts = UnpackCounts()
+    items: list[dict[str, Any]] = []
+
+    conn = _open_snapshot_db(opts.snapshot_db)
+    try:
+        cursor = conn.execute(
+            """
+            SELECT
+                paper_id,
+                title,
+                source_hash,
+                pdf_content_hash
+            FROM paper
+            ORDER BY paper_index, title
+            """
+        )
+        for row in cursor.fetchall():
+            counts.total += 1
+            paper_id = str(row["paper_id"])
+            pdf_hash = row["pdf_content_hash"]
+            if not (pdf_hash and pdf_hash in pdf_index):
+                counts.missing_pdf += 1
+
+            summary_path = opts.static_export_dir / "summary" / paper_id / f"{opts.template}.json"
+            fallback_path = opts.static_export_dir / "summary" / f"{paper_id}.json"
+            target_path = summary_path if summary_path.exists() else fallback_path
+            used_fallback = target_path == fallback_path
+            if not target_path.exists():
+                counts.failed += 1
+                continue
+            try:
+                payload = json.loads(target_path.read_text(encoding="utf-8"))
+            except json.JSONDecodeError:
+                counts.failed += 1
+                continue
+            if not isinstance(payload, dict):
+                counts.failed += 1
+                continue
+
+            base = ""
+            if pdf_hash and pdf_hash in pdf_index:
+                base = pdf_index[pdf_hash].stem
+            else:
+                base = _sanitize_filename(str(row["title"] or ""))
+            source_path = f"{base}.md" if base else ""
+
+            payload["paper_id"] = paper_id
+            payload["paper_title"] = str(row["title"] or "")
+            payload["source_path"] = source_path
+            payload["source_hash"] = str(row["source_hash"] or "")
+
+            if used_fallback:
+                counts.failed += 1
+            else:
+                counts.succeeded += 1
+            items.append(payload)
+    finally:
+        conn.close()
+
+    opts.output_json.parent.mkdir(parents=True, exist_ok=True)
+    opts.output_json.write_text(json.dumps(items, ensure_ascii=False, indent=2), encoding="utf-8")
+    _print_summary("snapshot unpack info summary", counts)

{deepresearch_flow-0.6.1.dist-info → deepresearch_flow-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepresearch-flow
-Version: 0.6.1
+Version: 0.7.0
 Summary: Workflow tools for paper extraction, review, and research automation.
 Author-email: DengQi <dengqi935@gmail.com>
 License: MIT License
@@ -49,6 +49,7 @@ Requires-Dist: jinja2>=3.1.3
 Requires-Dist: json-repair>=0.55.1
 Requires-Dist: jsonschema>=4.26.0
 Requires-Dist: markdown-it-py>=3.0.0
+Requires-Dist: fastmcp>=3.0.0b1
 Requires-Dist: mdit-py-plugins>=0.4.0
 Requires-Dist: pypdf>=6.6.2
 Requires-Dist: pylatexenc>=2.10
@@ -56,7 +57,7 @@ Requires-Dist: pybtex>=0.24.0
 Requires-Dist: rich>=14.3.1
 Requires-Dist: rumdl>=0.1.6
 Requires-Dist: starlette>=0.52.1
-Requires-Dist: tqdm>=4.66.4
+Requires-Dist: tqdm>=4.67.2
 Requires-Dist: uvicorn>=0.27.1
 Dynamic: license-file
 

{deepresearch_flow-0.6.1.dist-info → deepresearch_flow-0.7.0.dist-info}/RECORD

@@ -4,7 +4,7 @@ deepresearch_flow/cli.py,sha256=t4oowCNWldL0DrVJ4d0UlRkuGU2qHej_G0mAc_quteQ,455
 deepresearch_flow/paper/__init__.py,sha256=sunaOkcgAJBrfmcaJTumcWbPGVUSGWvOv2a2Yidzy0A,43
 deepresearch_flow/paper/cli.py,sha256=68d-yccScU0yL6d7eqZVdudPO6i_in8F4v-hKDWILMo,13647
 deepresearch_flow/paper/config.py,sha256=V7z4ApPXCV1acSl2FU3nZGq6nt8uisMhm0GtOq5zzmg,12021
-deepresearch_flow/paper/db.py,sha256=Bxhrd8NCaPZ9Ijtp1uiOplwh2Uy0n2Dyn1bO0d4A9bE,91780
+deepresearch_flow/paper/db.py,sha256=RvUN9jeoaEgLNvf8NhWYD-cgIIMZwdZRK3cq17pNWZI,94727
 deepresearch_flow/paper/db_ops.py,sha256=cb64jn2ax39i3tCS-0DYmlsJdGX3uBS2u5ncUIbUBic,73980
 deepresearch_flow/paper/extract.py,sha256=78ASAyNLfCl1AsAk2o_v1vskZCNZuayaCHgr0S4V2Vs,87632
 deepresearch_flow/paper/llm.py,sha256=mHfs5IkT3Q6BOh46MDlfUmgVTX24WRf0IKKoOnN8nV8,4007
@@ -43,11 +43,14 @@ deepresearch_flow/paper/schemas/default_paper_schema.json,sha256=6h_2ayHolJj8JMn
 deepresearch_flow/paper/schemas/eight_questions_schema.json,sha256=VFKKpdZkgPdQkYIW5jyrZQ7c2TlQZwB4svVWfoiwxdg,1005
 deepresearch_flow/paper/schemas/three_pass_schema.json,sha256=8aNr4EdRiilxszIRBCC4hRNXrfIOcdnVW4Qhe6Fnh0o,689
 deepresearch_flow/paper/snapshot/__init__.py,sha256=1VLO36xxDB3J5Yoo-HH9vyI-4ev2HcivXN0sNLg8O5k,102
-deepresearch_flow/paper/snapshot/api.py,sha256=WgkOgS7n_2Fx-Bl4KnLrh5nhRJAsWJaPjXu7vX5ubxY,36960
+deepresearch_flow/paper/snapshot/api.py,sha256=F_qehvCjxTBTGj9FmqP4NnJQayUPJm0N5e_8mm5JlDQ,37405
 deepresearch_flow/paper/snapshot/builder.py,sha256=HbRcfNteMoP4RnQ4y2onZCm9XfnIvzXLn_EwsLZsDzY,38692
+deepresearch_flow/paper/snapshot/common.py,sha256=KAhlGlPgabOCe9Faps8BoDqin71qpkCfaL_ADCr_9vg,917
 deepresearch_flow/paper/snapshot/identity.py,sha256=k9x1EZPFBU1qgxzkTGvwVtDjLgcosmM_udPuvRLl0uI,7748
+deepresearch_flow/paper/snapshot/mcp_server.py,sha256=lvgbXmuZCZ_zaQMdZEMjN-OChHPdoZ9MmuuQ-7ORias,22901
 deepresearch_flow/paper/snapshot/schema.py,sha256=DcVmAklLYyEeDoVV9jYw7hoMHnHd9Eziivl-LP2busY,8991
 deepresearch_flow/paper/snapshot/text.py,sha256=0RnxLowa6AdirdLsUYym6BhWbjwiP2Qj2oZeA-pjmdE,4368
+deepresearch_flow/paper/snapshot/unpacker.py,sha256=ScKSFdrQLJHrITHe9KAxgAEH-vAAnXLolvW9zeJ3wsc,8575
 deepresearch_flow/paper/snapshot/tests/__init__.py,sha256=G0IowrxHjGUIaqxcw6SvlcLFAtE5ZsleG6ECgd-sIdk,52
 deepresearch_flow/paper/snapshot/tests/test_identity.py,sha256=KDFixAUU9l68KOum7gf1IrD0Oy18dBCSXG7RbJTqflA,4520
 deepresearch_flow/paper/templates/__init__.py,sha256=p8W6kINvrf-T2X6Ow4GMr28syVOorFuMn0pbmieVzAw,35
@@ -463,9 +466,9 @@ deepresearch_flow/translator/placeholder.py,sha256=mEgqA-dPdOsIhno0h_hzfpXpY2asb
 deepresearch_flow/translator/prompts.py,sha256=EvfBvBIpQXARDj4m87GAyFXJGL8EJeahj_rOmp9mv68,5556
 deepresearch_flow/translator/protector.py,sha256=yUMuS2FgVofK_MRXrcauLRiwNvdCCjNAnh6CcNd686o,11777
 deepresearch_flow/translator/segment.py,sha256=rBFMCLTrvm2GrPc_hNFymi-8Ih2DAtUQlZHCRE9nLaM,5146
-deepresearch_flow-0.6.1.dist-info/licenses/LICENSE,sha256=hT8F2Py1pe6flxq3Ufdm2UKFk0B8CBm0aAQfsLXfvjw,1063
-deepresearch_flow-0.6.1.dist-info/METADATA,sha256=y_CHy1YJ-3P31W43Q_fd8dEkznj7LKLRrCF6F-sGHaQ,26696
-deepresearch_flow-0.6.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-deepresearch_flow-0.6.1.dist-info/entry_points.txt,sha256=1uIKscs0YRMg_mFsg9NjsaTt4CvQqQ_-zGERUKhhL_Y,65
-deepresearch_flow-0.6.1.dist-info/top_level.txt,sha256=qBl4RvPJNJUbL8CFfMNWxY0HpQLx5RlF_ko-z_aKpm0,18
-deepresearch_flow-0.6.1.dist-info/RECORD,,
+deepresearch_flow-0.7.0.dist-info/licenses/LICENSE,sha256=hT8F2Py1pe6flxq3Ufdm2UKFk0B8CBm0aAQfsLXfvjw,1063
+deepresearch_flow-0.7.0.dist-info/METADATA,sha256=aluWW1CXPeSWCLKopChdbgl_GHEQHByua1fBohr6Mzg,26728
+deepresearch_flow-0.7.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+deepresearch_flow-0.7.0.dist-info/entry_points.txt,sha256=1uIKscs0YRMg_mFsg9NjsaTt4CvQqQ_-zGERUKhhL_Y,65
+deepresearch_flow-0.7.0.dist-info/top_level.txt,sha256=qBl4RvPJNJUbL8CFfMNWxY0HpQLx5RlF_ko-z_aKpm0,18
+deepresearch_flow-0.7.0.dist-info/RECORD,,