deepresearch-flow 0.6.1__tar.gz → 0.7.1__tar.gz

{deepresearch_flow-0.6.1/python/deepresearch_flow.egg-info → deepresearch_flow-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepresearch-flow
-Version: 0.6.1
+Version: 0.7.1
 Summary: Workflow tools for paper extraction, review, and research automation.
 Author-email: DengQi <dengqi935@gmail.com>
 License: MIT License
@@ -49,6 +49,7 @@ Requires-Dist: jinja2>=3.1.3
 Requires-Dist: json-repair>=0.55.1
 Requires-Dist: jsonschema>=4.26.0
 Requires-Dist: markdown-it-py>=3.0.0
+Requires-Dist: fastmcp>=3.0.0b1
 Requires-Dist: mdit-py-plugins>=0.4.0
 Requires-Dist: pypdf>=6.6.2
 Requires-Dist: pylatexenc>=2.10
@@ -56,7 +57,7 @@ Requires-Dist: pybtex>=0.24.0
 Requires-Dist: rich>=14.3.1
 Requires-Dist: rumdl>=0.1.6
 Requires-Dist: starlette>=0.52.1
-Requires-Dist: tqdm>=4.66.4
+Requires-Dist: tqdm>=4.67.2
 Requires-Dist: uvicorn>=0.27.1
 Dynamic: license-file
 
@@ -519,7 +520,14 @@ server {
   }
 
   location /api/ {
-    proxy_pass http://127.0.0.1:8001/;
+    proxy_pass http://127.0.0.1:8001;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+  }
+
+  location ^~ /mcp {
+    proxy_pass http://127.0.0.1:8001;
     proxy_set_header Host $host;
     proxy_set_header X-Real-IP $remote_addr;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -554,6 +562,147 @@ uv run deepresearch-flow paper db api serve \
   --host 0.0.0.0 --port 8001
 ```
 
+### 3.1) MCP (FastMCP Streamable HTTP)
+
+This project exposes an MCP server mounted at `/mcp` on the snapshot API:
+
+- Endpoint: `http://<host>:8001/mcp` (same host/port as `paper db api serve`)
+- Transport: Streamable HTTP via `POST` only (no SSE; `GET` returns 405)
+- Protocol header: optional `mcp-protocol-version` (`2025-03-26` or `2025-06-18`)
+- Static reads: summary/source/translation are served as **text content** by reading snapshot static assets (local-first via `PAPER_DB_STATIC_EXPORT_DIR`, HTTP fallback via `PAPER_DB_STATIC_BASE` / `PAPER_DB_STATIC_BASE_URL`)
+
+Optional (avoid HTTP fetch by reading exported assets directly on the API host):
+
+```bash
+export PAPER_DB_STATIC_EXPORT_DIR=/data/paper-static
+```
+
+#### MCP Tools (API functions)
+
+<details>
+<summary><strong>search_papers(query, limit=10)</strong> — full-text search (relevance-ranked)</summary>
+
+- Args:
+  - `query` (str): keywords / topic query
+  - `limit` (int): number of results (clamped to API max page size)
+- Returns: list of `{ paper_id, title, year, venue, snippet_markdown }`
+
+</details>
+
+<details>
+<summary><strong>search_papers_by_keyword(keyword, limit=10)</strong> — facet keyword search</summary>
+
+- Args:
+  - `keyword` (str): keyword substring
+  - `limit` (int): number of results (clamped)
+- Returns: list of `{ paper_id, title, year, venue, snippet_markdown }`
+
+</details>
+
+<details>
+<summary><strong>get_paper_metadata(paper_id)</strong> — metadata + available summary templates</summary>
+
+- Args:
+  - `paper_id` (str)
+- Returns: dict with:
+  - `paper_id`, `title`, `year`, `venue`
+  - `doi`, `arxiv_id`, `openreview_id`, `paper_pw_url`
+  - `preferred_summary_template`, `available_summary_templates`
+
+</details>
+
+<details>
+<summary><strong>get_paper_summary(paper_id, template=None, max_chars=None)</strong> — summary JSON as raw text</summary>
+
+- Notes:
+  - Uses `preferred_summary_template` if `template` is omitted
+  - Returns the **full JSON content** (not a URL)
+- Args:
+  - `paper_id` (str)
+  - `template` (str | null)
+  - `max_chars` (int | null): truncation limit
+- Returns: JSON string (may include a `[truncated: ...]` marker)
+
+</details>
+
+<details>
+<summary><strong>get_paper_source(paper_id, max_chars=None)</strong> — source markdown as raw text</summary>
+
+- Args:
+  - `paper_id` (str)
+  - `max_chars` (int | null): truncation limit
+- Returns: markdown string (may include a `[truncated: ...]` marker)
+
+</details>
+
+<details>
+<summary><strong>get_database_stats()</strong> — snapshot-level stats</summary>
+
+- Returns:
+  - `total`
+  - `years`, `months`: list of `{ value, paper_count }`
+  - `authors`, `venues`, `institutions`, `keywords`, `tags`: top lists of `{ value, paper_count }`
+
+</details>
+
+<details>
+<summary><strong>list_top_facets(category, limit=20)</strong> — top values for one facet</summary>
+
+- Args:
+  - `category`: `author | venue | keyword | institution | tag`
+  - `limit` (int)
+- Returns: list of `{ value, paper_count }`
+
+</details>
+
+<details>
+<summary><strong>filter_papers(author=None, venue=None, year=None, keyword=None, tag=None, limit=10)</strong> — structured filtering</summary>
+
+- Args (all optional except `limit`):
+  - `author`, `venue`, `keyword`, `tag`: substring match
+  - `year`: exact match
+  - `limit` (int): number of results (clamped)
+- Returns: list of `{ paper_id, title, year, venue }`
+
+</details>
+
+#### MCP Resources (URI access)
+
+<details>
+<summary><strong>paper://{paper_id}/metadata</strong> — metadata JSON</summary>
+
+Returns the same content as `get_paper_metadata(paper_id)` (as a JSON string).
+
+</details>
+
+<details>
+<summary><strong>paper://{paper_id}/summary</strong> — preferred summary JSON</summary>
+
+Returns the same content as `get_paper_summary(paper_id)` (preferred template; JSON string).
+
+</details>
+
+<details>
+<summary><strong>paper://{paper_id}/summary/{template}</strong> — summary JSON for template</summary>
+
+Returns the same content as `get_paper_summary(paper_id, template=template)` (JSON string).
+
+</details>
+
+<details>
+<summary><strong>paper://{paper_id}/source</strong> — source markdown</summary>
+
+Returns the same content as `get_paper_source(paper_id)` (markdown string).
+
+</details>
+
+<details>
+<summary><strong>paper://{paper_id}/translation/{lang}</strong> — translated markdown</summary>
+
+Returns translated markdown for `lang` (e.g. `zh`, `ja`) when available.
+
+</details>
+
 ### 4) Frontend (static build or dev)
 
 ```bash
@@ -810,7 +959,7 @@ docker run --rm -p 8899:8899 \
 ```
 
 Notes:
-- nginx listens on 8899 and proxies `/api` to the internal API at `127.0.0.1:8000`.
+- nginx listens on 8899 and proxies `/api` and `/mcp` to the internal API at `127.0.0.1:8000`.
 - Mount your snapshot DB to `/db/papers.db` inside the container.
 - Mount snapshot static assets to `/static` when serving assets from this container (default `PAPER_DB_STATIC_BASE` is `/static`).
 - If `PAPER_DB_STATIC_BASE` is a full URL (e.g. `https://static.example.com`), nginx still serves the frontend locally, while API responses use that external static base for asset links.

{deepresearch_flow-0.6.1 → deepresearch_flow-0.7.1}/README.md

@@ -457,7 +457,14 @@ server {
   }
 
   location /api/ {
-    proxy_pass http://127.0.0.1:8001/;
+    proxy_pass http://127.0.0.1:8001;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+  }
+
+  location ^~ /mcp {
+    proxy_pass http://127.0.0.1:8001;
     proxy_set_header Host $host;
     proxy_set_header X-Real-IP $remote_addr;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -492,6 +499,147 @@ uv run deepresearch-flow paper db api serve \
   --host 0.0.0.0 --port 8001
 ```
 
+### 3.1) MCP (FastMCP Streamable HTTP)
+
+This project exposes an MCP server mounted at `/mcp` on the snapshot API:
+
+- Endpoint: `http://<host>:8001/mcp` (same host/port as `paper db api serve`)
+- Transport: Streamable HTTP via `POST` only (no SSE; `GET` returns 405)
+- Protocol header: optional `mcp-protocol-version` (`2025-03-26` or `2025-06-18`)
+- Static reads: summary/source/translation are served as **text content** by reading snapshot static assets (local-first via `PAPER_DB_STATIC_EXPORT_DIR`, HTTP fallback via `PAPER_DB_STATIC_BASE` / `PAPER_DB_STATIC_BASE_URL`)
+
+Optional (avoid HTTP fetch by reading exported assets directly on the API host):
+
+```bash
+export PAPER_DB_STATIC_EXPORT_DIR=/data/paper-static
+```
+
+#### MCP Tools (API functions)
+
+<details>
+<summary><strong>search_papers(query, limit=10)</strong> — full-text search (relevance-ranked)</summary>
+
+- Args:
+  - `query` (str): keywords / topic query
+  - `limit` (int): number of results (clamped to API max page size)
+- Returns: list of `{ paper_id, title, year, venue, snippet_markdown }`
+
+</details>
+
+<details>
+<summary><strong>search_papers_by_keyword(keyword, limit=10)</strong> — facet keyword search</summary>
+
+- Args:
+  - `keyword` (str): keyword substring
+  - `limit` (int): number of results (clamped)
+- Returns: list of `{ paper_id, title, year, venue, snippet_markdown }`
+
+</details>
+
+<details>
+<summary><strong>get_paper_metadata(paper_id)</strong> — metadata + available summary templates</summary>
+
+- Args:
+  - `paper_id` (str)
+- Returns: dict with:
+  - `paper_id`, `title`, `year`, `venue`
+  - `doi`, `arxiv_id`, `openreview_id`, `paper_pw_url`
+  - `preferred_summary_template`, `available_summary_templates`
+
+</details>
+
+<details>
+<summary><strong>get_paper_summary(paper_id, template=None, max_chars=None)</strong> — summary JSON as raw text</summary>
+
+- Notes:
+  - Uses `preferred_summary_template` if `template` is omitted
+  - Returns the **full JSON content** (not a URL)
+- Args:
+  - `paper_id` (str)
+  - `template` (str | null)
+  - `max_chars` (int | null): truncation limit
+- Returns: JSON string (may include a `[truncated: ...]` marker)
+
+</details>
+
+<details>
+<summary><strong>get_paper_source(paper_id, max_chars=None)</strong> — source markdown as raw text</summary>
+
+- Args:
+  - `paper_id` (str)
+  - `max_chars` (int | null): truncation limit
+- Returns: markdown string (may include a `[truncated: ...]` marker)
+
+</details>
+
+<details>
+<summary><strong>get_database_stats()</strong> — snapshot-level stats</summary>
+
+- Returns:
+  - `total`
+  - `years`, `months`: list of `{ value, paper_count }`
+  - `authors`, `venues`, `institutions`, `keywords`, `tags`: top lists of `{ value, paper_count }`
+
+</details>
+
+<details>
+<summary><strong>list_top_facets(category, limit=20)</strong> — top values for one facet</summary>
+
+- Args:
+  - `category`: `author | venue | keyword | institution | tag`
+  - `limit` (int)
+- Returns: list of `{ value, paper_count }`
+
+</details>
+
+<details>
+<summary><strong>filter_papers(author=None, venue=None, year=None, keyword=None, tag=None, limit=10)</strong> — structured filtering</summary>
+
+- Args (all optional except `limit`):
+  - `author`, `venue`, `keyword`, `tag`: substring match
+  - `year`: exact match
+  - `limit` (int): number of results (clamped)
+- Returns: list of `{ paper_id, title, year, venue }`
+
+</details>
+
+#### MCP Resources (URI access)
+
+<details>
+<summary><strong>paper://{paper_id}/metadata</strong> — metadata JSON</summary>
+
+Returns the same content as `get_paper_metadata(paper_id)` (as a JSON string).
+
+</details>
+
+<details>
+<summary><strong>paper://{paper_id}/summary</strong> — preferred summary JSON</summary>
+
+Returns the same content as `get_paper_summary(paper_id)` (preferred template; JSON string).
+
+</details>
+
+<details>
+<summary><strong>paper://{paper_id}/summary/{template}</strong> — summary JSON for template</summary>
+
+Returns the same content as `get_paper_summary(paper_id, template=template)` (JSON string).
+
+</details>
+
+<details>
+<summary><strong>paper://{paper_id}/source</strong> — source markdown</summary>
+
+Returns the same content as `get_paper_source(paper_id)` (markdown string).
+
+</details>
+
+<details>
+<summary><strong>paper://{paper_id}/translation/{lang}</strong> — translated markdown</summary>
+
+Returns translated markdown for `lang` (e.g. `zh`, `ja`) when available.
+
+</details>
+
 ### 4) Frontend (static build or dev)
 
 ```bash
@@ -748,7 +896,7 @@ docker run --rm -p 8899:8899 \
 ```
 
 Notes:
-- nginx listens on 8899 and proxies `/api` to the internal API at `127.0.0.1:8000`.
+- nginx listens on 8899 and proxies `/api` and `/mcp` to the internal API at `127.0.0.1:8000`.
 - Mount your snapshot DB to `/db/papers.db` inside the container.
 - Mount snapshot static assets to `/static` when serving assets from this container (default `PAPER_DB_STATIC_BASE` is `/static`).
 - If `PAPER_DB_STATIC_BASE` is a full URL (e.g. `https://static.example.com`), nginx still serves the frontend locally, while API responses use that external static base for asset links.

{deepresearch_flow-0.6.1 → deepresearch_flow-0.7.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "deepresearch-flow"
-version = "0.6.1"
+version = "0.7.1"
 description = "Workflow tools for paper extraction, review, and research automation."
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE" }
@@ -18,6 +18,7 @@ dependencies = [
     "json-repair>=0.55.1",
     "jsonschema>=4.26.0",
     "markdown-it-py>=3.0.0",
+    "fastmcp>=3.0.0b1",
     "mdit-py-plugins>=0.4.0",
     "pypdf>=6.6.2",
     "pylatexenc>=2.10",
@@ -25,7 +26,7 @@ dependencies = [
     "rich>=14.3.1",
     "rumdl>=0.1.6",
     "starlette>=0.52.1",
-    "tqdm>=4.66.4",
+    "tqdm>=4.67.2",
     "uvicorn>=0.27.1",
 ]
 keywords = ["research", "papers", "pdf", "ocr", "llm", "workflow"]

{deepresearch_flow-0.6.1 → deepresearch_flow-0.7.1}/python/deepresearch_flow/__init__.py

@@ -2,4 +2,4 @@
 
 __all__ = ["__version__"]
 
-__version__ = "0.1.0"
+__version__ = "0.7.1"

{deepresearch_flow-0.6.1 → deepresearch_flow-0.7.1}/python/deepresearch_flow/paper/db.py

@@ -424,6 +424,87 @@ def register_db_commands(db_group: click.Group) -> None:
         click.echo(f"Wrote snapshot DB: {opts.output_db}")
         click.echo(f"Wrote static export: {opts.static_export_dir}")
 
+    @snapshot_group.group("unpack")
+    def snapshot_unpack_group() -> None:
+        """Unpack snapshot artifacts."""
+
+    @snapshot_unpack_group.command("md")
+    @click.option("--snapshot-db", "snapshot_db", required=True, help="Path to snapshot database")
+    @click.option(
+        "--static-export-dir",
+        "static_export_dir",
+        required=True,
+        help="Path to static export directory",
+    )
+    @click.option(
+        "--pdf-root",
+        "pdf_roots",
+        multiple=True,
+        required=True,
+        help="PDF root directories for name alignment (repeatable)",
+    )
+    @click.option("--md-output-dir", "md_output_dir", required=True, help="Output directory for Markdown")
+    @click.option(
+        "--md-translated-output-dir",
+        "md_translated_output_dir",
+        required=True,
+        help="Output directory for translated Markdown",
+    )
+    def snapshot_unpack_md(
+        snapshot_db: str,
+        static_export_dir: str,
+        pdf_roots: tuple[str, ...],
+        md_output_dir: str,
+        md_translated_output_dir: str,
+    ) -> None:
+        """Unpack source/translated markdown and align filenames to PDFs."""
+        from deepresearch_flow.paper.snapshot.unpacker import SnapshotUnpackMdOptions, unpack_md
+
+        opts = SnapshotUnpackMdOptions(
+            snapshot_db=Path(snapshot_db),
+            static_export_dir=Path(static_export_dir),
+            pdf_roots=[Path(path) for path in pdf_roots],
+            md_output_dir=Path(md_output_dir),
+            md_translated_output_dir=Path(md_translated_output_dir),
+        )
+        unpack_md(opts)
+
+    @snapshot_unpack_group.command("info")
+    @click.option("--snapshot-db", "snapshot_db", required=True, help="Path to snapshot database")
+    @click.option(
+        "--static-export-dir",
+        "static_export_dir",
+        required=True,
+        help="Path to static export directory",
+    )
+    @click.option(
+        "--pdf-root",
+        "pdf_roots",
+        multiple=True,
+        required=True,
+        help="PDF root directories for name alignment (repeatable)",
+    )
+    @click.option("--template", "template", required=True, help="Summary template tag")
+    @click.option("--output-json", "output_json", required=True, help="Output JSON file path")
+    def snapshot_unpack_info(
+        snapshot_db: str,
+        static_export_dir: str,
+        pdf_roots: tuple[str, ...],
+        template: str,
+        output_json: str,
+    ) -> None:
+        """Unpack aggregated paper_infos.json from snapshot summaries."""
+        from deepresearch_flow.paper.snapshot.unpacker import SnapshotUnpackInfoOptions, unpack_info
+
+        opts = SnapshotUnpackInfoOptions(
+            snapshot_db=Path(snapshot_db),
+            static_export_dir=Path(static_export_dir),
+            pdf_roots=[Path(path) for path in pdf_roots],
+            template=template,
+            output_json=Path(output_json),
+        )
+        unpack_info(opts)
+
     @db_group.group("api")
     def api_group() -> None:
         """Read-only JSON API server backed by a snapshot DB."""

{deepresearch_flow-0.6.1 → deepresearch_flow-0.7.1}/python/deepresearch_flow/paper/snapshot/api.py

@@ -11,8 +11,9 @@ from starlette.applications import Starlette
 from starlette.middleware.cors import CORSMiddleware
 from starlette.requests import Request
 from starlette.responses import JSONResponse, Response
-from starlette.routing import Route
+from starlette.routing import Mount, Route
 
+from deepresearch_flow.paper.snapshot.common import ApiLimits, _open_ro_conn
 from deepresearch_flow.paper.snapshot.text import merge_adjacent_markers, remove_cjk_spaces, rewrite_search_query
 
 _WHITESPACE_RE = re.compile(r"\s+")
@@ -87,13 +88,6 @@ _FACET_TYPE_TO_KEY = {
 }
 
 
-@dataclass(frozen=True)
-class ApiLimits:
-    max_query_length: int = 500
-    max_page_size: int = 100
-    max_pagination_offset: int = 10_000  # page * page_size
-
-
 @dataclass(frozen=True)
 class SnapshotApiConfig:
     snapshot_db: Path
@@ -110,12 +104,6 @@ def _json_error(status_code: int, *, error: str, detail: str) -> JSONResponse:
     return JSONResponse({"error": error, "detail": detail}, status_code=status_code)
 
 
-def _open_ro_conn(db_path: Path) -> sqlite3.Connection:
-    uri = f"file:{db_path.as_posix()}?mode=ro"
-    conn = sqlite3.connect(uri, uri=True)
-    conn.row_factory = sqlite3.Row
-    conn.execute("PRAGMA query_only=ON;")
-    return conn
 
 
 def _snapshot_build_id(conn: sqlite3.Connection) -> str:
@@ -917,6 +905,22 @@ def create_app(
         limits=limits or ApiLimits(),
     )
 
+    # Lazy import to avoid circular dependency
+    from deepresearch_flow.paper.snapshot.mcp_server import (
+        McpSnapshotConfig,
+        create_mcp_app,
+        resolve_static_export_dir,
+    )
+
+    mcp_config = McpSnapshotConfig(
+        snapshot_db=snapshot_db,
+        static_base_url=_normalize_base_url(static_base_url),
+        static_export_dir=resolve_static_export_dir(),
+        limits=limits or ApiLimits(),
+        origin_allowlist=cors_allowed_origins or ["*"],
+    )
+    mcp_app, mcp_lifespan = create_mcp_app(mcp_config)
+
     routes = [
         Route("/api/v1/config", _api_config, methods=["GET"]),
         Route("/api/v1/search", _api_search, methods=["GET"]),
@@ -927,9 +931,15 @@ def create_app(
         Route("/api/v1/facets/{facet:str}/{facet_id:str}/stats", _api_facet_stats, methods=["GET"]),
         Route("/api/v1/facets/{facet:str}/by-value/{value:str}/papers", _api_facet_by_value_papers, methods=["GET"]),
         Route("/api/v1/facets/{facet:str}/by-value/{value:str}/stats", _api_facet_by_value_stats, methods=["GET"]),
+        Mount("/mcp", app=mcp_app),
     ]
 
-    app = Starlette(routes=routes)
+    # Pass MCP lifespan to ensure session manager initializes properly
+    # https://gofastmcp.com/deployment/http#mounting-in-starlette
+    app = Starlette(
+        routes=routes,
+        lifespan=mcp_lifespan,
+    )
     if cfg.cors_allowed_origins:
         app.add_middleware(
             CORSMiddleware,

deepresearch_flow-0.7.1/python/deepresearch_flow/paper/snapshot/common.py

@@ -0,0 +1,34 @@
+"""Shared utilities for snapshot API and MCP server.
+
+This module contains common types, configuration, and utilities used by both
+the snapshot REST API and the MCP server to avoid circular imports.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+import sqlite3
+
+
+@dataclass(frozen=True)
+class ApiLimits:
+    """API rate and size limits."""
+
+    max_query_length: int = 500
+    max_page_size: int = 100
+    max_pagination_offset: int = 10_000
+
+
+def _open_ro_conn(db_path: Path) -> sqlite3.Connection:
+    """Open a read-only SQLite connection with Row factory.
+
+    Args:
+        db_path: Path to the SQLite database file.
+
+    Returns:
+        sqlite3.Connection: A read-only connection with row_factory set to Row.
+    """
+    conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, check_same_thread=False)
+    conn.row_factory = sqlite3.Row
+    return conn