deepresearch-flow 0.4.1__tar.gz → 0.5.0__tar.gz

{deepresearch_flow-0.4.1/python/deepresearch_flow.egg-info → deepresearch_flow-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepresearch-flow
-Version: 0.4.1
+Version: 0.5.0
 Summary: Workflow tools for paper extraction, review, and research automation.
 Author-email: DengQi <dengqi935@gmail.com>
 License: MIT License
@@ -87,7 +87,7 @@ Dynamic: license-file
   <a href="https://hub.docker.com/r/nerdneils/deepresearch-flow">
     <img src="https://img.shields.io/docker/v/nerdneils/deepresearch-flow?style=flat-square" />
   </a>
-  <a href="https://ghcr.io/nerdneilsfield/deepresearch-flow">
+  <a href="https://github.com/nerdneilsfield/ai-deepresearch-flow/pkgs/container/deepresearch-flow">
     <img src="https://img.shields.io/badge/ghcr.io-nerdneilsfield%2Fdeepresearch-flow-0f172a?style=flat-square" />
   </a>
   <a href="https://github.com/nerdneilsfield/ai-deepresearch-flow/blob/main/LICENSE">
@@ -216,6 +216,66 @@ uv run deepresearch-flow paper db serve \
 
 ---
 
+## Deployment (Static CDN)
+
+Use a separate static server (CDN) for PDFs/Markdown/images and keep the API/UI on another host.
+
+### 1) Export static assets
+
+```bash
+uv run deepresearch-flow paper db serve \
+  --input paper_infos.json \
+  --md-root ./docs \
+  --md-translated-root ./docs \
+  --pdf-root ./pdfs \
+  --static-mode prod \
+  --static-base-url https://static.example.com \
+  --static-export-dir /data/paper-static
+```
+
+Notes:
+- The API host must be able to read the original PDF/Markdown roots to build the index and hashes.
+- The CDN host only needs the exported directory (e.g. `/data/paper-static`).
+
+### 2) Serve the export directory with CORS + cache headers (Caddy example)
+
+```caddyfile
+:8002 {
+  root * /data/paper-static
+  encode zstd gzip
+
+  @static path /pdf/* /md/* /md_translate/* /images/*
+  header @static {
+    Access-Control-Allow-Origin *
+    Access-Control-Allow-Methods GET,HEAD,OPTIONS
+    Access-Control-Allow-Headers *
+    Cache-Control "public, max-age=31536000, immutable"
+  }
+
+  @options method OPTIONS
+  respond @options 204
+
+  file_server
+}
+```
+
+### 3) Start the API/UI with static base
+
+```bash
+export PAPER_DB_STATIC_BASE_URL="https://static.example.com"
+export PAPER_DB_STATIC_MODE="prod"
+export PAPER_DB_STATIC_EXPORT_DIR="/data/paper-static"
+export PAPER_DB_PDFJS_CDN_BASE_URL="https://cdnjs.cloudflare.com/ajax/libs/pdf.js/4.0.379"
+
+uv run deepresearch-flow paper db serve \
+  --input paper_infos.json \
+  --md-root ./docs \
+  --md-translated-root ./docs \
+  --pdf-root ./pdfs
+```
+
+---
+
 ## Comprehensive Guide
 
 <details>

{deepresearch_flow-0.4.1 → deepresearch_flow-0.5.0}/README.md

@@ -25,7 +25,7 @@
   <a href="https://hub.docker.com/r/nerdneils/deepresearch-flow">
     <img src="https://img.shields.io/docker/v/nerdneils/deepresearch-flow?style=flat-square" />
   </a>
-  <a href="https://ghcr.io/nerdneilsfield/deepresearch-flow">
+  <a href="https://github.com/nerdneilsfield/ai-deepresearch-flow/pkgs/container/deepresearch-flow">
     <img src="https://img.shields.io/badge/ghcr.io-nerdneilsfield%2Fdeepresearch-flow-0f172a?style=flat-square" />
   </a>
   <a href="https://github.com/nerdneilsfield/ai-deepresearch-flow/blob/main/LICENSE">
@@ -154,6 +154,66 @@ uv run deepresearch-flow paper db serve \
 
 ---
 
+## Deployment (Static CDN)
+
+Use a separate static server (CDN) for PDFs/Markdown/images and keep the API/UI on another host.
+
+### 1) Export static assets
+
+```bash
+uv run deepresearch-flow paper db serve \
+  --input paper_infos.json \
+  --md-root ./docs \
+  --md-translated-root ./docs \
+  --pdf-root ./pdfs \
+  --static-mode prod \
+  --static-base-url https://static.example.com \
+  --static-export-dir /data/paper-static
+```
+
+Notes:
+- The API host must be able to read the original PDF/Markdown roots to build the index and hashes.
+- The CDN host only needs the exported directory (e.g. `/data/paper-static`).
+
+### 2) Serve the export directory with CORS + cache headers (Caddy example)
+
+```caddyfile
+:8002 {
+  root * /data/paper-static
+  encode zstd gzip
+
+  @static path /pdf/* /md/* /md_translate/* /images/*
+  header @static {
+    Access-Control-Allow-Origin *
+    Access-Control-Allow-Methods GET,HEAD,OPTIONS
+    Access-Control-Allow-Headers *
+    Cache-Control "public, max-age=31536000, immutable"
+  }
+
+  @options method OPTIONS
+  respond @options 204
+
+  file_server
+}
+```
+
+### 3) Start the API/UI with static base
+
+```bash
+export PAPER_DB_STATIC_BASE_URL="https://static.example.com"
+export PAPER_DB_STATIC_MODE="prod"
+export PAPER_DB_STATIC_EXPORT_DIR="/data/paper-static"
+export PAPER_DB_PDFJS_CDN_BASE_URL="https://cdnjs.cloudflare.com/ajax/libs/pdf.js/4.0.379"
+
+uv run deepresearch-flow paper db serve \
+  --input paper_infos.json \
+  --md-root ./docs \
+  --md-translated-root ./docs \
+  --pdf-root ./pdfs
+```
+
+---
+
 ## Comprehensive Guide
 
 <details>

{deepresearch_flow-0.4.1 → deepresearch_flow-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "deepresearch-flow"
-version = "0.4.1"
+version = "0.5.0"
 description = "Workflow tools for paper extraction, review, and research automation."
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE" }

{deepresearch_flow-0.4.1 → deepresearch_flow-0.5.0}/python/deepresearch_flow/paper/db.py

@@ -600,6 +600,32 @@ def register_db_commands(db_group: click.Group) -> None:
     )
     @click.option("--cache-dir", "cache_dir", default=None, help="Cache directory for merged inputs")
     @click.option("--no-cache", "no_cache", is_flag=True, help="Disable cache for db serve")
+    @click.option(
+        "--static-base-url",
+        "static_base_url",
+        default=None,
+        help="Static asset base URL (e.g. https://static.example.com)",
+    )
+    @click.option(
+        "--static-mode",
+        "static_mode",
+        type=click.Choice(["auto", "dev", "prod"]),
+        default="auto",
+        show_default=True,
+        help="Static asset mode (dev uses local assets, prod uses static base URL)",
+    )
+    @click.option(
+        "--static-export-dir",
+        "static_export_dir",
+        default=None,
+        help="Optional export directory for hashed static assets",
+    )
+    @click.option(
+        "--pdfjs-cdn-base-url",
+        "pdfjs_cdn_base_url",
+        default=None,
+        help="PDF.js CDN base URL (defaults to jsDelivr)",
+    )
     @click.option("--host", default="127.0.0.1", show_default=True, help="Bind host")
     @click.option("--port", default=8000, type=int, show_default=True, help="Bind port")
     @click.option(
@@ -617,6 +643,10 @@ def register_db_commands(db_group: click.Group) -> None:
         pdf_roots: tuple[str, ...],
         cache_dir: str | None,
         no_cache: bool,
+        static_base_url: str | None,
+        static_mode: str,
+        static_export_dir: str | None,
+        pdfjs_cdn_base_url: str | None,
         host: str,
         port: int,
         fallback_language: str,
@@ -635,6 +665,10 @@ def register_db_commands(db_group: click.Group) -> None:
                 pdf_roots=[Path(root) for root in pdf_roots],
                 cache_dir=Path(cache_dir) if cache_dir else None,
                 use_cache=not no_cache,
+                static_base_url=static_base_url,
+                static_mode=static_mode,
+                static_export_dir=Path(static_export_dir) if static_export_dir else None,
+                pdfjs_cdn_base_url=pdfjs_cdn_base_url,
             )
         except Exception as exc:
             raise click.ClickException(str(exc)) from exc

deepresearch_flow-0.5.0/python/deepresearch_flow/paper/web/app.py

@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+
+from starlette.applications import Starlette
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.routing import Mount, Route
+from starlette.staticfiles import StaticFiles
+
+from deepresearch_flow.paper.db_ops import build_index, load_and_merge_papers
+from deepresearch_flow.paper.web.constants import DEFAULT_PDFJS_CDN_BASE_URL, PDFJS_STATIC_DIR, STATIC_DIR
+from deepresearch_flow.paper.web.handlers import (
+    api_markdown,
+    api_papers,
+    api_pdf,
+    api_stats,
+    index_page,
+    paper_detail,
+    robots_txt,
+    stats_page,
+)
+from deepresearch_flow.paper.web.markdown import create_md_renderer
+from deepresearch_flow.paper.web.static_assets import build_static_assets
+
+logger = logging.getLogger(__name__)
+
+
+class _NoIndexMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):  # type: ignore[override]
+        response = await call_next(request)
+        response.headers["X-Robots-Tag"] = "noindex, nofollow, noarchive, nosnippet, noai, noimageai"
+        return response
+
+
+class _StaticAssetFiles(StaticFiles):
+    def __init__(self, *args, cache_control: str | None = None, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._cache_control = cache_control
+
+    async def get_response(self, path: str, scope):  # type: ignore[override]
+        response = await super().get_response(path, scope)
+        if self._cache_control and response.status_code == 200:
+            response.headers.setdefault("Cache-Control", self._cache_control)
+        return response
+
+
+def _normalize_static_mode(value: str | None) -> str:
+    if not value:
+        return "auto"
+    normalized = value.strip().lower()
+    if normalized in {"dev", "development"}:
+        return "dev"
+    if normalized in {"prod", "production"}:
+        return "prod"
+    return "auto"
+
+
+def _resolve_static_mode(value: str, static_base_url: str | None) -> str:
+    if value == "auto":
+        return "prod" if static_base_url else "dev"
+    return value
+
+
+def create_app(
+    *,
+    db_paths: list[Path],
+    fallback_language: str = "en",
+    bibtex_path: Path | None = None,
+    md_roots: list[Path] | None = None,
+    md_translated_roots: list[Path] | None = None,
+    pdf_roots: list[Path] | None = None,
+    cache_dir: Path | None = None,
+    use_cache: bool = True,
+    static_base_url: str | None = None,
+    static_mode: str | None = None,
+    static_export_dir: Path | None = None,
+    pdfjs_cdn_base_url: str | None = None,
+) -> Starlette:
+    papers = load_and_merge_papers(db_paths, bibtex_path, cache_dir, use_cache, pdf_roots=pdf_roots)
+
+    md_roots = md_roots or []
+    md_translated_roots = md_translated_roots or []
+    pdf_roots = pdf_roots or []
+    index = build_index(
+        papers,
+        md_roots=md_roots,
+        md_translated_roots=md_translated_roots,
+        pdf_roots=pdf_roots,
+    )
+    md = create_md_renderer()
+    static_base_url = static_base_url or os.getenv("PAPER_DB_STATIC_BASE_URL")
+    static_mode = _normalize_static_mode(static_mode or os.getenv("PAPER_DB_STATIC_MODE"))
+    resolved_mode = _resolve_static_mode(static_mode, static_base_url)
+    export_dir_value = static_export_dir or os.getenv("PAPER_DB_STATIC_EXPORT_DIR")
+    export_dir = Path(export_dir_value) if export_dir_value else None
+    pdfjs_cdn_base_url = (
+        pdfjs_cdn_base_url
+        or os.getenv("PAPER_DB_PDFJS_CDN_BASE_URL")
+        or DEFAULT_PDFJS_CDN_BASE_URL
+    )
+    if pdfjs_cdn_base_url:
+        lowered = pdfjs_cdn_base_url.strip().lower()
+        if lowered in {"none", "off", "local"}:
+            pdfjs_cdn_base_url = None
+        else:
+            pdfjs_cdn_base_url = pdfjs_cdn_base_url.rstrip("/")
+
+    asset_config = None
+    if resolved_mode == "prod":
+        if not static_base_url:
+            logger.warning("Static mode set to prod without base URL; falling back to dev asset routes.")
+            resolved_mode = "dev"
+        else:
+            asset_config = build_static_assets(
+                index,
+                static_base_url=static_base_url,
+                static_export_dir=export_dir,
+            )
+    if resolved_mode == "dev" and export_dir:
+        asset_config = build_static_assets(
+            index,
+            static_base_url="",
+            static_export_dir=export_dir,
+            allow_empty_base=True,
+        )
+    if asset_config is None:
+        asset_config = build_static_assets(index, static_base_url=None)
+
+    routes = [
+        Route("/", index_page, methods=["GET"]),
+        Route("/robots.txt", robots_txt, methods=["GET"]),
+        Route("/stats", stats_page, methods=["GET"]),
+        Route("/paper/{source_hash:str}", paper_detail, methods=["GET"]),
+        Route("/api/papers", api_papers, methods=["GET"]),
+        Route("/api/stats", api_stats, methods=["GET"]),
+        Route("/api/pdf/{source_hash:str}", api_pdf, methods=["GET"]),
+        Route("/api/dev/markdown/{source_hash:str}", api_markdown, methods=["GET"]),
+    ]
+    if PDFJS_STATIC_DIR.exists():
+        routes.append(
+            Mount(
+                "/pdfjs",
+                app=StaticFiles(directory=str(PDFJS_STATIC_DIR), html=True),
+                name="pdfjs",
+            )
+        )
+    elif pdf_roots:
+        logger.warning(
+            "PDF.js viewer assets not found at %s; PDF Viewer mode will be unavailable.",
+            PDFJS_STATIC_DIR,
+        )
+    if STATIC_DIR.exists():
+        routes.append(
+            Mount(
+                "/static",
+                app=StaticFiles(directory=str(STATIC_DIR)),
+                name="static",
+            )
+        )
+    if export_dir and export_dir.exists() and asset_config.enabled and not asset_config.base_url:
+        cache_header = "public, max-age=31536000, immutable"
+        routes.extend(
+            [
+                Mount(
+                    "/pdf",
+                    app=_StaticAssetFiles(directory=str(export_dir / "pdf"), cache_control=cache_header),
+                    name="static_pdf",
+                ),
+                Mount(
+                    "/images",
+                    app=_StaticAssetFiles(directory=str(export_dir / "images"), cache_control=cache_header),
+                    name="static_images",
+                ),
+                Mount(
+                    "/md",
+                    app=_StaticAssetFiles(directory=str(export_dir / "md"), cache_control=cache_header),
+                    name="static_md",
+                ),
+                Mount(
+                    "/md_translate",
+                    app=_StaticAssetFiles(directory=str(export_dir / "md_translate"), cache_control=cache_header),
+                    name="static_md_translate",
+                ),
+            ]
+        )
+    app = Starlette(routes=routes)
+    app.add_middleware(_NoIndexMiddleware)
+    app.state.index = index
+    app.state.md = md
+    app.state.fallback_language = fallback_language
+    app.state.pdf_roots = pdf_roots
+    app.state.static_mode = resolved_mode
+    app.state.asset_config = asset_config
+    app.state.static_export_dir = export_dir
+    app.state.pdfjs_cdn_base_url = pdfjs_cdn_base_url
+    return app

{deepresearch_flow-0.4.1 → deepresearch_flow-0.5.0}/python/deepresearch_flow/paper/web/constants.py

@@ -12,6 +12,7 @@ CDN_KATEX_AUTO = "https://cdn.jsdelivr.net/npm/katex@0.16.27/dist/contrib/auto-r
 # Use legacy builds to ensure `pdfjsLib` is available as a global.
 CDN_PDFJS = "https://cdn.jsdelivr.net/npm/pdfjs-dist@3.11.174/legacy/build/pdf.min.js"
 CDN_PDFJS_WORKER = "https://cdn.jsdelivr.net/npm/pdfjs-dist@3.11.174/legacy/build/pdf.worker.min.js"
+DEFAULT_PDFJS_CDN_BASE_URL = "https://cdn.jsdelivr.net/npm/pdfjs-dist@3.11.174"
 
 # PDF.js viewer configuration
 PDFJS_VIEWER_PATH = "/pdfjs/web/viewer.html"

{deepresearch_flow-0.4.1 → deepresearch_flow-0.5.0}/python/deepresearch_flow/paper/web/handlers/__init__.py

@@ -1,12 +1,13 @@
 """Route handlers for paper web UI."""
 
-from .api import api_papers, api_pdf, api_stats
+from .api import api_markdown, api_papers, api_pdf, api_stats
 from .pages import index_page, paper_detail, robots_txt, stats_page
 
 __all__ = [
     "api_papers",
     "api_pdf",
     "api_stats",
+    "api_markdown",
     "index_page",
     "paper_detail",
     "robots_txt",

{deepresearch_flow-0.4.1 → deepresearch_flow-0.5.0}/python/deepresearch_flow/paper/web/handlers/api.py

@@ -19,6 +19,8 @@ from deepresearch_flow.paper.web.filters import (
     presence_filter,
     sorted_ids,
 )
+from deepresearch_flow.paper.web.markdown import normalize_markdown_images
+from deepresearch_flow.paper.web.static_assets import resolve_asset_urls
 from deepresearch_flow.paper.web.text import extract_summary_snippet, normalize_title, normalize_venue
 from deepresearch_flow.paper.web.query import Query, QueryTerm, parse_query
 
@@ -92,9 +94,18 @@ def _apply_query(index: PaperIndex, query: Query) -> set[int]:
     return result
 
 
+def _safe_read_text(path: Path) -> str:
+    try:
+        return path.read_text(encoding="utf-8")
+    except UnicodeDecodeError:
+        return path.read_text(encoding="latin-1")
+
+
 async def api_papers(request: Request) -> JSONResponse:
     """API endpoint for paper list with filtering, sorting, and pagination."""
     index: PaperIndex = request.app.state.index
+    asset_config = request.app.state.asset_config
+    prefer_local = request.app.state.static_mode == "dev"
     filters = parse_filters(request)
     page = int(filters["page"])
     page_size = int(filters["page_size"])
@@ -165,6 +176,7 @@ async def api_papers(request: Request) -> JSONResponse:
         source_hash = str(paper.get("source_hash") or stable_hash(str(paper.get("source_path") or idx)))
         translations = index.translated_md_by_hash.get(source_hash, {})
         translation_languages = sorted(translations.keys(), key=str.lower)
+        asset_urls = resolve_asset_urls(index, source_hash, asset_config, prefer_local=prefer_local)
         items.append(
             {
                 "source_hash": source_hash,
@@ -183,6 +195,10 @@ async def api_papers(request: Request) -> JSONResponse:
                 "has_summary": bool(paper.get("_has_summary")),
                 "is_pdf_only": bool(paper.get("_is_pdf_only")),
                 "translation_languages": translation_languages,
+                "pdf_url": asset_urls["pdf_url"],
+                "md_url": asset_urls["md_url"],
+                "md_translated_url": asset_urls["md_translated_url"],
+                "images_base_url": asset_urls["images_base_url"],
             }
         )
 
@@ -215,3 +231,42 @@ async def api_pdf(request: Request) -> Response:
     if allowed_roots and not _ensure_under_roots(pdf_path, allowed_roots):
         return Response("Forbidden", status_code=403)
     return FileResponse(pdf_path)
+
+
+async def api_markdown(request: Request) -> Response:
+    """Dev-only API endpoint to serve raw markdown content."""
+    if request.app.state.static_mode != "dev":
+        return Response("Not Found", status_code=404)
+    index: PaperIndex = request.app.state.index
+    asset_config = request.app.state.asset_config
+    export_dir = request.app.state.static_export_dir
+    source_hash = request.path_params["source_hash"]
+    lang = request.query_params.get("lang")
+    md_path = None
+    if export_dir and asset_config and asset_config.enabled and (asset_config.base_url or "") == "":
+        if lang:
+            translated_url = asset_config.translated_md_urls.get(source_hash, {}).get(lang.lower())
+            if translated_url:
+                rel_path = translated_url.lstrip("/")
+                export_path = export_dir / rel_path
+                if export_path.exists():
+                    raw = _safe_read_text(export_path)
+                    return Response(raw, media_type="text/markdown")
+        else:
+            md_url = asset_config.md_urls.get(source_hash)
+            if md_url:
+                rel_path = md_url.lstrip("/")
+                export_path = export_dir / rel_path
+                if export_path.exists():
+                    raw = _safe_read_text(export_path)
+                    return Response(raw, media_type="text/markdown")
+    if lang:
+        md_path = index.translated_md_by_hash.get(source_hash, {}).get(lang.lower())
+    else:
+        md_path = index.md_path_by_hash.get(source_hash)
+    if not md_path:
+        return Response("Markdown not found", status_code=404)
+    raw = _safe_read_text(md_path)
+    if lang:
+        raw = normalize_markdown_images(raw)
+    return Response(raw, media_type="text/markdown")