deepresearch-flow 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

deepresearch_flow/paper/web/templates.py

@@ -0,0 +1,85 @@
+"""Jinja2 template utilities for paper web UI.
+
+This module provides Jinja2 environment setup and template rendering functions.
+Templates are stored in the 'templates' directory and use the PackageLoader
+for installed package compatibility.
+"""
+
+from __future__ import annotations
+
+from urllib.parse import quote
+
+from jinja2 import Environment, FileSystemLoader, PackageLoader
+
+from importlib import metadata
+
+from deepresearch_flow import __version__
+from deepresearch_flow.paper.web.constants import PDFJS_VIEWER_PATH, REPO_URL, TEMPLATES_DIR
+
+
+def get_jinja_env() -> Environment:
+    """Get a Jinja2 environment configured for web templates.
+
+    Uses PackageLoader for installed packages (works after pip install).
+    Falls back to FileSystemLoader for development mode.
+    """
+    try:
+        # Try PackageLoader first (works in installed package)
+        env = Environment(
+            loader=PackageLoader("deepresearch_flow.paper.web", "templates"),
+            autoescape=True,
+        )
+        return env
+    except Exception:
+        # Fallback to FileSystemLoader for development
+        env = Environment(
+            loader=FileSystemLoader(str(TEMPLATES_DIR)),
+            autoescape=True,
+        )
+        return env
+
+
+# Global Jinja2 environment
+_jinja_env = None
+
+
+def get_template_env() -> Environment:
+    """Get the shared Jinja2 environment for web handlers."""
+    global _jinja_env
+    if _jinja_env is None:
+        _jinja_env = get_jinja_env()
+    return _jinja_env
+
+
+def render_template(template_name: str, **context) -> str:
+    """Render a template with the given context.
+
+    Args:
+        template_name: Name of the template file (e.g., "detail.html")
+        **context: Key-value pairs to pass to the template
+
+    Returns:
+        Rendered HTML string
+    """
+    env = get_template_env()
+    try:
+        resolved_version = metadata.version("deepresearch-flow")
+    except metadata.PackageNotFoundError:
+        resolved_version = __version__
+    context.setdefault("app_version", resolved_version)
+    context.setdefault("repo_url", REPO_URL)
+    template = env.get_template(template_name)
+    return template.render(**context)
+
+
+def build_pdfjs_viewer_url(pdf_url: str) -> str:
+    """Build a PDF.js viewer URL for the given PDF URL.
+
+    Args:
+        pdf_url: The URL of the PDF file
+
+    Returns:
+        Full URL to the PDF.js viewer with the PDF file as a query parameter
+    """
+    encoded = quote(pdf_url, safe="")
+    return f"{PDFJS_VIEWER_PATH}?file={encoded}"

deepresearch_flow/paper/web/text.py

@@ -0,0 +1,68 @@
+"""Text normalization helpers for web rendering."""
+
+from __future__ import annotations
+
+import html
+import re
+
+_INLINE_FORMULA_RE = re.compile(r"<inline-formula[^>]*>.*?</inline-formula>", re.IGNORECASE | re.DOTALL)
+_TEX_MATH_RE = re.compile(r"<tex-math[^>]*>(.*?)</tex-math>", re.IGNORECASE | re.DOTALL)
+_TAG_RE = re.compile(r"<[^>]+>")
+_WS_RE = re.compile(r"\s+")
+_VENUE_BRACE_RE = re.compile(r"\{\{|\}\}")
+
+
+def normalize_title(raw: str) -> str:
+    """Normalize paper titles for display by stripping XML/HTML noise."""
+    if not raw:
+        return ""
+
+    def replace_inline(match: re.Match[str]) -> str:
+        block = match.group(0)
+        tex = _TEX_MATH_RE.search(block)
+        if tex:
+            return tex.group(1)
+        return ""
+
+    text = _INLINE_FORMULA_RE.sub(replace_inline, raw)
+    text = _TAG_RE.sub("", text)
+    text = html.unescape(text)
+    text = _WS_RE.sub(" ", text).strip()
+    return text
+
+
+def normalize_venue(raw: str) -> str:
+    """Normalize venue strings by removing extra BibTeX braces."""
+    if not raw:
+        return ""
+    text = _VENUE_BRACE_RE.sub("", raw)
+    text = _WS_RE.sub(" ", text).strip()
+    return text
+
+
+def extract_summary_snippet(paper: dict[str, object], max_len: int = 280) -> str:
+    """Extract a short summary snippet, preferring the 'simple' template."""
+    summary = ""
+    templates = paper.get("templates")
+    if isinstance(templates, dict):
+        simple = templates.get("simple")
+        if isinstance(simple, dict):
+            for key in ("summary", "abstract"):
+                value = simple.get(key)
+                if isinstance(value, str) and value.strip():
+                    summary = value.strip()
+                    break
+    if not summary:
+        for key in ("summary", "abstract"):
+            value = paper.get(key)
+            if isinstance(value, str) and value.strip():
+                summary = value.strip()
+                break
+    if not summary:
+        return ""
+    summary = _TAG_RE.sub("", summary)
+    summary = html.unescape(summary)
+    summary = _WS_RE.sub(" ", summary).strip()
+    if len(summary) > max_len:
+        return summary[: max_len - 1].rstrip() + "…"
+    return summary

{deepresearch_flow-0.3.0.dist-info → deepresearch_flow-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepresearch-flow
-Version: 0.3.0
+Version: 0.4.0
 Summary: Workflow tools for paper extraction, review, and research automation.
 Author-email: DengQi <dengqi935@gmail.com>
 License: MIT License
@@ -121,6 +121,7 @@ DeepResearch Flow provides a unified pipeline to **Repair**, **Translate**, **Ex
 - **Smart Extraction**: Turn unstructured Markdown into schema-enforced JSON (summaries, metadata, Q&A) using LLMs (OpenAI, Claude, Gemini, etc.).
 - **Precision Translation**: Translate OCR Markdown to Chinese/Japanese (`.zh.md`, `.ja.md`) while **freezing** formulas, code, tables, and references. No more broken layout.
 - **Local Knowledge DB**: A high-performance local Web UI to browse papers with **Split View** (Source vs. Translated vs. Summary), full-text search, and multi-dimensional filtering.
+- **Coverage Compare**: Compare JSON/PDF/Markdown/Translated datasets to find missing artifacts and export CSV reports.
 - **OCR Post-Processing**: Automatically fix broken references (`[1]` -> `[^1]`), merge split paragraphs, and standardize layouts.
 
 ---
@@ -246,7 +247,27 @@ uv run deepresearch-flow paper db serve \
 </details>
 
 <details>
-<summary><strong>4. Recognize: OCR Post-Processing</strong></summary>
+<summary><strong>4. Paper DB Compare: Coverage Audit</strong></summary>
+
+Compare two datasets (A/B) to find missing PDFs, markdowns, translations, or JSON items, with match metadata.
+
+```bash
+uv run deepresearch-flow paper db compare \
+  --input-a ./a.json \
+  --md-root-b ./md_root \
+  --output-csv ./compare.csv
+
+# Compare translated markdowns by language
+uv run deepresearch-flow paper db compare \
+  --md-translated-root-a ./translated_a \
+  --md-translated-root-b ./translated_b \
+  --lang zh
+```
+
+</details>
+
+<details>
+<summary><strong>5. Recognize: OCR Post-Processing</strong></summary>
 
 Tools to clean up raw outputs from OCR engines like MinerU.
 

{deepresearch_flow-0.3.0.dist-info → deepresearch_flow-0.4.0.dist-info}/RECORD

@@ -4,7 +4,8 @@ deepresearch_flow/cli.py,sha256=t4oowCNWldL0DrVJ4d0UlRkuGU2qHej_G0mAc_quteQ,455
 deepresearch_flow/paper/__init__.py,sha256=sunaOkcgAJBrfmcaJTumcWbPGVUSGWvOv2a2Yidzy0A,43
 deepresearch_flow/paper/cli.py,sha256=4UY3KHi6BUGztL1vB4w0cCMiIAo9KNxrfQn1GBHt6fA,11153
 deepresearch_flow/paper/config.py,sha256=totVBGzouh0KS6mhRNPneXZYPuuw0SHiOGdO3r6HSfc,9289
-deepresearch_flow/paper/db.py,sha256=ymVLzSEXDksdhLNSdvNA2IWLzT5lQOG1CpJlPU9CSQ8,33586
+deepresearch_flow/paper/db.py,sha256=i3v3n-YrG-kPpc62C9-InhEfInoZMBQd-r_pYz_fO_A,41847
+deepresearch_flow/paper/db_ops.py,sha256=l0lNPP1v00ZtdQb7ZAWE_tUf2JUzqKWxU1wwzlEjDrw,69766
 deepresearch_flow/paper/extract.py,sha256=ID1dd2r6LTB0kRF4qBSH6bGtBGv0znw--g_mXYBcoeU,32314
 deepresearch_flow/paper/llm.py,sha256=mHfs5IkT3Q6BOh46MDlfUmgVTX24WRf0IKKoOnN8nV8,4007
 deepresearch_flow/paper/prompts.py,sha256=mV7cEXw8pwukBUE4Trah0SjEPSSDgg5-RGaNaUdo4EU,519
@@ -40,8 +41,16 @@ deepresearch_flow/paper/templates/default_paper.md.j2,sha256=3azu48534QtLtHrCwI1
 deepresearch_flow/paper/templates/eight_questions.md.j2,sha256=Ecz4CD3nd7jZ4Dg8himZkTwF4WDkk0ILWk8V728uOPI,3038
 deepresearch_flow/paper/templates/three_pass.md.j2,sha256=ZRj-NkpZePnqp0gSE8OT1dN5Lr5RW4vdOYdeVejYJW0,1576
 deepresearch_flow/paper/web/__init__.py,sha256=eQBtBjvOYsNEdivHTI0aO286SCG2c86xI02tf-0jz5I,39
-deepresearch_flow/paper/web/app.py,sha256=nb4uzsDJ2R5dz_WA69NKwTgVgMqAyZv5OZ88GxFTWLQ,133311
+deepresearch_flow/paper/web/app.py,sha256=rXnQjffyzH5b64oCwv6ucihU_y5zaFbpzdEB5PRUvHc,3063
+deepresearch_flow/paper/web/constants.py,sha256=DzE1TO5Sd-1pfy8ww71J6LnS6cta_Je84jmXyxJ6DNI,1085
+deepresearch_flow/paper/web/filters.py,sha256=OVMB4GfigP9GPD5dXytHyeLYtnVXEK-QjYfA_k7QbaA,8315
+deepresearch_flow/paper/web/markdown.py,sha256=QHrxUYKB-uAZjG5jVGmkQ6EIT2dSxQNzlibgjGIIKuA,18888
 deepresearch_flow/paper/web/query.py,sha256=vTegfm5zGVkYCd6_K3yNrXJEmKMccUUFKG9DePPcKMw,1938
+deepresearch_flow/paper/web/templates.py,sha256=suJ67-nwWdExNVx8vvcInwqiHu6bhslaEFS1ouifLto,2515
+deepresearch_flow/paper/web/text.py,sha256=OiqOEzNepPXxcCIal38bxkUarIkcOXG6a30luxObFOI,2199
+deepresearch_flow/paper/web/handlers/__init__.py,sha256=HGQud4xuEtdB9eVYPzzilXV9ool-1Db5UU29WJ6cjNk,295
+deepresearch_flow/paper/web/handlers/api.py,sha256=Z7H0nr1cSIj1-nR6ZxhxtU6-4sjiuqzy1U1OpK56B0g,9014
+deepresearch_flow/paper/web/handlers/pages.py,sha256=euORL0_Avmqy-kOPKOfVQxyeQjLU4a6EBIufmwoLeCM,12247
 deepresearch_flow/paper/web/pdfjs/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
 deepresearch_flow/paper/web/pdfjs/build/pdf.js,sha256=2Ddm8gpMMfvOWinZh4nN--94GxR0QdpFvh0Qeejg-Bw,568294
 deepresearch_flow/paper/web/pdfjs/build/pdf.js.map,sha256=W0nwVFY4inhYxz1raDU6NZ6-rNA21FxLj13txVAqbm4,1434098
@@ -412,6 +421,15 @@ deepresearch_flow/paper/web/pdfjs/web/standard_fonts/LiberationSans-Bold.ttf,sha
 deepresearch_flow/paper/web/pdfjs/web/standard_fonts/LiberationSans-BoldItalic.ttf,sha256=oiQHWsF0la0KOvO8CkGawHBKiz_RCVRWIB-5sJX8KB0,135124
 deepresearch_flow/paper/web/pdfjs/web/standard_fonts/LiberationSans-Italic.ttf,sha256=gytEBtvvI2KIANOqrSEEhTSshNfjrZVb6DuBcu2O9RI,162036
 deepresearch_flow/paper/web/pdfjs/web/standard_fonts/LiberationSans-Regular.ttf,sha256=-Kzh-JKyvZ3BeSun8Jf6dYj4T-1IMhSA4E3lOQgoIh8,139512
+deepresearch_flow/paper/web/static/css/main.css,sha256=oUuEFEi4YP6bIlEQlIz-zQEQje7hRq3j63imvtJ6IQ4,15386
+deepresearch_flow/paper/web/static/js/detail.js,sha256=9bZmTID74otrZxJfHDJRMWuI_x1pgk71E3Zu2Q6sBIA,13368
+deepresearch_flow/paper/web/static/js/index.js,sha256=bbQz8QAewmu3TT8ImAzUqNtTWQCMKwVOQfU0Lkw6Lv0,10460
+deepresearch_flow/paper/web/static/js/outline.js,sha256=e9ydLcBqaTXOYULXt-1OKgKIzrZcZaH1RebPXWBbLvE,1882
+deepresearch_flow/paper/web/static/js/stats.js,sha256=USGIAx9cPQTMeyFwYu_bTYPJM7OoiqimhCYuAjoP0-s,1420
+deepresearch_flow/paper/web/templates/base.html,sha256=4gWJLvjOuDSnBYRpJqxhGKmKC6UuOl19q_Q_cOjhL-g,1806
+deepresearch_flow/paper/web/templates/detail.html,sha256=jM7rkMu1rQs-kx7LHfLwUvv4yJlhua4eE-wZoOJPufA,16332
+deepresearch_flow/paper/web/templates/index.html,sha256=eQJgjr-RuKPcDxdCJG_hM6KvlMYIp-OSG0oQEJHO7Is,6117
+deepresearch_flow/paper/web/templates/stats.html,sha256=bcQBawoZ9KoRkM0NNo9WJBVeN_8O1WU2xNiye-Fugyo,671
 deepresearch_flow/recognize/__init__.py,sha256=yMAqbdCzpdRSiwFhq9j7yx9ZWxqz_Zq3vfYlTLFCWek,33
 deepresearch_flow/recognize/cli.py,sha256=zWUsqvou2h6c5zR_myGaySvK6cG9ItJp9cJFtqqJk7Y,21597
 deepresearch_flow/recognize/markdown.py,sha256=y-PMJbGqrfWCNBVGanXK1M4OuMP9e1eqh7HDYye5a7Q,8757
@@ -425,9 +443,9 @@ deepresearch_flow/translator/placeholder.py,sha256=mEgqA-dPdOsIhno0h_hzfpXpY2asb
 deepresearch_flow/translator/prompts.py,sha256=kl_9O2YvmtXC1w6WLnsLuVZKz4mcOtUF887SiTaOvc0,4754
 deepresearch_flow/translator/protector.py,sha256=sXwNJ1Y8tyPm7dgm8-7S8HkcPe23TGsBdwRxH6mKL70,11291
 deepresearch_flow/translator/segment.py,sha256=rBFMCLTrvm2GrPc_hNFymi-8Ih2DAtUQlZHCRE9nLaM,5146
-deepresearch_flow-0.3.0.dist-info/licenses/LICENSE,sha256=hT8F2Py1pe6flxq3Ufdm2UKFk0B8CBm0aAQfsLXfvjw,1063
-deepresearch_flow-0.3.0.dist-info/METADATA,sha256=AJ4RfKW-V9BPhrrlFSP8stAoXG4SwpF-AvZH5HEtWyw,10831
-deepresearch_flow-0.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deepresearch_flow-0.3.0.dist-info/entry_points.txt,sha256=1uIKscs0YRMg_mFsg9NjsaTt4CvQqQ_-zGERUKhhL_Y,65
-deepresearch_flow-0.3.0.dist-info/top_level.txt,sha256=qBl4RvPJNJUbL8CFfMNWxY0HpQLx5RlF_ko-z_aKpm0,18
-deepresearch_flow-0.3.0.dist-info/RECORD,,
+deepresearch_flow-0.4.0.dist-info/licenses/LICENSE,sha256=hT8F2Py1pe6flxq3Ufdm2UKFk0B8CBm0aAQfsLXfvjw,1063
+deepresearch_flow-0.4.0.dist-info/METADATA,sha256=FKueIvCHzloXlQk71dliJK29rxEC9tyMOMS7ISXdfAY,11476
+deepresearch_flow-0.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deepresearch_flow-0.4.0.dist-info/entry_points.txt,sha256=1uIKscs0YRMg_mFsg9NjsaTt4CvQqQ_-zGERUKhhL_Y,65
+deepresearch_flow-0.4.0.dist-info/top_level.txt,sha256=qBl4RvPJNJUbL8CFfMNWxY0HpQLx5RlF_ko-z_aKpm0,18
+deepresearch_flow-0.4.0.dist-info/RECORD,,