deepresearch-flow 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

deepresearch_flow/paper/web/static/js/outline.js

@@ -1,58 +1,72 @@
 /* Outline functionality extracted from outline_assets */
 (function() {
-  function initOutline() {
-    const content = document.getElementById('content');
-    if (!content) return;
+  var bound = false;
 
-    const headings = content.querySelectorAll('h2, h3, h4');
-    if (headings.length === 0) return;
+  function buildOutline() {
+    var content = document.getElementById('content');
+    if (!content) return;
 
-    const outline = document.getElementById('outline');
-    const toggle = document.getElementById('outlineToggle');
-    const close = document.getElementById('outlineClose');
-    const outlineContent = document.getElementById('outlineContent');
+    var headings = content.querySelectorAll('h2, h3, h4');
+    var outline = document.getElementById('outline');
+    var toggle = document.getElementById('outlineToggle');
+    var close = document.getElementById('outlineClose');
+    var outlineContent = document.getElementById('outlineContent');
 
     if (!outline || !toggle || !close || !outlineContent) return;
 
-    for (let i = 0; i < headings.length; i++) {
-      const h = headings[i];
+    outlineContent.innerHTML = '';
+    if (headings.length === 0) return;
+    for (var i = 0; i < headings.length; i++) {
+      var h = headings[i];
       if (!h.id) h.id = 'heading-' + i;
-      const a = document.createElement('a');
+      var a = document.createElement('a');
       a.href = '#' + h.id;
       a.textContent = h.textContent.trim();
       a.className = 'outline-' + h.tagName.toLowerCase();
       outlineContent.appendChild(a);
     }
 
-    toggle.addEventListener('click', function() {
-      outline.style.display = 'block';
-      toggle.style.display = 'none';
-    });
-
-    close.addEventListener('click', function() {
-      outline.style.display = 'none';
-      toggle.style.display = 'block';
-    });
-
-    const savedState = sessionStorage.getItem('outlineVisible');
-    if (savedState === 'true') {
-      outline.style.display = 'block';
-      toggle.style.display = 'none';
-    } else {
-      toggle.style.display = 'block';
+    if (!bound) {
+      toggle.addEventListener('click', function() {
+        outline.style.display = 'block';
+        toggle.style.display = 'none';
+        document.body.classList.add('outline-open');
+      });
+
+      close.addEventListener('click', function() {
+        outline.style.display = 'none';
+        toggle.style.display = 'block';
+        document.body.classList.remove('outline-open');
+      });
+
+      var savedState = sessionStorage.getItem('outlineVisible');
+      if (savedState === 'true') {
+        outline.style.display = 'block';
+        toggle.style.display = 'none';
+        document.body.classList.add('outline-open');
+      } else {
+        toggle.style.display = 'block';
+        document.body.classList.remove('outline-open');
+      }
+
+      var observer = new MutationObserver(function() {
+        var isVisible = outline.style.display === 'block';
+        sessionStorage.setItem('outlineVisible', String(isVisible));
+      });
+      observer.observe(outline, { attributes: true, attributeFilter: ['style'] });
+      bound = true;
     }
+  }
 
-    const observer = new MutationObserver(function() {
-      const isVisible = outline.style.display === 'block';
-      sessionStorage.setItem('outlineVisible', String(isVisible));
-    });
-    observer.observe(outline, { attributes: true, attributeFilter: ['style'] });
+  function initOutline() {
+    buildOutline();
   }
 
-  // Run when DOM is ready
   if (document.readyState === 'loading') {
     document.addEventListener('DOMContentLoaded', initOutline);
   } else {
     initOutline();
   }
+
+  document.addEventListener('content:updated', initOutline);
 })();

deepresearch_flow/paper/web/static_assets.py

@@ -0,0 +1,289 @@
+"""Static asset export and URL mapping for paper web UI."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import base64
+import hashlib
+import mimetypes
+import re
+from pathlib import Path
+from typing import Any
+
+from deepresearch_flow.paper.db_ops import PaperIndex
+
+_IMAGE_PATTERN = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
+_DATA_URL_PATTERN = re.compile(r"^data:([^;,]+)(;base64)?,(.*)$", re.DOTALL)
+_IMG_TAG_PATTERN = re.compile(r"<img\b[^>]*>", re.IGNORECASE)
+_SRC_ATTR_PATTERN = re.compile(r"\bsrc\s*=\s*(\"[^\"]*\"|'[^']*'|[^\s>]+)", re.IGNORECASE | re.DOTALL)
+
+_EXTENSION_OVERRIDES = {
+    ".jpe": ".jpg",
+}
+
+
+@dataclass(frozen=True)
+class StaticAssetConfig:
+    enabled: bool
+    base_url: str | None
+    images_base_url: str | None
+    pdf_urls: dict[str, str]
+    md_urls: dict[str, str]
+    translated_md_urls: dict[str, dict[str, str]]
+
+
+def _normalize_base_url(value: str) -> str:
+    return value.rstrip("/")
+
+
+def _extension_from_mime(mime: str) -> str | None:
+    ext = mimetypes.guess_extension(mime, strict=False)
+    if ext in _EXTENSION_OVERRIDES:
+        return _EXTENSION_OVERRIDES[ext]
+    return ext
+
+
+def _parse_data_url(target: str) -> tuple[str, bytes] | None:
+    match = _DATA_URL_PATTERN.match(target)
+    if not match:
+        return None
+    mime = match.group(1) or ""
+    if not mime.startswith("image/"):
+        return None
+    if match.group(2) != ";base64":
+        return None
+    payload = match.group(3) or ""
+    try:
+        return mime, base64.b64decode(payload)
+    except Exception:
+        return None
+
+
+def _hash_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _hash_text(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8", errors="ignore")).hexdigest()
+
+
+def _hash_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _split_link_target(raw_link: str) -> tuple[str, str, str, str]:
+    link = raw_link.strip()
+    if link.startswith("<"):
+        end = link.find(">")
+        if end != -1:
+            return link[1:end], link[end + 1 :], "<", ">"
+    parts = link.split()
+    if not parts:
+        return "", "", "", ""
+    target = parts[0]
+    suffix = link[len(target) :]
+    return target, suffix, "", ""
+
+
+class _ImageStore:
+    def __init__(self, output_dir: Path | None) -> None:
+        self.output_dir = output_dir
+        self._written: set[str] = set()
+        if output_dir:
+            output_dir.mkdir(parents=True, exist_ok=True)
+
+    def add_image(self, mime: str, data: bytes) -> str | None:
+        ext = _extension_from_mime(mime)
+        if not ext:
+            return None
+        digest = _hash_bytes(data)
+        filename = f"{digest}{ext}"
+        if self.output_dir and filename not in self._written:
+            dest = self.output_dir / filename
+            if not dest.exists():
+                dest.write_bytes(data)
+            self._written.add(filename)
+        return f"images/{filename}"
+
+
+def _rewrite_markdown_images(text: str, store: _ImageStore) -> str:
+    def replace_md(match: re.Match[str]) -> str:
+        alt_text = match.group(1)
+        raw_link = match.group(2)
+        target, suffix, prefix, postfix = _split_link_target(raw_link)
+        parsed = _parse_data_url(target)
+        if parsed is None:
+            return match.group(0)
+        mime, data = parsed
+        replacement = store.add_image(mime, data)
+        if not replacement:
+            return match.group(0)
+        new_link = f"{prefix}{replacement}{postfix}{suffix}"
+        return f"![{alt_text}]({new_link})"
+
+    text = _IMAGE_PATTERN.sub(replace_md, text)
+
+    def replace_img(match: re.Match[str]) -> str:
+        tag = match.group(0)
+        src_match = _SRC_ATTR_PATTERN.search(tag)
+        if not src_match:
+            return tag
+        raw_value = src_match.group(1)
+        quote = ""
+        if raw_value and raw_value[0] in {"\"", "'"}:
+            quote = raw_value[0]
+            value = raw_value[1:-1]
+        else:
+            value = raw_value
+        parsed = _parse_data_url(value)
+        if parsed is None:
+            return tag
+        mime, data = parsed
+        replacement = store.add_image(mime, data)
+        if not replacement:
+            return tag
+        new_src = f"{quote}{replacement}{quote}" if quote else replacement
+        return tag[: src_match.start(1)] + new_src + tag[src_match.end(1) :]
+
+    return _IMG_TAG_PATTERN.sub(replace_img, text)
+
+
+def _safe_read_text(path: Path) -> str:
+    try:
+        return path.read_text(encoding="utf-8")
+    except UnicodeDecodeError:
+        return path.read_text(encoding="latin-1")
+
+
+def build_static_assets(
+    index: PaperIndex,
+    *,
+    static_base_url: str | None,
+    static_export_dir: Path | None = None,
+    allow_empty_base: bool = False,
+) -> StaticAssetConfig:
+    if static_base_url is None:
+        if not allow_empty_base:
+            return StaticAssetConfig(
+                enabled=False,
+                base_url=None,
+                images_base_url=None,
+                pdf_urls={},
+                md_urls={},
+                translated_md_urls={},
+            )
+        base_url = ""
+    else:
+        base_url = _normalize_base_url(static_base_url)
+        if not base_url and not allow_empty_base:
+            return StaticAssetConfig(
+                enabled=False,
+                base_url=None,
+                images_base_url=None,
+                pdf_urls={},
+                md_urls={},
+                translated_md_urls={},
+            )
+
+    images_base_url = f"{base_url}/images"
+
+    pdf_urls: dict[str, str] = {}
+    md_urls: dict[str, str] = {}
+    translated_md_urls: dict[str, dict[str, str]] = {}
+
+    images_dir = static_export_dir / "images" if static_export_dir else None
+    md_dir = static_export_dir / "md" if static_export_dir else None
+    md_translate_dir = static_export_dir / "md_translate" if static_export_dir else None
+    pdf_dir = static_export_dir / "pdf" if static_export_dir else None
+
+    store = _ImageStore(images_dir)
+
+    if md_dir:
+        md_dir.mkdir(parents=True, exist_ok=True)
+    if md_translate_dir:
+        md_translate_dir.mkdir(parents=True, exist_ok=True)
+    if pdf_dir:
+        pdf_dir.mkdir(parents=True, exist_ok=True)
+
+    for source_hash, md_path in index.md_path_by_hash.items():
+        raw = _safe_read_text(md_path)
+        rewritten = _rewrite_markdown_images(raw, store)
+        md_hash = _hash_text(rewritten)
+        md_urls[source_hash] = f"{base_url}/md/{md_hash}.md"
+        if md_dir:
+            target = md_dir / f"{md_hash}.md"
+            if not target.exists():
+                target.write_text(rewritten, encoding="utf-8")
+
+    for source_hash, translations in index.translated_md_by_hash.items():
+        translated_md_urls[source_hash] = {}
+        for lang, md_path in translations.items():
+            raw = _safe_read_text(md_path)
+            rewritten = _rewrite_markdown_images(raw, store)
+            md_hash = _hash_text(rewritten)
+            translated_md_urls[source_hash][lang] = f"{base_url}/md_translate/{lang}/{md_hash}.md"
+            if md_translate_dir:
+                lang_dir = md_translate_dir / lang
+                lang_dir.mkdir(parents=True, exist_ok=True)
+                target = lang_dir / f"{md_hash}.md"
+                if not target.exists():
+                    target.write_text(rewritten, encoding="utf-8")
+
+    for source_hash, pdf_path in index.pdf_path_by_hash.items():
+        pdf_hash = _hash_file(pdf_path)
+        pdf_urls[source_hash] = f"{base_url}/pdf/{pdf_hash}.pdf"
+        if pdf_dir:
+            target = pdf_dir / f"{pdf_hash}.pdf"
+            if not target.exists():
+                target.write_bytes(pdf_path.read_bytes())
+
+    return StaticAssetConfig(
+        enabled=True,
+        base_url=base_url,
+        images_base_url=images_base_url,
+        pdf_urls=pdf_urls,
+        md_urls=md_urls,
+        translated_md_urls=translated_md_urls,
+    )
+
+
+def resolve_asset_urls(
+    index: PaperIndex,
+    source_hash: str,
+    asset_config: StaticAssetConfig | None,
+    *,
+    prefer_local: bool = False,
+) -> dict[str, Any]:
+    """Resolve asset URLs for a paper based on static asset config or local endpoints."""
+    if prefer_local:
+        translations = index.translated_md_by_hash.get(source_hash, {})
+        images_base_url = asset_config.images_base_url if asset_config and asset_config.enabled else None
+        return {
+            "pdf_url": f"/api/pdf/{source_hash}" if source_hash in index.pdf_path_by_hash else None,
+            "md_url": f"/api/dev/markdown/{source_hash}" if source_hash in index.md_path_by_hash else None,
+            "md_translated_url": {
+                lang: f"/api/dev/markdown/{source_hash}?lang={lang}" for lang in translations
+            },
+            "images_base_url": images_base_url,
+        }
+    if asset_config and asset_config.enabled:
+        return {
+            "pdf_url": asset_config.pdf_urls.get(source_hash),
+            "md_url": asset_config.md_urls.get(source_hash),
+            "md_translated_url": asset_config.translated_md_urls.get(source_hash, {}),
+            "images_base_url": asset_config.images_base_url,
+        }
+
+    translations = index.translated_md_by_hash.get(source_hash, {})
+    return {
+        "pdf_url": f"/api/pdf/{source_hash}" if source_hash in index.pdf_path_by_hash else None,
+        "md_url": f"/api/dev/markdown/{source_hash}" if source_hash in index.md_path_by_hash else None,
+        "md_translated_url": {
+            lang: f"/api/dev/markdown/{source_hash}?lang={lang}" for lang in translations
+        },
+        "images_base_url": None,
+    }

deepresearch_flow/paper/web/templates/detail.html

@@ -1,7 +1,7 @@
 {% extends "base.html" %}
 
 {% block extra_head %}
-<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/katex.min.css" />
+<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.16.27/dist/katex.min.css" />
 <style>
   #content img {
     max-width: 100%;
@@ -162,9 +162,10 @@
           <button id="outlineToggle" class="inline-flex h-9 items-center justify-center rounded-md bg-slate-900 px-3 text-sm text-white shadow hover:bg-slate-800">☰ Outline</button>
         </div>
         {% endif %}
-        <article id="content" class="content">{{ body_html|safe }}</article>
+        <article id="content" class="content"{% if source_markdown_url %}{% if body_html %} data-raw-markdown-url="{{ source_markdown_url }}"{% else %} data-markdown-url="{{ source_markdown_url }}"{% endif %}{% endif %}{% if images_base_url %} data-images-base-url="{{ images_base_url }}"{% endif %}>{{ body_html|safe }}</article>
+        <div id="markdownStatus" class="muted" style="margin-top:10px;">{% if body_html %}Loading raw markdown...{% else %}Loading markdown...{% endif %}</div>
         <details style="margin-top:12px;"><summary>Raw markdown</summary>
-          <pre><code>{{ raw_content|escape }}</code></pre>
+          <pre><code id="rawMarkdown"></code></pre>
         </details>
       {% endif %}
 
@@ -200,9 +201,10 @@
           <button id="outlineToggle" class="inline-flex h-9 items-center justify-center rounded-md bg-slate-900 px-3 text-sm text-white shadow hover:bg-slate-800">☰ Outline</button>
         </div>
         {% endif %}
-        <article id="content" class="content">{{ body_html|safe }}</article>
+        <article id="content" class="content"{% if translated_markdown_url %}{% if body_html %} data-raw-markdown-url="{{ translated_markdown_url }}"{% else %} data-markdown-url="{{ translated_markdown_url }}"{% endif %}{% endif %}{% if images_base_url %} data-images-base-url="{{ images_base_url }}"{% endif %}>{{ body_html|safe }}</article>
+        <div id="markdownStatus" class="muted" style="margin-top:10px;">{% if body_html %}Loading raw markdown...{% else %}Loading markdown...{% endif %}</div>
         <details style="margin-top:12px;"><summary>Raw markdown</summary>
-          <pre><code>{{ raw_content|escape }}</code></pre>
+          <pre><code id="rawMarkdown"></code></pre>
         </details>
       {% endif %}
 
@@ -251,82 +253,57 @@
 
 {% block extra_scripts %}
 {# CDN scripts for markdown rendering #}
-<script src="https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/katex.min.js"></script>
-<script src="https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/contrib/auto-render.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/katex@0.16.27/dist/katex.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/katex@0.16.27/dist/contrib/auto-render.min.js"></script>
 <script src="https://cdn.jsdelivr.net/npm/mermaid@10.6.1/dist/mermaid.min.js"></script>
 <script src="https://cdn.jsdelivr.net/npm/markmap-lib@0.15.4/dist/browser/index.min.js"></script>
-<script src="https://cdn.jsdelivr.net/npm/markmap-view@0.15.4/dist/index.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/markmap-view@0.15.4/dist/browser/index.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/marked@12.0.1/marked.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/dompurify@3.0.6/dist/purify.min.js"></script>
 
 {# PDF.js for PDF view #}
 {% if current_view == 'pdf' %}
-<script src="https://cdnjs.cloudflare.com/ajax/libs/pdf.js/4.0.379/pdf.min.js"></script>
-{% endif %}
-
-{# Main detail JavaScript #}
-<script src="/static/js/detail.js"></script>
-
-{# Outline functionality #}
-{% if show_outline %}
 <script>
 (function() {
-  function initOutline() {
-    var content = document.getElementById('content');
-    if (!content) return;
-    var headings = content.querySelectorAll('h2, h3, h4');
-    if (headings.length === 0) return;
-
-    var outline = document.getElementById('outline');
-    var toggle = document.getElementById('outlineToggle');
-    var close = document.getElementById('outlineClose');
-    var outlineContent = document.getElementById('outlineContent');
-
-    if (!outline || !toggle || !close || !outlineContent) return;
-
-    for (var i = 0; i < headings.length; i++) {
-      var h = headings[i];
-      if (!h.id) h.id = 'heading-' + i;
-      var a = document.createElement('a');
-      a.href = '#' + h.id;
-      a.textContent = h.textContent.trim();
-      a.className = 'outline-' + h.tagName.toLowerCase();
-      outlineContent.appendChild(a);
-    }
-
-    toggle.addEventListener('click', function() {
-      outline.style.display = 'block';
-      toggle.style.display = 'none';
-      document.body.classList.add('outline-open');
-    });
-
-    close.addEventListener('click', function() {
-      outline.style.display = 'none';
-      toggle.style.display = 'block';
-      document.body.classList.remove('outline-open');
-    });
-
-    var savedState = sessionStorage.getItem('outlineVisible');
-    if (savedState === 'true') {
-      outline.style.display = 'block';
-      toggle.style.display = 'none';
-      document.body.classList.add('outline-open');
-    } else {
-      toggle.style.display = 'block';
-      document.body.classList.remove('outline-open');
+  var primary = {{ pdfjs_script_url|tojson }};
+  var fallback = "/pdfjs/build/pdf.js";
+  if (!primary) {
+    primary = fallback;
+    fallback = "";
+  }
+  window.PDFJS_SCRIPT_SRC = primary;
+  window.PDFJS_WORKER_SRC = {{ pdfjs_worker_url|tojson }} || "/pdfjs/build/pdf.worker.js";
+  function notifyLoaded() {
+    try {
+      window.dispatchEvent(new Event('pdfjs:loaded'));
+    } catch (err) {
+      var event = document.createEvent('Event');
+      event.initEvent('pdfjs:loaded', true, true);
+      window.dispatchEvent(event);
     }
-
-    var observer = new MutationObserver(function() {
-      var isVisible = outline.style.display === 'block';
-      sessionStorage.setItem('outlineVisible', String(isVisible));
-    });
-    observer.observe(outline, { attributes: true, attributeFilter: ['style'] });
   }
-
-  if (document.readyState === 'loading') {
-    document.addEventListener('DOMContentLoaded', initOutline);
+  function loadScript(src, onerror) {
+    var script = document.createElement('script');
+    script.src = src;
+    script.defer = true;
+    script.onload = notifyLoaded;
+    if (onerror) script.onerror = onerror;
+    document.head.appendChild(script);
+  }
+  if (fallback && primary !== fallback) {
+    loadScript(primary, function() { loadScript(fallback); });
   } else {
-    initOutline();
+    loadScript(primary);
   }
 })();
 </script>
 {% endif %}
+
+{# Main detail JavaScript #}
+<script src="/static/js/detail.js"></script>
+
+{# Outline functionality #}
+{% if show_outline %}
+<script src="/static/js/outline.js"></script>
+{% endif %}
 {% endblock %}

deepresearch_flow/paper/web/templates/index.html

@@ -1,7 +1,7 @@
 {% extends "base.html" %}
 
 {% block extra_head %}
-<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/katex.min.css" />
+<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.16.27/dist/katex.min.css" />
 <style>
   #content img {
     max-width: 100%;
@@ -108,7 +108,7 @@
 {% endblock %}
 
 {% block extra_scripts %}
-<script src="https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/katex.min.js"></script>
-<script src="https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/contrib/auto-render.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/katex@0.16.27/dist/katex.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/katex@0.16.27/dist/contrib/auto-render.min.js"></script>
 <script src="/static/js/index.js"></script>
 {% endblock %}

deepresearch_flow/paper/web/templates.py

@@ -7,7 +7,7 @@ for installed package compatibility.
 
 from __future__ import annotations
 
-from urllib.parse import quote
+from urllib.parse import urlencode
 
 from jinja2 import Environment, FileSystemLoader, PackageLoader
 
@@ -72,14 +72,17 @@ def render_template(template_name: str, **context) -> str:
     return template.render(**context)
 
 
-def build_pdfjs_viewer_url(pdf_url: str) -> str:
+def build_pdfjs_viewer_url(pdf_url: str, *, cdn_base_url: str | None = None) -> str:
     """Build a PDF.js viewer URL for the given PDF URL.
 
     Args:
         pdf_url: The URL of the PDF file
+        cdn_base_url: Optional CDN base URL for PDF.js assets
 
     Returns:
         Full URL to the PDF.js viewer with the PDF file as a query parameter
     """
-    encoded = quote(pdf_url, safe="")
-    return f"{PDFJS_VIEWER_PATH}?file={encoded}"
+    params = {"file": pdf_url, "allow_origin": "1"}
+    if cdn_base_url:
+        params["cdn"] = cdn_base_url.rstrip("/")
+    return f"{PDFJS_VIEWER_PATH}?{urlencode(params)}"