deepresearch-flow 0.1.1__tar.gz → 0.2.0__tar.gz

{deepresearch_flow-0.1.1/python/deepresearch_flow.egg-info → deepresearch_flow-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepresearch-flow
-Version: 0.1.1
+Version: 0.2.0
 Summary: Workflow tools for paper extraction, review, and research automation.
 Author-email: DengQi <dengqi935@gmail.com>
 License: MIT License
@@ -49,6 +49,7 @@ Requires-Dist: jinja2>=3.1.3
 Requires-Dist: json-repair>=0.31.0
 Requires-Dist: jsonschema>=4.21.1
 Requires-Dist: markdown-it-py>=3.0.0
+Requires-Dist: pypdf>=3.0.0
 Requires-Dist: pybtex>=0.24.0
 Requires-Dist: rich>=13.7.1
 Requires-Dist: starlette>=0.37.2
@@ -152,6 +153,7 @@ Incremental behavior:
 - Use `--retry-failed` to retry only failed documents listed in `paper_errors.json`.
 - Use `--verbose` for detailed logs alongside progress bars.
 - Extract-time rendering defaults to the same built-in template as `--prompt-template`.
+- Output JSON is written as `{"template_tag": "...", "papers": [...]}`.
 - A summary table prints input/prompt/output character totals, token estimates, and throughput after each run.
 - Progress bars include a live prompt/completion/total token ticker.
 
@@ -207,18 +209,23 @@ Render outputs, compute stats, and serve a local web UI over paper JSON.
 
 JSON input formats:
 
-- For `db render-md`, `db statistics`, `db filter`, and `db generate-tags`, the input is the aggregated JSON list.
+- For `db render-md`, `db statistics`, `db filter`, and `db generate-tags`, the input can be either an aggregated JSON list or `{"template_tag": "...", "papers": [...]}` (the commands operate on `papers`).
 - For `db serve`, each input JSON must be an object: `{"template_tag": "simple", "papers": [...]}`.
-  When `template_tag` is missing, the server attempts to infer it as a fallback.
+  When `template_tag` is missing, the server attempts to infer it as a fallback (legacy list-only inputs are rejected).
 
 Web UI highlights:
 
 - Summary/Source/PDF/PDF Viewer views with tab navigation.
 - Split view: choose left/right panes independently (summary/source/pdf/pdf viewer) via URL params.
-- Summary view includes a collapsible outline panel (top-left) and a back-to-top control (bottom-left).
+- Summary/Source views include a collapsible outline panel (top-left) and a back-to-top control (bottom-left).
 - Summary template dropdown shows only available templates per paper.
-- Source view renders Markdown and supports embedded HTML tables plus `data:image/...;base64` `<img>` tags.
+- Homepage filters: PDF/Source/Summary availability and template tags, plus a filter syntax input (`tmpl:...`, `has:pdf`, `no:source`).
+- Homepage stats: total and filtered counts for PDF/Source/Summary plus per-template totals.
+- Stats page includes keyword frequency charts.
+- Source view renders Markdown and supports embedded HTML tables plus `data:image/...;base64` `<img>` tags (images are constrained to the content width).
 - PDF Viewer is served locally (PDF.js viewer assets) to avoid cross-origin issues with local PDFs.
+- PDF-only entries are surfaced for unmatched PDFs under `--pdf-root` (metadata title if available, otherwise filename), with badges and detail warnings.
+- PDF-only entries are excluded from stats counts.
 - Merge behavior for multi-input serve: title similarity (>= 0.95), preferring `bibtex.fields.title` and falling back to `paper_title`.
 - Cache merged inputs with `--cache-dir`; bypass with `--no-cache`.
 
@@ -250,6 +257,7 @@ deepresearch-flow paper db filter \
 deepresearch-flow paper db statistics \
   --input paper_infos.json \
   --top-n 20
+# Statistics also include keyword frequency (normalized to lowercase)
 
 # Serve a local read-only web UI (loads charts/libs via CDN)
 deepresearch-flow paper db serve \
@@ -278,7 +286,7 @@ Web search syntax (Scholar-style):
 - Quoted phrases: `title:"nearest neighbor"`
 - OR: `fpga OR asic`
 - Negation: `-survey` or `-tag:survey`
-- Fields: `title:`, `author:`, `tag:`, `venue:`, `year:`, `month:`
+- Fields: `title:`, `author:`, `tag:`, `venue:`, `year:`, `month:` (content tags only)
 - Year range: `year:2020..2024`
 
 Other database helpers:

{deepresearch_flow-0.1.1 → deepresearch_flow-0.2.0}/README.md

@@ -94,6 +94,7 @@ Incremental behavior:
 - Use `--retry-failed` to retry only failed documents listed in `paper_errors.json`.
 - Use `--verbose` for detailed logs alongside progress bars.
 - Extract-time rendering defaults to the same built-in template as `--prompt-template`.
+- Output JSON is written as `{"template_tag": "...", "papers": [...]}`.
 - A summary table prints input/prompt/output character totals, token estimates, and throughput after each run.
 - Progress bars include a live prompt/completion/total token ticker.
 
@@ -149,18 +150,23 @@ Render outputs, compute stats, and serve a local web UI over paper JSON.
 
 JSON input formats:
 
-- For `db render-md`, `db statistics`, `db filter`, and `db generate-tags`, the input is the aggregated JSON list.
+- For `db render-md`, `db statistics`, `db filter`, and `db generate-tags`, the input can be either an aggregated JSON list or `{"template_tag": "...", "papers": [...]}` (the commands operate on `papers`).
 - For `db serve`, each input JSON must be an object: `{"template_tag": "simple", "papers": [...]}`.
-  When `template_tag` is missing, the server attempts to infer it as a fallback.
+  When `template_tag` is missing, the server attempts to infer it as a fallback (legacy list-only inputs are rejected).
 
 Web UI highlights:
 
 - Summary/Source/PDF/PDF Viewer views with tab navigation.
 - Split view: choose left/right panes independently (summary/source/pdf/pdf viewer) via URL params.
-- Summary view includes a collapsible outline panel (top-left) and a back-to-top control (bottom-left).
+- Summary/Source views include a collapsible outline panel (top-left) and a back-to-top control (bottom-left).
 - Summary template dropdown shows only available templates per paper.
-- Source view renders Markdown and supports embedded HTML tables plus `data:image/...;base64` `<img>` tags.
+- Homepage filters: PDF/Source/Summary availability and template tags, plus a filter syntax input (`tmpl:...`, `has:pdf`, `no:source`).
+- Homepage stats: total and filtered counts for PDF/Source/Summary plus per-template totals.
+- Stats page includes keyword frequency charts.
+- Source view renders Markdown and supports embedded HTML tables plus `data:image/...;base64` `<img>` tags (images are constrained to the content width).
 - PDF Viewer is served locally (PDF.js viewer assets) to avoid cross-origin issues with local PDFs.
+- PDF-only entries are surfaced for unmatched PDFs under `--pdf-root` (metadata title if available, otherwise filename), with badges and detail warnings.
+- PDF-only entries are excluded from stats counts.
 - Merge behavior for multi-input serve: title similarity (>= 0.95), preferring `bibtex.fields.title` and falling back to `paper_title`.
 - Cache merged inputs with `--cache-dir`; bypass with `--no-cache`.
 
@@ -192,6 +198,7 @@ deepresearch-flow paper db filter \
 deepresearch-flow paper db statistics \
   --input paper_infos.json \
   --top-n 20
+# Statistics also include keyword frequency (normalized to lowercase)
 
 # Serve a local read-only web UI (loads charts/libs via CDN)
 deepresearch-flow paper db serve \
@@ -220,7 +227,7 @@ Web search syntax (Scholar-style):
 - Quoted phrases: `title:"nearest neighbor"`
 - OR: `fpga OR asic`
 - Negation: `-survey` or `-tag:survey`
-- Fields: `title:`, `author:`, `tag:`, `venue:`, `year:`, `month:`
+- Fields: `title:`, `author:`, `tag:`, `venue:`, `year:`, `month:` (content tags only)
 - Year range: `year:2020..2024`
 
 Other database helpers:

{deepresearch_flow-0.1.1 → deepresearch_flow-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "deepresearch-flow"
-version = "0.1.1"
+version = "0.2.0"
 description = "Workflow tools for paper extraction, review, and research automation."
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE" }
@@ -18,6 +18,7 @@ dependencies = [
     "json-repair>=0.31.0",
     "jsonschema>=4.21.1",
     "markdown-it-py>=3.0.0",
+    "pypdf>=3.0.0",
     "pybtex>=0.24.0",
     "rich>=13.7.1",
     "starlette>=0.37.2",

deepresearch_flow-0.2.0/python/deepresearch_flow/__main__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from deepresearch_flow.cli import main
+
+
+if __name__ == "__main__":
+    main()

{deepresearch_flow-0.1.1 → deepresearch_flow-0.2.0}/python/deepresearch_flow/paper/db.py

@@ -30,7 +30,12 @@ except ImportError:
 
 
 def load_json(path: Path) -> list[dict[str, Any]]:
-    return json.loads(path.read_text(encoding="utf-8"))
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if isinstance(data, dict) and isinstance(data.get("papers"), list):
+        return data["papers"]
+    if isinstance(data, list):
+        return data
+    raise click.ClickException("Input JSON must be a list or {template_tag, papers}")
 
 
 def write_json(path: Path, data: Any) -> None:
@@ -378,9 +383,25 @@ def register_db_commands(db_group: click.Group) -> None:
         month_counts: dict[str, int] = {}
         author_counts: dict[str, int] = {}
         tag_counts: dict[str, int] = {}
+        keyword_counts: dict[str, int] = {}
         journal_counts: dict[str, int] = {}
         conference_counts: dict[str, int] = {}
         other_venue_counts: dict[str, int] = {}
+        def normalize_keywords(value: Any) -> list[str]:
+            if value is None:
+                return []
+            if isinstance(value, list):
+                items = value
+            elif isinstance(value, str):
+                items = re.split(r"[;,]", value)
+            else:
+                items = [value]
+            normalized: list[str] = []
+            for item in items:
+                token = str(item).strip().lower()
+                if token:
+                    normalized.append(token)
+            return normalized
         for paper in papers:
             bibtex_fields = {}
             bibtex_type = None
@@ -406,6 +427,8 @@ def register_db_commands(db_group: click.Group) -> None:
                 author_counts[author] = author_counts.get(author, 0) + 1
             for tag in paper.get("ai_generated_tags") or []:
                 tag_counts[tag] = tag_counts.get(tag, 0) + 1
+            for keyword in normalize_keywords(paper.get("keywords")):
+                keyword_counts[keyword] = keyword_counts.get(keyword, 0) + 1
 
             venue = None
             if bibtex_type in {"article"}:
@@ -541,6 +564,16 @@ def register_db_commands(db_group: click.Group) -> None:
                 tag_table.add_row(tag, str(count), f"{percentage:.1f}%")
             console.print(tag_table)
 
+        if keyword_counts:
+            keyword_table = Table(title=f"Top {top_n} Keywords")
+            keyword_table.add_column("Keyword", style="cyan")
+            keyword_table.add_column("Count", style="green", justify="right")
+            keyword_table.add_column("Percentage", style="yellow", justify="right")
+            for keyword, count in sorted(keyword_counts.items(), key=lambda item: item[1], reverse=True)[:top_n]:
+                percentage = (count / total * 100) if total else 0
+                keyword_table.add_row(keyword, str(count), f"{percentage:.1f}%")
+            console.print(keyword_table)
+
     @db_group.command("serve")
     @click.option("-i", "--input", "input_paths", multiple=True, required=True, help="Input JSON file path")
     @click.option("-b", "--bibtex", "bibtex_path", default=None, help="Optional BibTeX file path")

{deepresearch_flow-0.1.1 → deepresearch_flow-0.2.0}/python/deepresearch_flow/paper/extract.py

@@ -285,9 +285,14 @@ def load_existing(path: Path) -> list[dict[str, Any]]:
     if not path.exists():
         return []
     try:
-        return json.loads(path.read_text(encoding="utf-8"))
+        data = json.loads(path.read_text(encoding="utf-8"))
     except json.JSONDecodeError:
         return []
+    if isinstance(data, dict) and isinstance(data.get("papers"), list):
+        return data["papers"]
+    if isinstance(data, list):
+        return data
+    return []
 
 
 def load_errors(path: Path) -> list[dict[str, Any]]:
@@ -462,6 +467,7 @@ async def extract_documents(
 ) -> None:
     start_time = time.monotonic()
     markdown_files = discover_markdown(inputs, glob_pattern, recursive=True)
+    template_tag = prompt_template if not custom_prompt else "custom"
 
     if retry_failed:
         error_entries = load_errors(errors_path)
@@ -798,7 +804,8 @@ async def extract_documents(
         if path not in seen:
             final_results.append(entry)
 
-    write_json(output_path, final_results)
+    output_payload = {"template_tag": template_tag, "papers": final_results}
+    write_json(output_path, output_payload)
 
     error_payload = [
         {
@@ -823,7 +830,7 @@ async def extract_documents(
                 continue
             base_name = split_output_name(Path(source_path))
             file_name = unique_split_name(base_name, used_names, source_path)
-            write_json(target_dir / f"{file_name}.json", entry)
+            write_json(target_dir / f"{file_name}.json", {"template_tag": template_tag, "papers": [entry]})
 
     if render_md:
         try: