visual-parser 1.0.2__tar.gz → 2.0.1__tar.gz

{visual_parser-1.0.2 → visual_parser-2.0.1}/PKG-INFO

@@ -1,8 +1,9 @@
 Metadata-Version: 2.4
 Name: visual-parser
-Version: 1.0.2
-Summary: Standalone Visual-RAG PDF Parser — text extraction + Vision-LLM figure descriptions → JSONL
-License: MIT
+Version: 2.0.1
+Summary: Standalone Visual-RAG PDF Parser - text extraction and Vision-LLM figure descriptions to JSONL
+Author-email: "Zavier N. Ndum" <zavier.ndum@tamu.edu>
+License: Apache-2.0
 Project-URL: Homepage, https://github.com/SmartLabNuclear/RADIANT_LLM
 Project-URL: Repository, https://github.com/SmartLabNuclear/RADIANT_LLM
 Project-URL: Docker Hub, https://hub.docker.com/r/zev94/radiant-llm
@@ -11,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: License :: OSI Approved :: MIT License
+Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing :: Markup
@@ -73,7 +74,7 @@ Prebuilt images are on **[zev94/radiant-llm](https://hub.docker.com/r/zev94/radi
 
 | Tag | Description |
 |-----|-------------|
-| `visual-parser-1.0` | Pinned release |
+| `visual-parser-2.0` | Pinned release (v2.0) |
 | `visual-parser-latest` | Latest visual-parser build |
 
 ### 1) Install Docker
@@ -81,7 +82,7 @@ Prebuilt images are on **[zev94/radiant-llm](https://hub.docker.com/r/zev94/radi
 
 ### 2) Pull the image
 ```bash
-docker pull zev94/radiant-llm:visual-parser-1.0
+docker pull zev94/radiant-llm:visual-parser-2.0
 ```
 
 ### 3) Run (input + output on the same mounted folder)
@@ -89,7 +90,7 @@ Windows PowerShell:
 ```powershell
 docker run --rm --env-file .env `
   -v "C:\path\to\pdfs:/data" `
-  zev94/radiant-llm:visual-parser-1.0 `
+  zev94/radiant-llm:visual-parser-2.0 `
   --input-dir /data --output-dir /data
 ```
 
@@ -97,7 +98,7 @@ Linux / WSL:
 ```bash
 docker run --rm --env-file .env \
   -v "/path/to/pdfs:/data" \
-  zev94/radiant-llm:visual-parser-1.0 \
+  zev94/radiant-llm:visual-parser-2.0 \
   --input-dir /data --output-dir /data
 ```
 
@@ -107,7 +108,7 @@ Windows PowerShell:
 docker run --rm --env-file .env `
   -v "C:\path\to\pdfs:/data" `
   -v "C:\path\to\out:/out" `
-  zev94/radiant-llm:visual-parser-1.0 `
+  zev94/radiant-llm:visual-parser-2.0 `
   --input-dir /data --output-dir /out
 ```
 
@@ -124,7 +125,7 @@ Default vision model is **GPT-5.5** when using `--vision-provider gpt`. Override
 
 ```powershell
 docker run --rm --env-file .env -v "C:\path\to\pdfs:/data" `
-  zev94/radiant-llm:visual-parser-1.0 `
+  zev94/radiant-llm:visual-parser-2.0 `
   --input-dir /data --output-dir /data --vision-model gpt-5.4
 ```
 
@@ -140,7 +141,7 @@ python visual-parser.py --input-dir "C:\path\to\pdfs"
 After pulling the image, run:
 
 ```bash
-docker run --rm zev94/radiant-llm:visual-parser-1.0 --help
+docker run --rm zev94/radiant-llm:visual-parser-2.0 --help
 ```
 
 For copy-paste **Docker** examples (vision presets, text modes, workers, rebuild), see [`docker-usage-examples.md`](docker-usage-examples.md).

{visual_parser-1.0.2 → visual_parser-2.0.1}/README.md

@@ -32,7 +32,7 @@ Prebuilt images are on **[zev94/radiant-llm](https://hub.docker.com/r/zev94/radi
 
 | Tag | Description |
 |-----|-------------|
-| `visual-parser-1.0` | Pinned release |
+| `visual-parser-2.0` | Pinned release (v2.0) |
 | `visual-parser-latest` | Latest visual-parser build |
 
 ### 1) Install Docker
@@ -40,7 +40,7 @@ Prebuilt images are on **[zev94/radiant-llm](https://hub.docker.com/r/zev94/radi
 
 ### 2) Pull the image
 ```bash
-docker pull zev94/radiant-llm:visual-parser-1.0
+docker pull zev94/radiant-llm:visual-parser-2.0
 ```
 
 ### 3) Run (input + output on the same mounted folder)
@@ -48,7 +48,7 @@ Windows PowerShell:
 ```powershell
 docker run --rm --env-file .env `
   -v "C:\path\to\pdfs:/data" `
-  zev94/radiant-llm:visual-parser-1.0 `
+  zev94/radiant-llm:visual-parser-2.0 `
   --input-dir /data --output-dir /data
 ```
 
@@ -56,7 +56,7 @@ Linux / WSL:
 ```bash
 docker run --rm --env-file .env \
   -v "/path/to/pdfs:/data" \
-  zev94/radiant-llm:visual-parser-1.0 \
+  zev94/radiant-llm:visual-parser-2.0 \
   --input-dir /data --output-dir /data
 ```
 
@@ -66,7 +66,7 @@ Windows PowerShell:
 docker run --rm --env-file .env `
   -v "C:\path\to\pdfs:/data" `
   -v "C:\path\to\out:/out" `
-  zev94/radiant-llm:visual-parser-1.0 `
+  zev94/radiant-llm:visual-parser-2.0 `
   --input-dir /data --output-dir /out
 ```
 
@@ -83,7 +83,7 @@ Default vision model is **GPT-5.5** when using `--vision-provider gpt`. Override
 
 ```powershell
 docker run --rm --env-file .env -v "C:\path\to\pdfs:/data" `
-  zev94/radiant-llm:visual-parser-1.0 `
+  zev94/radiant-llm:visual-parser-2.0 `
   --input-dir /data --output-dir /data --vision-model gpt-5.4
 ```
 
@@ -99,7 +99,7 @@ python visual-parser.py --input-dir "C:\path\to\pdfs"
 After pulling the image, run:
 
 ```bash
-docker run --rm zev94/radiant-llm:visual-parser-1.0 --help
+docker run --rm zev94/radiant-llm:visual-parser-2.0 --help
 ```
 
 For copy-paste **Docker** examples (vision presets, text modes, workers, rebuild), see [`docker-usage-examples.md`](docker-usage-examples.md).

{visual_parser-1.0.2 → visual_parser-2.0.1}/pyproject.toml

@@ -1,14 +1,17 @@
 [build-system]
-requires      = ["setuptools>=61", "wheel"]
+requires = ["setuptools>=61", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name        = "visual-parser"
-version     = "1.0.2"
-description = "Standalone Visual-RAG PDF Parser — text extraction + Vision-LLM figure descriptions → JSONL"
-readme      = "README.md"
+name = "visual-parser"
+version = "2.0.1"
+description = "Standalone Visual-RAG PDF Parser - text extraction and Vision-LLM figure descriptions to JSONL"
+readme = "README.md"
 requires-python = ">=3.10"
-license     = { text = "MIT" }
+license = { text = "Apache-2.0" }
+authors = [
+    { name = "Zavier N. Ndum", email = "zavier.ndum@tamu.edu" },
+]
 
 keywords = [
     "pdf", "rag", "nougat", "vision-llm", "ocr",
@@ -20,7 +23,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
-    "License :: OSI Approved :: MIT License",
+    "License :: OSI Approved :: Apache Software License",
     "Operating System :: OS Independent",
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
     "Topic :: Text Processing :: Markup",
@@ -45,13 +48,13 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-ocr = ["pytesseract==0.3.13"]   # requires Tesseract binary installed separately
+ocr = ["pytesseract==0.3.13"]
 dev = ["pytest", "ruff", "mypy"]
 
 [project.urls]
-Homepage      = "https://github.com/SmartLabNuclear/RADIANT_LLM"
-Repository    = "https://github.com/SmartLabNuclear/RADIANT_LLM"
-"Docker Hub"  = "https://hub.docker.com/r/zev94/radiant-llm"
+Homepage = "https://github.com/SmartLabNuclear/RADIANT_LLM"
+Repository = "https://github.com/SmartLabNuclear/RADIANT_LLM"
+"Docker Hub" = "https://hub.docker.com/r/zev94/radiant-llm"
 
 [project.scripts]
 visual-parser = "visual_parser.cli_main:main"

{visual_parser-1.0.2 → visual_parser-2.0.1}/visual_parser/__init__.py

@@ -17,4 +17,4 @@ from visual_parser.config import ParserConfig
 from visual_parser.pipeline import run_pipeline
 
 __all__ = ["ParserConfig", "run_pipeline"]
-__version__ = "1.0.2"
+__version__ = "2.0.0"

{visual_parser-1.0.2 → visual_parser-2.0.1}/visual_parser/cli.py

@@ -173,6 +173,17 @@ def _build_arg_parser() -> argparse.ArgumentParser:
             "Use after changing prompts, chunking strategy, or switching models."
         ),
     )
+    misc_group.add_argument(
+        "--skip-text",
+        action="store_true",
+        help=(
+            "Skip text extraction (Step 1) and resume only the vision steps "
+            "(figure descriptions + metadata). Use when chunking already completed "
+            "but the run was interrupted mid-vision (e.g. API credit exhaustion). "
+            "PDFs already present in 02_visuals_kb.jsonl / 03_metadata_kb.jsonl "
+            "are skipped automatically — no duplicates."
+        ),
+    )
     misc_group.add_argument(
         "--log-level",
         choices=["DEBUG", "INFO", "WARNING", "ERROR"],
@@ -216,6 +227,7 @@ def main(argv=None) -> int:
         metadata_pages        = args.metadata_pages,
         max_workers           = args.max_workers,
         rebuild               = args.rebuild,
+        skip_text             = args.skip_text,
         log_level             = args.log_level,
     )
 

{visual_parser-1.0.2 → visual_parser-2.0.1}/visual_parser/cli_main.py

@@ -168,6 +168,17 @@ def _build_arg_parser() -> argparse.ArgumentParser:
             "Use after changing prompts, chunking strategy, or switching models."
         ),
     )
+    misc_group.add_argument(
+        "--skip-text",
+        action="store_true",
+        help=(
+            "Skip text extraction (Step 1) and resume only the vision steps "
+            "(figure descriptions + metadata). Use when chunking already completed "
+            "but the run was interrupted mid-vision (e.g. API credit exhaustion). "
+            "PDFs already present in 02_visuals_kb.jsonl / 03_metadata_kb.jsonl "
+            "are skipped automatically — no duplicates."
+        ),
+    )
     misc_group.add_argument(
         "--log-level",
         choices=["DEBUG", "INFO", "WARNING", "ERROR"],
@@ -206,6 +217,7 @@ def main(argv=None) -> int:
         metadata_pages=args.metadata_pages,
         max_workers=args.max_workers,
         rebuild=args.rebuild,
+        skip_text=args.skip_text,
         log_level=args.log_level,
     )
 

{visual_parser-1.0.2 → visual_parser-2.0.1}/visual_parser/config.py

@@ -121,6 +121,15 @@ class ParserConfig:
     rebuild: bool = False
     """If True, reprocess all PDFs even if already recorded in 04_processed_pdfs.txt."""
 
+    skip_text: bool = False
+    """
+    If True, skip text extraction (Step 1) entirely.
+    Use when chunking already completed but the vision steps (figures / metadata)
+    failed mid-run (e.g. API credit exhaustion).  All PDFs in input_dir are
+    re-queued for vision steps; PDFs already present in 02_visuals_kb.jsonl /
+    03_metadata_kb.jsonl are skipped automatically.
+    """
+
     log_level: str = "ERROR"
 
     # -------------------------------------------------------------------------
@@ -145,6 +154,7 @@ class ParserConfig:
             metadata_pages       = int(os.getenv("VISUAL_PARSER_METADATA_PAGES", "2")),
             max_workers          = int(os.getenv("VISUAL_PARSER_MAX_WORKERS", "4")),
             rebuild              = os.getenv("VISUAL_PARSER_REBUILD", "false").lower() == "true",
+            skip_text            = os.getenv("VISUAL_PARSER_SKIP_TEXT", "false").lower() == "true",
             log_level            = os.getenv("VISUAL_PARSER_LOG_LEVEL", "ERROR"),
         )
 

{visual_parser-1.0.2 → visual_parser-2.0.1}/visual_parser/figure_describer.py

@@ -8,7 +8,7 @@ function in PDFAnalyser.py.
 Output
 ------
 One record per figure (or per page that contains at least one figure) is
-appended to ``02_visuals_kb.jsonl`` in *output_dir*:
+appended to ``02_visuals_kb.jsonl`` in *output_dir*:
 
     {
         "source":        "myreport.pdf",
@@ -84,16 +84,16 @@ def describe_figures_for_new_pdfs(
     vision_detail: str = "low",
     raster_dpi: int = 200,
     figure_prompt: str = FIGURE_PROMPT,
-    reasoning_effort: Optional[str] = "medium",
-) -> None:
+    reasoning_effort: Optional[str] = "medium",
+) -> None:
     """
     For each PDF in *new_pdf_paths*, rasterise every page at *raster_dpi* DPI,
     call the Vision LLM page-by-page, parse the figure descriptions, and
-    append the results to ``02_visuals_kb.jsonl`` in *output_dir*.
+    append the results to ``02_visuals_kb.jsonl`` in *output_dir*.
 
     Args:
         new_pdf_paths:    Full paths of PDFs to describe.
-        output_dir:       Directory where ``02_visuals_kb.jsonl`` is written.
+        output_dir:       Directory where ``02_visuals_kb.jsonl`` is written.
         vision_provider:  ``'gpt'`` or ``'gemini'``.
         vision_api_key:   API key for the chosen provider.
         vision_model:     Vision model name string.
@@ -125,7 +125,7 @@ def describe_figures_for_new_pdfs(
 
     if not page_images:
         logger.info("No pages to describe (all PDFs failed to rasterise).")
-        return
+        return
 
     # -----------------------------------------------------------------------
     # Step 2 – Group page images by PDF name
@@ -135,16 +135,47 @@ def describe_figures_for_new_pdfs(
         pages_by_pdf[record["pdf"]].append(record)
 
     # -----------------------------------------------------------------------
-    # Step 3 – Call Vision LLM once per page
+    # Step 2.5 – Build set of (source, page) pairs already on disk so a
+    #            mid-run crash can be resumed at page granularity.
     # -----------------------------------------------------------------------
-    descriptions_by_page: Dict[tuple, List[str]] = {}
+    figures_path = os.path.join(output_dir, "02_visuals_kb.jsonl")
+    done_pages: set = set()
+    if os.path.exists(figures_path):
+        with open(figures_path, encoding="utf-8") as _fh:
+            for _line in _fh:
+                _line = _line.strip()
+                if not _line:
+                    continue
+                try:
+                    _rec = json.loads(_line)
+                    _src = _rec.get("source", "")
+                    _pg  = _rec.get("page")
+                    if _src and _pg is not None:
+                        done_pages.add((_src, _pg))
+                except Exception:
+                    pass
+    if done_pages:
+        logger.info(
+            "Resuming: %d page(s) already in 02_visuals_kb.jsonl — will skip.",
+            len(done_pages),
+        )
+
+    # -----------------------------------------------------------------------
+    # Step 3 – Call Vision LLM per page; flush each page to disk immediately.
+    #          Figure records are independent so per-page atomicity is safe.
+    # -----------------------------------------------------------------------
+    total_written = 0
 
     for pdf_name, image_records in pages_by_pdf.items():
-        per_pdf_count = 0
+        pdf_written = 0
 
         for record in image_records:
-            page_number  = record["page"]
-            image_bytes  = record["bytes"]
+            page_number = record["page"]
+            image_bytes = record["bytes"]
+
+            if (pdf_name, page_number) in done_pages:
+                logger.debug("Skipping %s page %d (already in KB).", pdf_name, page_number)
+                continue
 
             try:
                 raw_response = call_vision_llm(
@@ -159,8 +190,6 @@ def describe_figures_for_new_pdfs(
 
                 captions = _parse_llm_response(raw_response, pdf_name, page_number)
 
-                # Normalise: the model should return a list, but sometimes
-                # returns a single dict for single-figure pages.
                 if isinstance(captions, dict):
                     captions = [captions]
 
@@ -171,15 +200,27 @@ def describe_figures_for_new_pdfs(
                     )
                     continue
 
-                for caption in captions:
+                document_id = make_document_id(pdf_name)
+                page_rows: List[Dict] = []
+                for fig_idx, caption in enumerate(captions):
                     if not isinstance(caption, dict):
                         continue
                     description = caption.get("description")
                     if description is None:
                         continue
-                    key = (pdf_name, page_number)
-                    descriptions_by_page.setdefault(key, []).append(description)
-                    per_pdf_count += 1
+                    page_rows.append({
+                        "source":       pdf_name,
+                        "page":         page_number,
+                        "document_id":  document_id,
+                        "figure_index": fig_idx,
+                        "figure_id":    f"{document_id}:p{page_number}:f{fig_idx}",
+                        "description":  description,
+                    })
+
+                if page_rows:
+                    append_to_jsonl(figures_path, page_rows)
+                    pdf_written   += len(page_rows)
+                    total_written += len(page_rows)
 
             except Exception as exc:
                 logger.error(
@@ -187,32 +228,12 @@ def describe_figures_for_new_pdfs(
                     pdf_name, page_number, exc,
                 )
 
-        logger.info("[FIGURES] %s: %d figure(s) extracted.", pdf_name, per_pdf_count)
-
-    total = sum(len(v) for v in descriptions_by_page.values())
-    logger.info("Total figures captured: %d across %d PDF(s).", total,
-                len({k[0] for k in descriptions_by_page}))
+        if pdf_written:
+            logger.info("[FIGURES] %s: %d figure(s) written.", pdf_name, pdf_written)
+        else:
+            logger.info("[FIGURES] %s: no new figures (all pages done or none detected).", pdf_name)
 
-    # -----------------------------------------------------------------------
-    # Step 4 – Write figure descriptions to 02_visuals_kb.jsonl
-    # -----------------------------------------------------------------------
-    figure_rows: List[Dict] = []
-
-    for (pdf_name, page_number), descriptions in descriptions_by_page.items():
-        document_id = make_document_id(pdf_name)
-        for fig_idx, description in enumerate(descriptions):
-            figure_rows.append({
-                "source":        pdf_name,
-                "page":          page_number,
-                "document_id":   document_id,
-                "figure_index":  fig_idx,
-                "figure_id":     f"{document_id}:p{page_number}:f{fig_idx}",
-                "description":   description,
-            })
-
-    if figure_rows:
-        figures_path = os.path.join(output_dir, "02_visuals_kb.jsonl")
-        append_to_jsonl(figures_path, figure_rows)
-        print(f"[FIGURES] Wrote {len(figure_rows)} figure record(s) to 02_visuals_kb.jsonl.")
+    if total_written:
+        print(f"[FIGURES] Wrote {total_written} figure record(s) to 02_visuals_kb.jsonl.")
     else:
-        logger.info("No figures detected — 02_visuals_kb.jsonl not updated.")
+        logger.info("No new figures written — all pages already processed or none detected.")