visual-parser 2.2.2__py3-none-any.whl → 2.2.4__py3-none-any.whl

visual_parser/__init__.py

@@ -1,14 +1,16 @@
 """
-visual_parser — Standalone Visual-RAG PDF Parser
-=================================================
-Detects new PDFs in a user-supplied directory, extracts text (via Nougat or
-lightweight PyMuPDF/PyPDFLoader), describes every figure/chart/schematic using
-a Vision LLM (OpenAI GPT, Google Gemini, or a local model via Ollama), and
-writes three JSONL knowledge bases ready for any downstream RAG system:
+visual_parser — Standalone Visual-RAG PDF & Image Parser
+=========================================================
+Detects new PDFs (and standalone images alongside them) in a user-supplied
+directory, extracts text (via Nougat or lightweight PyMuPDF/PyPDFLoader),
+describes every figure/chart/schematic and every standalone image using a
+Vision LLM (OpenAI GPT, Google Gemini, or a local model via Ollama), and
+writes JSONL knowledge bases ready for any downstream RAG system:
 
-    01_chunks_kb.jsonl   – text chunks with stable IDs
-    02_visuals_kb.jsonl  – per-figure visual descriptions
-    03_metadata_kb.jsonl – document-level metadata (title, authors, DOI …)
+    01_chunks_kb.jsonl          – text chunks with stable IDs
+    02_visuals_kb.jsonl         – per-figure visual descriptions
+    03_metadata_kb.jsonl        – document-level metadata (title, authors, DOI …)
+    image_descriptions_kb.jsonl – one holistic description per standalone image
 
 No chatbot, no vector store, no retrieval – just a robust parser.
 """
@@ -17,4 +19,4 @@ from visual_parser.config import ParserConfig
 from visual_parser.pipeline import run_pipeline
 
 __all__ = ["ParserConfig", "run_pipeline"]
-__version__ = "2.2.2"
+__version__ = "2.2.4"

visual_parser/cli.py

@@ -44,13 +44,13 @@ def _build_arg_parser() -> argparse.ArgumentParser:
     p = argparse.ArgumentParser(
         prog="visual-parser",
         description=(
-            "Visual-RAG PDF Parser — detects new PDFs and standalone images, "
+            "Visual-RAG PDF & Image Parser — detects new PDFs and standalone images, "
             "extracts text and figure/image descriptions, and writes JSONL "
             "knowledge bases:\n"
-            "  01_chunks_kb.jsonl       text chunks\n"
-            "  02_visuals_kb.jsonl      PDF figure descriptions\n"
-            "  03_metadata_kb.jsonl     document metadata\n"
-            "  image_descriptions.jsonl standalone image descriptions"
+            "  01_chunks_kb.jsonl            text chunks\n"
+            "  02_visuals_kb.jsonl           PDF figure descriptions\n"
+            "  03_metadata_kb.jsonl          document metadata\n"
+            "  image_descriptions_kb.jsonl   standalone image descriptions"
         ),
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog=USAGE_EXAMPLES,

visual_parser/cli_main.py

@@ -1,5 +1,5 @@
 """
-cli_main.py - ASCII-safe CLI entry point for the Visual-RAG PDF Parser.
+cli_main.py - ASCII-safe CLI entry point for the Visual-RAG PDF & Image Parser.
 
 This module exists to keep Windows console help output stable for the
 installed ``visual-parser`` command and ``python -m visual_parser``.
@@ -42,13 +42,13 @@ def _build_arg_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog="visual-parser",
         description=(
-            "Visual-RAG PDF Parser - detects new PDFs and standalone images, "
+            "Visual-RAG PDF & Image Parser - detects new PDFs and standalone images, "
             "extracts text and figure/image descriptions, and writes JSONL "
             "knowledge bases:\n"
-            "  01_chunks_kb.jsonl       text chunks\n"
-            "  02_visuals_kb.jsonl      PDF figure descriptions\n"
-            "  03_metadata_kb.jsonl     document metadata\n"
-            "  image_descriptions.jsonl standalone image descriptions"
+            "  01_chunks_kb.jsonl            text chunks\n"
+            "  02_visuals_kb.jsonl           PDF figure descriptions\n"
+            "  03_metadata_kb.jsonl          document metadata\n"
+            "  image_descriptions_kb.jsonl   standalone image descriptions"
         ),
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog=USAGE_EXAMPLES,

visual_parser/figure_describer.py

@@ -16,7 +16,9 @@ appended to ``02_visuals_kb.jsonl`` in *output_dir*:
         "document_id":   "a1b2c3d4e5f6g7h8",
         "figure_index":  0,
         "figure_id":     "a1b2c3d4e5f6g7h8:p3:f0",
-        "description":   "**Subject:** ..."
+        "description":   "**Subject:** ...",
+        "vision_provider": "gpt",
+        "vision_model":    "gpt-5.4"
     }
 """
 
@@ -88,7 +90,7 @@ def describe_figures_for_new_pdfs(
     reasoning_effort: Optional[str] = "medium",
     max_workers: int = 4,
     context_pages: int = 0,
-) -> None:
+) -> List[str]:
     """
     For each PDF in *new_pdf_paths*, rasterise every page at *raster_dpi* DPI,
     call the Vision LLM for each page (concurrently, up to *max_workers* at
@@ -124,6 +126,21 @@ def describe_figures_for_new_pdfs(
     its basename from the processed-PDFs registry, then pass it in here as
     if new; this function has no bypass of its own, so there is no path to
     producing a duplicate row.
+
+    Each written row also carries ``vision_provider``/``vision_model``, so a
+    document resumed later under a different model is detectable. Detection
+    is active, not just recorded after the fact: if a PDF already has rows
+    from a *different* vision_model than this call's, its remaining pages
+    are skipped entirely (not mixed in under the new model) and its basename
+    is returned, so a resumed document never silently ends up with figures
+    described by two different models. Pre-upgrade rows with no recorded
+    vision_model are treated as unknown, not a mismatch, and do not block.
+    Callers that want the document redescribed consistently under the new
+    model must first clear it (see kb_redo.redo_entries(), e.g. via --redo).
+
+    Returns:
+        Basenames skipped in full due to a detected vision_model mismatch
+        against their existing rows (empty list when none were skipped).
     """
     # -----------------------------------------------------------------------
     # Step 1 – Rasterise every page of every new PDF
@@ -147,7 +164,7 @@ def describe_figures_for_new_pdfs(
 
     if not page_images:
         logger.info("No pages to describe (all PDFs failed to rasterise).")
-        return
+        return []
 
     # -----------------------------------------------------------------------
     # Step 2 – Group page images by PDF name. Order within each group is
@@ -165,6 +182,7 @@ def describe_figures_for_new_pdfs(
     # -----------------------------------------------------------------------
     figures_path = os.path.join(output_dir, "02_visuals_kb.jsonl")
     done_pages: set = set()
+    prior_models_by_pdf: Dict[str, set] = defaultdict(set)
     if os.path.exists(figures_path):
         with open(figures_path, encoding="utf-8") as _fh:
             for _line in _fh:
@@ -177,6 +195,9 @@ def describe_figures_for_new_pdfs(
                     _pg  = _rec.get("page")
                     if _src and _pg is not None:
                         done_pages.add((_src, _pg))
+                    _prior_model = _rec.get("vision_model")
+                    if _src and _prior_model:
+                        prior_models_by_pdf[_src].add(_prior_model)
                 except Exception:
                     pass
     if done_pages:
@@ -191,7 +212,20 @@ def describe_figures_for_new_pdfs(
     #          never crosses into a different PDF.
     # -----------------------------------------------------------------------
     tasks: List[Dict[str, Any]] = []
+    model_mismatch_skipped: List[str] = []
     for pdf_name, image_records in pages_by_pdf.items():
+        prior_models = prior_models_by_pdf.get(pdf_name)
+        if prior_models and vision_model not in prior_models:
+            msg = (
+                f"[FIGURES] Skipping {pdf_name}: already has figures described with "
+                f"{', '.join(sorted(prior_models))}, but this run is using {vision_model}. "
+                f"Run with --redo {pdf_name} to reprocess it entirely under the new model."
+            )
+            logger.error(msg)
+            print(msg)
+            model_mismatch_skipped.append(pdf_name)
+            continue
+
         for idx, record in enumerate(image_records):
             page_number = record["page"]
             if (pdf_name, page_number) in done_pages:
@@ -213,7 +247,7 @@ def describe_figures_for_new_pdfs(
 
     if not tasks:
         logger.info("No new pages to describe — all already processed.")
-        return
+        return model_mismatch_skipped
 
     # -----------------------------------------------------------------------
     # Step 4 – Call Vision LLM per task, concurrently. Each worker only
@@ -265,12 +299,14 @@ def describe_figures_for_new_pdfs(
                     )
                     continue
                 page_rows.append({
-                    "source":       pdf_name,
-                    "page":         page_number,
-                    "document_id":  document_id,
-                    "figure_index": fig_idx,
-                    "figure_id":    f"{document_id}:p{page_number}:f{fig_idx}",
-                    "description":  description,
+                    "source":          pdf_name,
+                    "page":            page_number,
+                    "document_id":     document_id,
+                    "figure_index":    fig_idx,
+                    "figure_id":       f"{document_id}:p{page_number}:f{fig_idx}",
+                    "description":     description,
+                    "vision_provider": vision_provider,
+                    "vision_model":    vision_model,
                 })
             return {"pdf_name": pdf_name, "page_number": page_number, "rows": page_rows}
 
@@ -306,10 +342,12 @@ def describe_figures_for_new_pdfs(
     for pdf_name, count in written_by_pdf.items():
         logger.info("[FIGURES] %s: %d figure(s) written.", pdf_name, count)
     for pdf_name in pages_by_pdf:
-        if pdf_name not in written_by_pdf:
+        if pdf_name not in written_by_pdf and pdf_name not in model_mismatch_skipped:
             logger.info("[FIGURES] %s: no new figures (all pages done or none detected).", pdf_name)
 
     if total_written:
         print(f"[FIGURES] Wrote {total_written} figure record(s) to 02_visuals_kb.jsonl.")
     else:
         logger.info("No new figures written — all pages already processed or none detected.")
+
+    return model_mismatch_skipped

visual_parser/image_describer.py

@@ -10,14 +10,16 @@ supplied image files instead of PyMuPDF-rasterized PDF pages.
 
 Output
 ------
-One record per image is appended to ``image_descriptions.jsonl`` in
+One record per image is appended to ``image_descriptions_kb.jsonl`` in
 *output_dir*:
 
     {
         "source":      "diagram1.png",
         "image_id":    "a1b2c3d4e5f6g7h8",
         "title":       "Vertical Parabolic Gate Schematic",
-        "description": "**Subject:** ..."
+        "description": "**Subject:** ...",
+        "vision_provider": "gpt",
+        "vision_model":    "gpt-5.4"
     }
 """
 
@@ -40,7 +42,7 @@ from visual_parser.vision_llm import call_vision_llm
 
 logger = logging.getLogger(__name__)
 
-IMAGE_DESCRIPTIONS_FILE = "image_descriptions.jsonl"
+IMAGE_DESCRIPTIONS_FILE = "image_descriptions_kb.jsonl"
 
 # Decoupled from figure_describer.py's context-window-scaled formula (10 min
 # for 1 image there): IMAGE_DESCRIPTION_PROMPT's "Verbatim Text Content"
@@ -171,7 +173,7 @@ def describe_images(
     """
     For each image in *new_image_paths*, call the Vision LLM once (no
     context window -- a single standalone image per call), parse the
-    description, and append the result to ``image_descriptions.jsonl`` in
+    description, and append the result to ``image_descriptions_kb.jsonl`` in
     *output_dir*.
 
     Returns the basenames actually written a row for -- NOT the full input
@@ -183,7 +185,7 @@ def describe_images(
 
     Args:
         new_image_paths:  Full paths of images to describe.
-        output_dir:       Directory where ``image_descriptions.jsonl`` is written.
+        output_dir:       Directory where ``image_descriptions_kb.jsonl`` is written.
         vision_provider:  ``'gpt'``, ``'gemini'``, or ``'ollama'``.
         vision_api_key:   API key for the chosen provider (unused for ollama).
         vision_model:     Vision model name string.
@@ -191,7 +193,7 @@ def describe_images(
         image_prompt:     The instruction prompt sent with each image.
         max_workers:      Thread-pool size for concurrent vision-LLM calls.
 
-    Always resumes from image_descriptions.jsonl's existing content (via the
+    Always resumes from image_descriptions_kb.jsonl's existing content (via the
     done_images set below) -- callers that want an image redescribed must
     first remove its existing row and its basename from the processed-
     images registry (see kb_redo.redo_entries()), then pass it in here as if
@@ -279,10 +281,12 @@ def describe_images(
                 return {"image_name": image_name, "row": None}
 
             row = {
-                "source":      image_name,
-                "image_id":    make_document_id(image_name),
-                "title":       _derive_title(description),
-                "description": description,
+                "source":          image_name,
+                "image_id":        make_document_id(image_name),
+                "title":           _derive_title(description),
+                "description":     description,
+                "vision_provider": vision_provider,
+                "vision_model":    vision_model,
             }
             return {"image_name": image_name, "row": row}
 

visual_parser/image_tracker.py

@@ -35,13 +35,10 @@ def find_new_images(
     Walk *input_dir* recursively and return full paths of standalone images
     that have NOT yet been processed.
 
-    Unlike find_new_pdfs(), *registry_path* is an explicit parameter rather
-    than being joined internally against *input_dir* -- this lets the caller
-    anchor the registry at effective_output_dir() unambiguously, sidestepping
-    a pre-existing bug on the PDF side where find_new_pdfs() always resolves
-    its registry against input_dir while pipeline.py separately writes it
-    against output_dir (the two diverge, and PDFs get reprocessed forever,
-    whenever --output-dir differs from --input-dir).
+    *registry_path* is an explicit required parameter (rather than a filename
+    joined internally against *input_dir*, as find_new_pdfs() used to do by
+    default) so the caller can anchor the registry at effective_output_dir()
+    unambiguously. find_new_pdfs() now accepts the same explicit override.
 
     Args:
         input_dir:     Root directory to search for image files.

visual_parser/kb_redo.py

@@ -14,7 +14,7 @@ dependency-free package and cannot depend on the sibling RADIANT-LLM repo)
 the proven basename-match + backup-once + atomic-rewrite pattern from
 RADIANT-LLM's knowledge_base_sanitizer() (radiant_lmm/RADIANT_LLM_ChatBot_
 Local/utils/pdf_helpers.py:1464-1607), extended to the 4th JSONL file
-(image_descriptions.jsonl) and both registries (04_processed_pdfs.txt,
+(image_descriptions_kb.jsonl) and both registries (04_processed_pdfs.txt,
 parsed_images.txt), which that function never touches at all.
 """
 
@@ -25,6 +25,7 @@ import os
 import shutil
 from typing import Any, Dict, Iterable, List
 
+from visual_parser.image_describer import IMAGE_DESCRIPTIONS_FILE
 from visual_parser.image_tracker import PROCESSED_IMAGES_REGISTRY
 from visual_parser.jsonl_writer import atomic_rewrite_jsonl
 from visual_parser.pdf_tracker import PROCESSED_REGISTRY, load_processed_pdfs, unmark_as_processed
@@ -33,12 +34,16 @@ logger = logging.getLogger(__name__)
 
 # Hardcoded, not glob("*.jsonl") like the RADIANT precedent -- output_dir
 # often equals input_dir (the documented default), so globbing risks
-# touching unrelated JSONL files a user happens to keep there.
+# touching unrelated JSONL files a user happens to keep there. The image
+# filename is imported (not hardcoded like the other three) because it was
+# renamed once already (image_descriptions.jsonl -> image_descriptions_kb.jsonl,
+# 2.2.3) with this exact file as a hardcoded duplicate that needed a manual
+# fix -- importing the constant means it can't drift out of sync again.
 KB_JSONL_FILENAMES = (
     "01_chunks_kb.jsonl",
     "02_visuals_kb.jsonl",
     "03_metadata_kb.jsonl",
-    "image_descriptions.jsonl",
+    IMAGE_DESCRIPTIONS_FILE,
 )
 REGISTRY_FILENAMES = (PROCESSED_REGISTRY, PROCESSED_IMAGES_REGISTRY)
 BACKUP_SUFFIX = "_backup_before_redo"

visual_parser/pdf_tracker.py

@@ -96,6 +96,8 @@ def find_new_pdfs(
     input_dir: str,
     registry_filename: str = PROCESSED_REGISTRY,
     rebuild: bool = False,
+    *,
+    registry_path: str | None = None,
 ) -> List[str]:
     """
     Walk *input_dir* recursively and return full paths of PDFs that have NOT
@@ -103,14 +105,24 @@ def find_new_pdfs(
 
     Args:
         input_dir:         Root directory to search for ``.pdf`` files.
-        registry_filename: Name of the tracking file inside *input_dir*.
+        registry_filename: Name of the tracking file inside *input_dir*,
+                           used only when *registry_path* is not given.
         rebuild:           When True, return *all* PDFs regardless of the
                            registry (forces a full re-parse).
+        registry_path:     Full path to the tracking file. When omitted,
+                           defaults to ``input_dir/registry_filename`` (the
+                           historical behavior, kept for callers -- such as
+                           RADIANT-LLM -- that use the same folder for both
+                           input and output). Callers whose output directory
+                           differs from their input directory must pass the
+                           registry's real location explicitly here, the
+                           same way find_new_images() already requires.
 
     Returns:
         Sorted list of absolute PDF paths.
     """
-    registry_path = os.path.join(input_dir, registry_filename)
+    if registry_path is None:
+        registry_path = os.path.join(input_dir, registry_filename)
     processed = set() if rebuild else set(load_processed_pdfs(registry_path))
 
     new_pdfs = [

visual_parser/pipeline.py

@@ -26,7 +26,8 @@ from __future__ import annotations
 
 import logging
 import os
-from typing import Dict, List, Optional
+from datetime import datetime, timezone
+from typing import Dict, List, Optional, Tuple
 
 from visual_parser.config import ParserConfig
 from visual_parser.figure_describer import describe_figures_for_new_pdfs
@@ -60,6 +61,41 @@ def _setup_logging(config: ParserConfig) -> None:
     logging.getLogger().addHandler(console)
 
 
+def _build_parser_block(
+    config: ParserConfig,
+    vision_model: str,
+    text_engine: str,
+    text_fallback: bool,
+) -> Dict:
+    """
+    Build the per-document ``parser`` provenance block attached to each
+    03_metadata_kb.jsonl row -- records which engine/model actually produced
+    this document's rows, since the configured settings aren't always what
+    ran (Nougat can fail and fall back to lightweight; Ollama can auto-select
+    a model at run time). New field only; does not change any existing KB
+    field or ID.
+    """
+    from visual_parser import __version__
+
+    block = {
+        "visual_parser":        __version__,
+        "parsed_at":            datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+        "text_engine":          text_engine,
+        "text_fallback":        text_fallback,
+        "vision_provider":      config.vision_provider,
+        "vision_model":         vision_model,
+        "vision_detail":        config.vision_detail,
+        "reasoning_effort":     config.gpt_reasoning_effort,
+        "metadata_pages":       config.metadata_pages,
+        "vision_context_pages": config.vision_context_pages,
+        "chunk_size":           config.chunk_size,
+        "chunk_overlap":        config.chunk_overlap,
+    }
+    if text_engine == "nougat":
+        block["nougat_model"] = config.nougat_model
+    return block
+
+
 def _recover_stale_registry_entries(input_dir: str, output_dir: str, registry_path: str) -> int:
     """
     Remove processed-PDF registry entries for files that have no chunk records.
@@ -134,6 +170,7 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
                 "redone_basenames":     List[str],
                 "redo_names_not_found": List[str],
                 "processed_basenames":  List[str],
+                "figure_model_mismatches": List[str],
             }
     """
     if config is None:
@@ -155,6 +192,7 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
         "failed_basenames":     [],
         "redone_basenames":     [],
         "redo_names_not_found": [],
+        "figure_model_mismatches": [],
         "status":               "success",
     }
 
@@ -174,7 +212,10 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
     # -----------------------------------------------------------------------
     redo_name_set = set(config.redo_names)
     if config.rebuild:
-        redo_name_set |= {os.path.basename(p) for p in find_new_pdfs(config.input_dir, rebuild=True)}
+        redo_name_set |= {
+            os.path.basename(p)
+            for p in find_new_pdfs(config.input_dir, rebuild=True, registry_path=registry_path)
+        }
         redo_name_set |= {
             os.path.basename(p)
             for p in find_new_images(config.input_dir, images_registry_path, rebuild=True)
@@ -384,7 +425,7 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
         ) if os.path.exists(figures_path) else 0
 
         print(f"[Step 2] Describing figures for {len(all_pdfs)} PDF(s) (page-level resume active) …")
-        describe_figures_for_new_pdfs(
+        summary["figure_model_mismatches"] = describe_figures_for_new_pdfs(
             new_pdf_paths    = all_pdfs,
             output_dir       = output_dir,
             vision_provider  = config.vision_provider,
@@ -417,6 +458,7 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
             row         = {"source": source, "document_id": document_id}
             if isinstance(meta, dict):
                 row.update(meta)
+            row["parser"] = _build_parser_block(config, _vision_model, "skipped", False)
             metadata_rows.append(row)
 
         if metadata_rows:
@@ -432,13 +474,13 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
     else:
         # ── Normal (full) pipeline ───────────────────────────────────────────
 
-        new_pdfs = find_new_pdfs(config.input_dir, rebuild=False)
+        new_pdfs = find_new_pdfs(config.input_dir, rebuild=False, registry_path=registry_path)
         summary["new_pdfs_found"] = len(new_pdfs)
 
         if not new_pdfs:
             recovered = _recover_stale_registry_entries(config.input_dir, output_dir, registry_path)
             if recovered:
-                new_pdfs = find_new_pdfs(config.input_dir, rebuild=False)
+                new_pdfs = find_new_pdfs(config.input_dir, rebuild=False, registry_path=registry_path)
                 summary["new_pdfs_found"] = len(new_pdfs)
 
         if not new_pdfs and not images_to_process:
@@ -482,7 +524,11 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
         # Step 1 — Text extraction and chunking. Guarded by `if new_pdfs:` so
         # an image-only pass (possible now that images are tracked/processed
         # independently of PDFs) never needlessly loads Nougat for zero PDFs.
+        # text_engine_by_basename records which engine actually produced each
+        # PDF's chunks (engine, text_fallback) -- the configured text_mode
+        # isn't always what ran, since Nougat can fail and fall back.
         processor = model = device = None  # only bound by the "nougat" branch below
+        text_engine_by_basename: Dict[str, Tuple[str, bool]] = {}
         if not new_pdfs:
             processed_basenames, failed_basenames, chunk_count = [], [], 0
         elif config.text_mode == "nougat":
@@ -504,6 +550,8 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
                     nougat_crop_margin = config.nougat_crop_margin,
                 )
                 print(nougat_summary)
+                for _name in processed_basenames:
+                    text_engine_by_basename[_name] = ("nougat", False)
             except Exception as exc:
                 logger.error("Nougat initialization/extraction failed: %s", exc)
                 print(f"[Step 1] Nougat failed ({exc}). Falling back to lightweight extraction.")
@@ -515,6 +563,8 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
                     max_workers        = config.max_workers,
                 )
                 print(lw_summary)
+                for _name in processed_basenames:
+                    text_engine_by_basename[_name] = ("lightweight", True)
             else:
                 if failed_basenames:
                     failed_set = set(failed_basenames)
@@ -534,6 +584,8 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
                     processed_basenames.extend(lw_processed)
                     failed_basenames = [name for name in failed_basenames if name not in set(lw_processed)]
                     chunk_count += lw_chunk_count
+                    for _name in lw_processed:
+                        text_engine_by_basename[_name] = ("lightweight", True)
 
         else:  # "lightweight"
             print("[Step 1] Running lightweight (PyMuPDF) text extraction …")
@@ -547,6 +599,8 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
                 max_workers        = config.max_workers,
             )
             print(lw_summary)
+            for _name in processed_basenames:
+                text_engine_by_basename[_name] = ("lightweight", False)
 
         summary["processed_basenames"] = processed_basenames
         summary["failed_basenames"]    = failed_basenames
@@ -587,7 +641,7 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
 
         if pdfs_for_figures:
             print(f"[Step 2] Describing figures in {len(pdfs_for_figures)} PDF(s) …")
-            describe_figures_for_new_pdfs(
+            summary["figure_model_mismatches"] = describe_figures_for_new_pdfs(
                 new_pdf_paths    = pdfs_for_figures,
                 output_dir       = output_dir,
                 vision_provider  = config.vision_provider,
@@ -621,6 +675,8 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
             row         = {"source": source, "document_id": document_id}
             if isinstance(meta, dict):
                 row.update(meta)
+            text_engine, text_fallback = text_engine_by_basename.get(source, ("unknown", False))
+            row["parser"] = _build_parser_block(config, _vision_model, text_engine, text_fallback)
             metadata_rows.append(row)
 
         if metadata_rows:

visual_parser/text_extractor.py

@@ -34,7 +34,7 @@ import pymupdf as fitz  # PyMuPDF -- "fitz" is the deprecated legacy import name
 from PIL import Image
 
 from visual_parser.jsonl_writer import append_to_jsonl, make_document_id
-from visual_parser.pdf_tracker import load_processed_pdfs
+from visual_parser.pdf_tracker import PROCESSED_REGISTRY, load_processed_pdfs
 
 logger = logging.getLogger(__name__)
 
@@ -101,7 +101,7 @@ def nougat_extract_pdfs(
 
     from visual_parser.nougat_engine import RasterizePaper, StoppingCriteriaScores
 
-    registry_path = os.path.join(output_dir, "04_processed_pdfs.txt")
+    registry_path = os.path.join(output_dir, PROCESSED_REGISTRY)
     processed_set = set(load_processed_pdfs(registry_path))
     pdfs_to_run   = [p for p in only_process_these if os.path.basename(p) not in processed_set]
 
@@ -284,7 +284,7 @@ def lightweight_extract_pdfs(
     """
     from io import BytesIO
 
-    registry_path = os.path.join(output_dir, "04_processed_pdfs.txt")
+    registry_path = os.path.join(output_dir, PROCESSED_REGISTRY)
     processed_set = set(load_processed_pdfs(registry_path))
     pdfs_to_run   = [p for p in only_process_these if os.path.basename(p) not in processed_set]
 

{visual_parser-2.2.2.dist-info → visual_parser-2.2.4.dist-info}/METADATA

@@ -1,14 +1,14 @@
 Metadata-Version: 2.4
 Name: visual-parser
-Version: 2.2.2
-Summary: Standalone Visual-RAG PDF Parser - text extraction and Vision-LLM figure descriptions to JSONL
+Version: 2.2.4
+Summary: Standalone Visual-RAG PDF & Image Parser - text extraction and Vision-LLM figure/image descriptions to JSONL
 Author: Zavier N. Ndum
 Author-email: zavier.ndum@tamu.edu
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://github.com/SmartLabNuclear/RADIANT_LLM
 Project-URL: Repository, https://github.com/SmartLabNuclear/RADIANT_LLM
 Project-URL: Docker Hub, https://hub.docker.com/r/zev94/radiant-llm
-Keywords: pdf,rag,nougat,vision-llm,ocr,document-parsing,jsonl,knowledge-base
+Keywords: pdf,image,rag,nougat,vision-llm,ocr,document-parsing,jsonl,knowledge-base
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -43,17 +43,17 @@ Requires-Dist: ruff; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
 Dynamic: license-file
 
-# visual-parser (Standalone Visual-RAG PDF Ingestion)
+# visual-parser (Standalone Visual-RAG PDF & Image Ingestion)
 
 ![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-brightgreen.svg)
 ![PyTorch](https://img.shields.io/badge/PyTorch-2.1%2B-ee4c2c.svg)
 ![LangChain](https://img.shields.io/badge/LangChain-0.1%2B-1C3C3C.svg)
 ![CUDA](https://img.shields.io/badge/CUDA-optional-76B900.svg)
 
-`visual-parser` converts PDFs (and standalone images alongside them) into a multi-modal JSONL knowledge base (text chunks, figure descriptions, standalone image descriptions, metadata). It was extracted from [RADIANT-LLM](https://github.com/SmartLabNuclear/RADIANT_LLM) as a standalone PDF-ingestion tool, independent of any chatbot.
+`visual-parser` converts PDFs (and standalone images alongside them) into a multi-modal JSONL knowledge base (text chunks, figure descriptions, standalone image descriptions, metadata). It was extracted from [RADIANT-LLM](https://github.com/SmartLabNuclear/RADIANT_LLM) as a standalone PDF-and-image ingestion tool, independent of any chatbot.
 
-1) Run `visual-parser` on curated PDFs to generate JSONL KB files.
-2) Point any downstream RAG system at the generated KB — RADIANT-LLM, AutoSAM, and AutoFLUKA all consume the identical JSONL/registry format, so the same KB works with any of them without re-parsing.
+1) Run `visual-parser` on a directory of curated PDFs and/or standalone images to generate JSONL KB files.
+2) Point any downstream RAG system at the generated KB — [RADIANT-LLM](https://github.com/SmartLabNuclear/RADIANT_LLM), [AutoSAM](https://github.com/SmartLabNuclear/AutoSAM), and [AutoFLUKA](https://github.com/SmartLabNuclear/AutoFLUKA) all consume the identical JSONL/registry format, so the same KB works with any of them without re-parsing.
 
 ## Table of Contents
 
@@ -74,10 +74,14 @@ By default, the pipeline writes:
 - `01_chunks_kb.jsonl`: chunked text extracted from PDFs (Nougat by default).
 - `02_visuals_kb.jsonl`: figure/page visual descriptions (Vision LLM), one row per figure found in a PDF.
 - `03_metadata_kb.jsonl`: document metadata rows (title/author/etc.).
-- `image_descriptions.jsonl`: one holistic description per **standalone image** file found alongside your PDFs in `--input-dir` (not a PDF's own figures) — auto-detected by extension, no separate flag needed; disable with `--skip-images`.
+- `image_descriptions_kb.jsonl`: one holistic description per **standalone image** file found alongside your PDFs in `--input-dir` (not a PDF's own figures) — auto-detected by extension, no separate flag needed; disable with `--skip-images`.
 
 Alongside the KB, the pipeline also writes bookkeeping files: `04_processed_pdfs.txt` / `parsed_images.txt` (track which PDFs/images have already been processed, so re-runs skip them unless `--rebuild` or `--redo NAME`) and `05_pipeline.log` (run log at the verbosity set by `--log-level`, default `ERROR`).
 
+> **New in 2.2.4 (additive, no existing field or ID changed):** every `02_visuals_kb.jsonl` and `image_descriptions_kb.jsonl` row now carries `vision_provider`/`vision_model`, and every `03_metadata_kb.jsonl` row carries a `parser` block (`visual_parser` version, `parsed_at`, `text_engine`, `text_fallback`, the resolved `vision_model`, and the run's chunking/context settings) — recording which engine and model actually produced that document's rows, since the configured settings aren't always what ran (Nougat can fail and fall back to lightweight; a local Ollama model can be auto-selected at run time). Older rows simply lack these fields. Figure description also now actively guards against a mixed-model document: if a PDF already has figures described under a different `vision_model` than the current run's, its remaining pages are skipped entirely (not partially re-described under the new model) and its basename is reported back — reprocess it with `--redo NAME` to redescribe it consistently under the new model.
+
+> **Breaking change in 2.2.3:** the standalone-image KB file was renamed from `image_descriptions.jsonl` to `image_descriptions_kb.jsonl` (matching the `NN_name_kb.jsonl` convention used by the other three KB files). Upgrading from an older version does **not** migrate existing data automatically — the resume check only looks at the new filename, so already-described images in an old `image_descriptions.jsonl` would otherwise appear "new" again and get re-described. If you have existing data, rename the file yourself before re-running: `image_descriptions.jsonl` → `image_descriptions_kb.jsonl`.
+
 ## API Keys (.env)
 
 Provide at least one provider:

visual_parser-2.2.4.dist-info/RECORD

@@ -0,0 +1,26 @@
+visual_parser/__init__.py,sha256=L8CC_lJgEBX38SyjUNrq_n9-krO9OuAIlgZP316rlAw,1039
+visual_parser/__main__.py,sha256=-p6rqjrnsjrtu71eFtmZdqFI1NgHXX9ASOnjuGqghNo,168
+visual_parser/cli.py,sha256=28u8JxiFwc6cHf-CGk3TiVFn7llQ0k_3smSH_KFR9tQ,16077
+visual_parser/cli_main.py,sha256=fZPCY6M_gMpp8rJ8U1F0-hJjiTRn3Ih03A-A_ZaPp1Q,11841
+visual_parser/config.py,sha256=mu52rGN1keQtho70IUP7kPuS6Wykpcc-gv3z0irg70c,12140
+visual_parser/figure_describer.py,sha256=PrJVAh7cmsEmm06XqJDnOKTMBRwUyzgY0CqJEsodgp0,15520
+visual_parser/image_describer.py,sha256=BIR42ofCGEr7tHSqvOvqWg_APD3P1ihMVs8kAVo4p4Y,14063
+visual_parser/image_tracker.py,sha256=mlEgAdTkheddG12P2xIL6P7dPAYwrygCSPu0qA2WX-w,2292
+visual_parser/jsonl_writer.py,sha256=klkDl_KKdCRtiWWjTNqecN2B6zXPc0e9f704Dvj8Fh0,6215
+visual_parser/kb_redo.py,sha256=5_FpHHaEYQCAXSAN-j-snaj6OlaL36Vqi97DMqPPROA,5897
+visual_parser/metadata_extractor.py,sha256=Z0jnA8Goo9R3SBThx5PQMetZkyrbevNvYCq36WsoQuQ,6579
+visual_parser/model_catalog.py,sha256=Voli3Wv0YYLWyocNCpiBCchzfCxX4IYFNR4eOIlFyIM,7739
+visual_parser/nougat_engine.py,sha256=o9u9XTxOL1EAo63mVARVt4z1IxfDMpvjhMD4URfTx1k,8369
+visual_parser/ollama_local.py,sha256=YQi1oT4OZcuxrp2yCVN5wo49jl9k7f6dvVFGt108v8g,13883
+visual_parser/openai_gateway.py,sha256=XebNY3qOY1I1SdrLFqw9yeax2_IwOSZyVi_XYWfYXj0,3683
+visual_parser/pdf_tracker.py,sha256=LEyhZtD5H05-BrWW6opGpGk-qivoKS3Tkar8-1aT1gs,5378
+visual_parser/pipeline.py,sha256=2g30AxNATec8XWztVuIDDQmnz-gTle2iXG5lQOLbRg0,33776
+visual_parser/prompts.py,sha256=hdjDUZjF0A9eugm6qt4K5E1qnqMO1JE9_mTDAGS7sgg,11186
+visual_parser/text_extractor.py,sha256=UoipnlAMdgFthsTQUHAnbhOeVLc1BUR4nTEOaijFsUg,17501
+visual_parser/vision_llm.py,sha256=TXiitsDAMIdo9fbzYVKqd-0sGt-hqiJryMU481eo1SM,17429
+visual_parser-2.2.4.dist-info/licenses/LICENSE,sha256=OEJRtZkWILIqPizQxk1m2CjYi1czEKW15RKX6WgLVBE,11344
+visual_parser-2.2.4.dist-info/METADATA,sha256=Bg8PQBKDh1WLyCeZt2tMN23kyrwu_VEGCncHvRRL47U,18291
+visual_parser-2.2.4.dist-info/WHEEL,sha256=a-udL2m4V_6Yn8b-uKzWb12RjRxwwMo6ohYFM24tpaY,91
+visual_parser-2.2.4.dist-info/entry_points.txt,sha256=yypghqnxA-nSiyoDyhlg99SK9vfOzloFwUiiwRw4EWI,62
+visual_parser-2.2.4.dist-info/top_level.txt,sha256=_aaZ41uFRN_WX-616YvmcXIroNq4k4m6Ph6FGetxLYY,14
+visual_parser-2.2.4.dist-info/RECORD,,

{visual_parser-2.2.2.dist-info → visual_parser-2.2.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (84.0.0)
+Generator: setuptools (84.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

visual_parser-2.2.2.dist-info/RECORD

@@ -1,26 +0,0 @@
-visual_parser/__init__.py,sha256=A1YmiufPZJW4uHVo2hfds0NX0ixxg3fDU7DpfMnPU6E,858
-visual_parser/__main__.py,sha256=-p6rqjrnsjrtu71eFtmZdqFI1NgHXX9ASOnjuGqghNo,168
-visual_parser/cli.py,sha256=nU9SAnaWR9J3WzhKOgO1qHL-9kA1RKntR5L2pVz1zX8,16049
-visual_parser/cli_main.py,sha256=SdABWlUeGDNnVSzUZ8Nky5cxUvsxwJsMuzGliYJFYzk,11805
-visual_parser/config.py,sha256=mu52rGN1keQtho70IUP7kPuS6Wykpcc-gv3z0irg70c,12140
-visual_parser/figure_describer.py,sha256=q7Jsccsn5iGK4sCbsVOD4EzLuycbvBeBSYtHX0e9jPw,13470
-visual_parser/image_describer.py,sha256=nkwqK4aDcwYjuBLCjV2AmZShzCT_pdUkPZJLy2KaV9M,13859
-visual_parser/image_tracker.py,sha256=MyQiUyqnW6b7e7R7yIefJ0k15OO0WEkK5wMiRD2nmfU,2496
-visual_parser/jsonl_writer.py,sha256=klkDl_KKdCRtiWWjTNqecN2B6zXPc0e9f704Dvj8Fh0,6215
-visual_parser/kb_redo.py,sha256=xdp9Cd2xEz19x02ORsWvj5QlKzMyvxmM0pbSrPOla3k,5517
-visual_parser/metadata_extractor.py,sha256=Z0jnA8Goo9R3SBThx5PQMetZkyrbevNvYCq36WsoQuQ,6579
-visual_parser/model_catalog.py,sha256=Voli3Wv0YYLWyocNCpiBCchzfCxX4IYFNR4eOIlFyIM,7739
-visual_parser/nougat_engine.py,sha256=o9u9XTxOL1EAo63mVARVt4z1IxfDMpvjhMD4URfTx1k,8369
-visual_parser/ollama_local.py,sha256=YQi1oT4OZcuxrp2yCVN5wo49jl9k7f6dvVFGt108v8g,13883
-visual_parser/openai_gateway.py,sha256=XebNY3qOY1I1SdrLFqw9yeax2_IwOSZyVi_XYWfYXj0,3683
-visual_parser/pdf_tracker.py,sha256=VDn5TOFVATNIcTjLt3i-HK99mtMEsu0Y9IRZGPb-a-g,4616
-visual_parser/pipeline.py,sha256=Bh03FO3ylIsC1bfRITvffGzL-e5r07dKVzgHEiNRK6Y,30944
-visual_parser/prompts.py,sha256=hdjDUZjF0A9eugm6qt4K5E1qnqMO1JE9_mTDAGS7sgg,11186
-visual_parser/text_extractor.py,sha256=kMmjTvTnPJBsFTIv5qN49v88k1SJhQMtTob8_LT1tfE,17491
-visual_parser/vision_llm.py,sha256=TXiitsDAMIdo9fbzYVKqd-0sGt-hqiJryMU481eo1SM,17429
-visual_parser-2.2.2.dist-info/licenses/LICENSE,sha256=OEJRtZkWILIqPizQxk1m2CjYi1czEKW15RKX6WgLVBE,11344
-visual_parser-2.2.2.dist-info/METADATA,sha256=WJGuFZVprE2_nb1BdBpwDnDFpV1J78AJ_Ntz-mjYnK8,16422
-visual_parser-2.2.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-visual_parser-2.2.2.dist-info/entry_points.txt,sha256=yypghqnxA-nSiyoDyhlg99SK9vfOzloFwUiiwRw4EWI,62
-visual_parser-2.2.2.dist-info/top_level.txt,sha256=_aaZ41uFRN_WX-616YvmcXIroNq4k4m6Ph6FGetxLYY,14
-visual_parser-2.2.2.dist-info/RECORD,,