visual-parser 2.2.1__py3-none-any.whl → 2.2.2__py3-none-any.whl

visual_parser/__init__.py

@@ -17,4 +17,4 @@ from visual_parser.config import ParserConfig
 from visual_parser.pipeline import run_pipeline
 
 __all__ = ["ParserConfig", "run_pipeline"]
-__version__ = "2.2.1"
+__version__ = "2.2.2"

visual_parser/cli.py

@@ -111,6 +111,15 @@ def _build_arg_parser() -> argparse.ArgumentParser:
         metavar="N",
         help="Overlap characters between adjacent chunks (default: 100).",
     )
+    text_group.add_argument(
+        "--nougat-crop-margin",
+        action="store_true",
+        help=(
+            "Let Nougat autocrop whitespace margins (its own default). "
+            "Off by default -- autocropping can silently crop into real "
+            "content on unusual page layouts."
+        ),
+    )
 
     # ---- Vision LLM ---------------------------------------------------------
     vision_group = p.add_argument_group("Vision LLM (figure descriptions & metadata)")
@@ -366,6 +375,7 @@ def main(argv=None) -> int:
         nougat_model          = args.nougat_model,
         chunk_size            = args.chunk_size,
         chunk_overlap         = args.chunk_overlap,
+        nougat_crop_margin    = args.nougat_crop_margin,
         vision_provider       = args.vision_provider,
         gpt_vision_model      = args.vision_model if args.vision_provider == "gpt"    else "gpt-5.4",
         gemini_vision_model   = args.vision_model if args.vision_provider == "gemini" else "gemini-3.8-flash",

visual_parser/cli_main.py

@@ -109,6 +109,15 @@ def _build_arg_parser() -> argparse.ArgumentParser:
         metavar="N",
         help="Overlap characters between adjacent chunks (default: 100).",
     )
+    text_group.add_argument(
+        "--nougat-crop-margin",
+        action="store_true",
+        help=(
+            "Let Nougat autocrop whitespace margins (its own default). "
+            "Off by default -- autocropping can silently crop into real "
+            "content on unusual page layouts."
+        ),
+    )
 
     vision_group = parser.add_argument_group("Vision LLM (figure descriptions & metadata)")
     vision_group.add_argument(
@@ -282,6 +291,7 @@ def main(argv=None) -> int:
         nougat_model=args.nougat_model,
         chunk_size=args.chunk_size,
         chunk_overlap=args.chunk_overlap,
+        nougat_crop_margin=args.nougat_crop_margin,
         vision_provider=args.vision_provider,
         gpt_vision_model=args.vision_model if args.vision_provider == "gpt" else "gpt-5.4",
         gemini_vision_model=(

visual_parser/config.py

@@ -79,6 +79,14 @@ class ParserConfig:
     nougat_model: str = "facebook/nougat-small"
     """HuggingFace model identifier for Nougat."""
 
+    nougat_crop_margin: bool = False
+    """
+    Passed through to Nougat's image processor as do_crop_margin. False
+    (default) matches this package's long-standing behavior -- autocropping
+    whitespace margins can silently crop into real content on unusual page
+    layouts. Set True to opt into Nougat's own autocrop behavior instead.
+    """
+
     chunk_size: int = 500
     """Target character count per text chunk."""
 
@@ -186,6 +194,7 @@ class ParserConfig:
             output_dir           = os.getenv("VISUAL_PARSER_OUTPUT_DIR", ""),
             text_mode            = os.getenv("VISUAL_PARSER_TEXT_MODE", "nougat"),           # type: ignore[arg-type]
             nougat_model         = os.getenv("VISUAL_PARSER_NOUGAT_MODEL", "facebook/nougat-small"),
+            nougat_crop_margin   = os.getenv("VISUAL_PARSER_NOUGAT_CROP_MARGIN", "false").lower() == "true",
             chunk_size           = int(os.getenv("VISUAL_PARSER_CHUNK_SIZE", "500")),
             chunk_overlap        = int(os.getenv("VISUAL_PARSER_CHUNK_OVERLAP", "100")),
             vision_provider      = os.getenv("VISUAL_PARSER_VISION_PROVIDER", "gpt"),        # type: ignore[arg-type]

visual_parser/pipeline.py

@@ -501,6 +501,7 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
                     chunk_size         = config.chunk_size,
                     chunk_overlap      = config.chunk_overlap,
                     max_workers        = config.max_workers,
+                    nougat_crop_margin = config.nougat_crop_margin,
                 )
                 print(nougat_summary)
             except Exception as exc:

visual_parser/text_extractor.py

@@ -69,6 +69,7 @@ def nougat_extract_pdfs(
     chunk_size: int = 500,
     chunk_overlap: int = 100,
     max_workers: int = 4,
+    nougat_crop_margin: bool = False,
 ) -> Tuple[str, List[str], List[str], int]:
     """
     Extract text from each PDF in *only_process_these* using the Nougat model,
@@ -83,6 +84,9 @@ def nougat_extract_pdfs(
         chunk_size:         Characters per chunk.
         chunk_overlap:      Overlap between adjacent chunks.
         max_workers:        Thread-pool size for parallel PDF processing.
+        nougat_crop_margin: Passed through to Nougat's image processor as
+                            do_crop_margin (default False -- see
+                            ParserConfig.nougat_crop_margin's docstring).
 
     Always resumes against the 04_processed_pdfs.txt registry -- callers
     that want a PDF reprocessed must first clear its existing chunks and
@@ -124,7 +128,7 @@ def nougat_extract_pdfs(
                     pixel_values = processor(
                         images=image,
                         return_tensors="pt",
-                        do_crop_margin=False,
+                        do_crop_margin=nougat_crop_margin,
                     ).pixel_values.to(device)
                 except TypeError:
                     pixel_values = processor(

{visual_parser-2.2.1.dist-info → visual_parser-2.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: visual-parser
-Version: 2.2.1
+Version: 2.2.2
 Summary: Standalone Visual-RAG PDF Parser - text extraction and Vision-LLM figure descriptions to JSONL
 Author: Zavier N. Ndum
 Author-email: zavier.ndum@tamu.edu
@@ -50,7 +50,7 @@ Dynamic: license-file
 ![LangChain](https://img.shields.io/badge/LangChain-0.1%2B-1C3C3C.svg)
 ![CUDA](https://img.shields.io/badge/CUDA-optional-76B900.svg)
 
-`visual-parser` converts PDFs into a multi-modal JSONL knowledge base (text chunks, figure descriptions, metadata). It was extracted from [RADIANT-LLM](https://github.com/SmartLabNuclear/RADIANT_LLM) as a standalone PDF-ingestion tool, independent of any chatbot.
+`visual-parser` converts PDFs (and standalone images alongside them) into a multi-modal JSONL knowledge base (text chunks, figure descriptions, standalone image descriptions, metadata). It was extracted from [RADIANT-LLM](https://github.com/SmartLabNuclear/RADIANT_LLM) as a standalone PDF-ingestion tool, independent of any chatbot.
 
 1) Run `visual-parser` on curated PDFs to generate JSONL KB files.
 2) Point any downstream RAG system at the generated KB — RADIANT-LLM, AutoSAM, and AutoFLUKA all consume the identical JSONL/registry format, so the same KB works with any of them without re-parsing.
@@ -72,10 +72,11 @@ Dynamic: license-file
 
 By default, the pipeline writes:
 - `01_chunks_kb.jsonl`: chunked text extracted from PDFs (Nougat by default).
-- `02_visuals_kb.jsonl`: figure/page visual descriptions (Vision LLM).
+- `02_visuals_kb.jsonl`: figure/page visual descriptions (Vision LLM), one row per figure found in a PDF.
 - `03_metadata_kb.jsonl`: document metadata rows (title/author/etc.).
+- `image_descriptions.jsonl`: one holistic description per **standalone image** file found alongside your PDFs in `--input-dir` (not a PDF's own figures) — auto-detected by extension, no separate flag needed; disable with `--skip-images`.
 
-Alongside the KB, the pipeline also writes two bookkeeping files: `04_processed_pdfs.txt` (tracks which PDFs have already been processed, so re-runs skip them unless `--rebuild`) and `05_pipeline.log` (run log at the verbosity set by `--log-level`, default `ERROR`).
+Alongside the KB, the pipeline also writes bookkeeping files: `04_processed_pdfs.txt` / `parsed_images.txt` (track which PDFs/images have already been processed, so re-runs skip them unless `--rebuild` or `--redo NAME`) and `05_pipeline.log` (run log at the verbosity set by `--log-level`, default `ERROR`).
 
 ## API Keys (.env)
 
@@ -262,6 +263,7 @@ Text extraction:
 - `--nougat-model facebook/nougat-small`
 - `--chunk-size 500`
 - `--chunk-overlap 100`
+- `--nougat-crop-margin` (off by default — lets Nougat autocrop whitespace margins instead of this package's long-standing uncropped behavior; autocropping can silently crop into real content on unusual page layouts)
 
 Vision LLM:
 - `--vision-provider gpt|gemini|ollama` (default: `gpt`) — see [Local Ollama vision provider](#local-ollama-vision-provider) for the `ollama` option
@@ -273,8 +275,10 @@ Vision LLM:
 
 Performance / misc:
 - `--max-workers 4`
-- `--rebuild` (reprocess everything; ignore `04_processed_pdfs.txt`)
+- `--rebuild` (reprocess everything; ignore `04_processed_pdfs.txt`/`parsed_images.txt`)
+- `--redo NAME` (surgically clear one PDF/image's rows from every KB file + both registries, so it's reprocessed fresh on the next run — a targeted alternative to `--rebuild`)
 - `--skip-text` (skip text extraction and resume only the vision steps; use after an interrupted run)
+- `--skip-images` (don't process standalone images in `--input-dir`, PDFs only)
 - `--list-models` — print the live, currently-available vision models for whichever provider(s) you have a key configured for, then exit (no PDF processing). Reflects Portkey's catalog too when that's active.
 - `--version` / `-V`
 - `--log-level DEBUG|INFO|WARNING|ERROR` (default: `ERROR`)

{visual_parser-2.2.1.dist-info → visual_parser-2.2.2.dist-info}/RECORD

@@ -1,8 +1,8 @@
-visual_parser/__init__.py,sha256=cu_RIM6pcI2Ra2diZ--9c58H84CfGJIMpXA6RkFEzLY,858
+visual_parser/__init__.py,sha256=A1YmiufPZJW4uHVo2hfds0NX0ixxg3fDU7DpfMnPU6E,858
 visual_parser/__main__.py,sha256=-p6rqjrnsjrtu71eFtmZdqFI1NgHXX9ASOnjuGqghNo,168
-visual_parser/cli.py,sha256=6E-e_XYBOJkKUaW4l1BekSs6jEeV0vJEU3FF7x_BSRw,15666
-visual_parser/cli_main.py,sha256=QwIos1Fn06hNbJYPt86Uzirgmc8Ts2-IS1MEOKERuas,11437
-visual_parser/config.py,sha256=KHD8YGg00FsMMRR8G4wKuSCLbKnyAAiAHpxSEaFC_TQ,11671
+visual_parser/cli.py,sha256=nU9SAnaWR9J3WzhKOgO1qHL-9kA1RKntR5L2pVz1zX8,16049
+visual_parser/cli_main.py,sha256=SdABWlUeGDNnVSzUZ8Nky5cxUvsxwJsMuzGliYJFYzk,11805
+visual_parser/config.py,sha256=mu52rGN1keQtho70IUP7kPuS6Wykpcc-gv3z0irg70c,12140
 visual_parser/figure_describer.py,sha256=q7Jsccsn5iGK4sCbsVOD4EzLuycbvBeBSYtHX0e9jPw,13470
 visual_parser/image_describer.py,sha256=nkwqK4aDcwYjuBLCjV2AmZShzCT_pdUkPZJLy2KaV9M,13859
 visual_parser/image_tracker.py,sha256=MyQiUyqnW6b7e7R7yIefJ0k15OO0WEkK5wMiRD2nmfU,2496
@@ -14,13 +14,13 @@ visual_parser/nougat_engine.py,sha256=o9u9XTxOL1EAo63mVARVt4z1IxfDMpvjhMD4URfTx1
 visual_parser/ollama_local.py,sha256=YQi1oT4OZcuxrp2yCVN5wo49jl9k7f6dvVFGt108v8g,13883
 visual_parser/openai_gateway.py,sha256=XebNY3qOY1I1SdrLFqw9yeax2_IwOSZyVi_XYWfYXj0,3683
 visual_parser/pdf_tracker.py,sha256=VDn5TOFVATNIcTjLt3i-HK99mtMEsu0Y9IRZGPb-a-g,4616
-visual_parser/pipeline.py,sha256=iC13zOgvbpYJEwzLJU7kY7duOOsOI80ZkiseLMIVaqg,30875
+visual_parser/pipeline.py,sha256=Bh03FO3ylIsC1bfRITvffGzL-e5r07dKVzgHEiNRK6Y,30944
 visual_parser/prompts.py,sha256=hdjDUZjF0A9eugm6qt4K5E1qnqMO1JE9_mTDAGS7sgg,11186
-visual_parser/text_extractor.py,sha256=YN8HvEsLizIw4fXLpUPyhQs9ty3Zehw1apdXlUl_M7s,17223
+visual_parser/text_extractor.py,sha256=kMmjTvTnPJBsFTIv5qN49v88k1SJhQMtTob8_LT1tfE,17491
 visual_parser/vision_llm.py,sha256=TXiitsDAMIdo9fbzYVKqd-0sGt-hqiJryMU481eo1SM,17429
-visual_parser-2.2.1.dist-info/licenses/LICENSE,sha256=OEJRtZkWILIqPizQxk1m2CjYi1czEKW15RKX6WgLVBE,11344
-visual_parser-2.2.1.dist-info/METADATA,sha256=7yasa5foMzteTD4vwTgV5nOZ1TQWi3CGYuCBRzxy82E,15534
-visual_parser-2.2.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-visual_parser-2.2.1.dist-info/entry_points.txt,sha256=yypghqnxA-nSiyoDyhlg99SK9vfOzloFwUiiwRw4EWI,62
-visual_parser-2.2.1.dist-info/top_level.txt,sha256=_aaZ41uFRN_WX-616YvmcXIroNq4k4m6Ph6FGetxLYY,14
-visual_parser-2.2.1.dist-info/RECORD,,
+visual_parser-2.2.2.dist-info/licenses/LICENSE,sha256=OEJRtZkWILIqPizQxk1m2CjYi1czEKW15RKX6WgLVBE,11344
+visual_parser-2.2.2.dist-info/METADATA,sha256=WJGuFZVprE2_nb1BdBpwDnDFpV1J78AJ_Ntz-mjYnK8,16422
+visual_parser-2.2.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+visual_parser-2.2.2.dist-info/entry_points.txt,sha256=yypghqnxA-nSiyoDyhlg99SK9vfOzloFwUiiwRw4EWI,62
+visual_parser-2.2.2.dist-info/top_level.txt,sha256=_aaZ41uFRN_WX-616YvmcXIroNq4k4m6Ph6FGetxLYY,14
+visual_parser-2.2.2.dist-info/RECORD,,