visual-parser 2.2.3__py3-none-any.whl → 2.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- visual_parser/__init__.py +1 -1
- visual_parser/figure_describer.py +49 -11
- visual_parser/image_describer.py +9 -5
- visual_parser/image_tracker.py +4 -7
- visual_parser/pdf_tracker.py +14 -2
- visual_parser/pipeline.py +62 -6
- visual_parser/text_extractor.py +3 -3
- {visual_parser-2.2.3.dist-info → visual_parser-2.2.4.dist-info}/METADATA +4 -2
- {visual_parser-2.2.3.dist-info → visual_parser-2.2.4.dist-info}/RECORD +13 -13
- {visual_parser-2.2.3.dist-info → visual_parser-2.2.4.dist-info}/WHEEL +1 -1
- {visual_parser-2.2.3.dist-info → visual_parser-2.2.4.dist-info}/entry_points.txt +0 -0
- {visual_parser-2.2.3.dist-info → visual_parser-2.2.4.dist-info}/licenses/LICENSE +0 -0
- {visual_parser-2.2.3.dist-info → visual_parser-2.2.4.dist-info}/top_level.txt +0 -0
visual_parser/__init__.py
CHANGED
|
@@ -16,7 +16,9 @@ appended to ``02_visuals_kb.jsonl`` in *output_dir*:
|
|
|
16
16
|
"document_id": "a1b2c3d4e5f6g7h8",
|
|
17
17
|
"figure_index": 0,
|
|
18
18
|
"figure_id": "a1b2c3d4e5f6g7h8:p3:f0",
|
|
19
|
-
"description": "**Subject:** ..."
|
|
19
|
+
"description": "**Subject:** ...",
|
|
20
|
+
"vision_provider": "gpt",
|
|
21
|
+
"vision_model": "gpt-5.4"
|
|
20
22
|
}
|
|
21
23
|
"""
|
|
22
24
|
|
|
@@ -88,7 +90,7 @@ def describe_figures_for_new_pdfs(
|
|
|
88
90
|
reasoning_effort: Optional[str] = "medium",
|
|
89
91
|
max_workers: int = 4,
|
|
90
92
|
context_pages: int = 0,
|
|
91
|
-
) ->
|
|
93
|
+
) -> List[str]:
|
|
92
94
|
"""
|
|
93
95
|
For each PDF in *new_pdf_paths*, rasterise every page at *raster_dpi* DPI,
|
|
94
96
|
call the Vision LLM for each page (concurrently, up to *max_workers* at
|
|
@@ -124,6 +126,21 @@ def describe_figures_for_new_pdfs(
|
|
|
124
126
|
its basename from the processed-PDFs registry, then pass it in here as
|
|
125
127
|
if new; this function has no bypass of its own, so there is no path to
|
|
126
128
|
producing a duplicate row.
|
|
129
|
+
|
|
130
|
+
Each written row also carries ``vision_provider``/``vision_model``, so a
|
|
131
|
+
document resumed later under a different model is detectable. Detection
|
|
132
|
+
is active, not just recorded after the fact: if a PDF already has rows
|
|
133
|
+
from a *different* vision_model than this call's, its remaining pages
|
|
134
|
+
are skipped entirely (not mixed in under the new model) and its basename
|
|
135
|
+
is returned, so a resumed document never silently ends up with figures
|
|
136
|
+
described by two different models. Pre-upgrade rows with no recorded
|
|
137
|
+
vision_model are treated as unknown, not a mismatch, and do not block.
|
|
138
|
+
Callers that want the document redescribed consistently under the new
|
|
139
|
+
model must first clear it (see kb_redo.redo_entries(), e.g. via --redo).
|
|
140
|
+
|
|
141
|
+
Returns:
|
|
142
|
+
Basenames skipped in full due to a detected vision_model mismatch
|
|
143
|
+
against their existing rows (empty list when none were skipped).
|
|
127
144
|
"""
|
|
128
145
|
# -----------------------------------------------------------------------
|
|
129
146
|
# Step 1 – Rasterise every page of every new PDF
|
|
@@ -147,7 +164,7 @@ def describe_figures_for_new_pdfs(
|
|
|
147
164
|
|
|
148
165
|
if not page_images:
|
|
149
166
|
logger.info("No pages to describe (all PDFs failed to rasterise).")
|
|
150
|
-
return
|
|
167
|
+
return []
|
|
151
168
|
|
|
152
169
|
# -----------------------------------------------------------------------
|
|
153
170
|
# Step 2 – Group page images by PDF name. Order within each group is
|
|
@@ -165,6 +182,7 @@ def describe_figures_for_new_pdfs(
|
|
|
165
182
|
# -----------------------------------------------------------------------
|
|
166
183
|
figures_path = os.path.join(output_dir, "02_visuals_kb.jsonl")
|
|
167
184
|
done_pages: set = set()
|
|
185
|
+
prior_models_by_pdf: Dict[str, set] = defaultdict(set)
|
|
168
186
|
if os.path.exists(figures_path):
|
|
169
187
|
with open(figures_path, encoding="utf-8") as _fh:
|
|
170
188
|
for _line in _fh:
|
|
@@ -177,6 +195,9 @@ def describe_figures_for_new_pdfs(
|
|
|
177
195
|
_pg = _rec.get("page")
|
|
178
196
|
if _src and _pg is not None:
|
|
179
197
|
done_pages.add((_src, _pg))
|
|
198
|
+
_prior_model = _rec.get("vision_model")
|
|
199
|
+
if _src and _prior_model:
|
|
200
|
+
prior_models_by_pdf[_src].add(_prior_model)
|
|
180
201
|
except Exception:
|
|
181
202
|
pass
|
|
182
203
|
if done_pages:
|
|
@@ -191,7 +212,20 @@ def describe_figures_for_new_pdfs(
|
|
|
191
212
|
# never crosses into a different PDF.
|
|
192
213
|
# -----------------------------------------------------------------------
|
|
193
214
|
tasks: List[Dict[str, Any]] = []
|
|
215
|
+
model_mismatch_skipped: List[str] = []
|
|
194
216
|
for pdf_name, image_records in pages_by_pdf.items():
|
|
217
|
+
prior_models = prior_models_by_pdf.get(pdf_name)
|
|
218
|
+
if prior_models and vision_model not in prior_models:
|
|
219
|
+
msg = (
|
|
220
|
+
f"[FIGURES] Skipping {pdf_name}: already has figures described with "
|
|
221
|
+
f"{', '.join(sorted(prior_models))}, but this run is using {vision_model}. "
|
|
222
|
+
f"Run with --redo {pdf_name} to reprocess it entirely under the new model."
|
|
223
|
+
)
|
|
224
|
+
logger.error(msg)
|
|
225
|
+
print(msg)
|
|
226
|
+
model_mismatch_skipped.append(pdf_name)
|
|
227
|
+
continue
|
|
228
|
+
|
|
195
229
|
for idx, record in enumerate(image_records):
|
|
196
230
|
page_number = record["page"]
|
|
197
231
|
if (pdf_name, page_number) in done_pages:
|
|
@@ -213,7 +247,7 @@ def describe_figures_for_new_pdfs(
|
|
|
213
247
|
|
|
214
248
|
if not tasks:
|
|
215
249
|
logger.info("No new pages to describe — all already processed.")
|
|
216
|
-
return
|
|
250
|
+
return model_mismatch_skipped
|
|
217
251
|
|
|
218
252
|
# -----------------------------------------------------------------------
|
|
219
253
|
# Step 4 – Call Vision LLM per task, concurrently. Each worker only
|
|
@@ -265,12 +299,14 @@ def describe_figures_for_new_pdfs(
|
|
|
265
299
|
)
|
|
266
300
|
continue
|
|
267
301
|
page_rows.append({
|
|
268
|
-
"source":
|
|
269
|
-
"page":
|
|
270
|
-
"document_id":
|
|
271
|
-
"figure_index":
|
|
272
|
-
"figure_id":
|
|
273
|
-
"description":
|
|
302
|
+
"source": pdf_name,
|
|
303
|
+
"page": page_number,
|
|
304
|
+
"document_id": document_id,
|
|
305
|
+
"figure_index": fig_idx,
|
|
306
|
+
"figure_id": f"{document_id}:p{page_number}:f{fig_idx}",
|
|
307
|
+
"description": description,
|
|
308
|
+
"vision_provider": vision_provider,
|
|
309
|
+
"vision_model": vision_model,
|
|
274
310
|
})
|
|
275
311
|
return {"pdf_name": pdf_name, "page_number": page_number, "rows": page_rows}
|
|
276
312
|
|
|
@@ -306,10 +342,12 @@ def describe_figures_for_new_pdfs(
|
|
|
306
342
|
for pdf_name, count in written_by_pdf.items():
|
|
307
343
|
logger.info("[FIGURES] %s: %d figure(s) written.", pdf_name, count)
|
|
308
344
|
for pdf_name in pages_by_pdf:
|
|
309
|
-
if pdf_name not in written_by_pdf:
|
|
345
|
+
if pdf_name not in written_by_pdf and pdf_name not in model_mismatch_skipped:
|
|
310
346
|
logger.info("[FIGURES] %s: no new figures (all pages done or none detected).", pdf_name)
|
|
311
347
|
|
|
312
348
|
if total_written:
|
|
313
349
|
print(f"[FIGURES] Wrote {total_written} figure record(s) to 02_visuals_kb.jsonl.")
|
|
314
350
|
else:
|
|
315
351
|
logger.info("No new figures written — all pages already processed or none detected.")
|
|
352
|
+
|
|
353
|
+
return model_mismatch_skipped
|
visual_parser/image_describer.py
CHANGED
|
@@ -17,7 +17,9 @@ One record per image is appended to ``image_descriptions_kb.jsonl`` in
|
|
|
17
17
|
"source": "diagram1.png",
|
|
18
18
|
"image_id": "a1b2c3d4e5f6g7h8",
|
|
19
19
|
"title": "Vertical Parabolic Gate Schematic",
|
|
20
|
-
"description": "**Subject:** ..."
|
|
20
|
+
"description": "**Subject:** ...",
|
|
21
|
+
"vision_provider": "gpt",
|
|
22
|
+
"vision_model": "gpt-5.4"
|
|
21
23
|
}
|
|
22
24
|
"""
|
|
23
25
|
|
|
@@ -279,10 +281,12 @@ def describe_images(
|
|
|
279
281
|
return {"image_name": image_name, "row": None}
|
|
280
282
|
|
|
281
283
|
row = {
|
|
282
|
-
"source":
|
|
283
|
-
"image_id":
|
|
284
|
-
"title":
|
|
285
|
-
"description":
|
|
284
|
+
"source": image_name,
|
|
285
|
+
"image_id": make_document_id(image_name),
|
|
286
|
+
"title": _derive_title(description),
|
|
287
|
+
"description": description,
|
|
288
|
+
"vision_provider": vision_provider,
|
|
289
|
+
"vision_model": vision_model,
|
|
286
290
|
}
|
|
287
291
|
return {"image_name": image_name, "row": row}
|
|
288
292
|
|
visual_parser/image_tracker.py
CHANGED
|
@@ -35,13 +35,10 @@ def find_new_images(
|
|
|
35
35
|
Walk *input_dir* recursively and return full paths of standalone images
|
|
36
36
|
that have NOT yet been processed.
|
|
37
37
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
anchor the registry at effective_output_dir()
|
|
41
|
-
|
|
42
|
-
its registry against input_dir while pipeline.py separately writes it
|
|
43
|
-
against output_dir (the two diverge, and PDFs get reprocessed forever,
|
|
44
|
-
whenever --output-dir differs from --input-dir).
|
|
38
|
+
*registry_path* is an explicit required parameter (rather than a filename
|
|
39
|
+
joined internally against *input_dir*, as find_new_pdfs() used to do by
|
|
40
|
+
default) so the caller can anchor the registry at effective_output_dir()
|
|
41
|
+
unambiguously. find_new_pdfs() now accepts the same explicit override.
|
|
45
42
|
|
|
46
43
|
Args:
|
|
47
44
|
input_dir: Root directory to search for image files.
|
visual_parser/pdf_tracker.py
CHANGED
|
@@ -96,6 +96,8 @@ def find_new_pdfs(
|
|
|
96
96
|
input_dir: str,
|
|
97
97
|
registry_filename: str = PROCESSED_REGISTRY,
|
|
98
98
|
rebuild: bool = False,
|
|
99
|
+
*,
|
|
100
|
+
registry_path: str | None = None,
|
|
99
101
|
) -> List[str]:
|
|
100
102
|
"""
|
|
101
103
|
Walk *input_dir* recursively and return full paths of PDFs that have NOT
|
|
@@ -103,14 +105,24 @@ def find_new_pdfs(
|
|
|
103
105
|
|
|
104
106
|
Args:
|
|
105
107
|
input_dir: Root directory to search for ``.pdf`` files.
|
|
106
|
-
registry_filename: Name of the tracking file inside *input_dir
|
|
108
|
+
registry_filename: Name of the tracking file inside *input_dir*,
|
|
109
|
+
used only when *registry_path* is not given.
|
|
107
110
|
rebuild: When True, return *all* PDFs regardless of the
|
|
108
111
|
registry (forces a full re-parse).
|
|
112
|
+
registry_path: Full path to the tracking file. When omitted,
|
|
113
|
+
defaults to ``input_dir/registry_filename`` (the
|
|
114
|
+
historical behavior, kept for callers -- such as
|
|
115
|
+
RADIANT-LLM -- that use the same folder for both
|
|
116
|
+
input and output). Callers whose output directory
|
|
117
|
+
differs from their input directory must pass the
|
|
118
|
+
registry's real location explicitly here, the
|
|
119
|
+
same way find_new_images() already requires.
|
|
109
120
|
|
|
110
121
|
Returns:
|
|
111
122
|
Sorted list of absolute PDF paths.
|
|
112
123
|
"""
|
|
113
|
-
registry_path
|
|
124
|
+
if registry_path is None:
|
|
125
|
+
registry_path = os.path.join(input_dir, registry_filename)
|
|
114
126
|
processed = set() if rebuild else set(load_processed_pdfs(registry_path))
|
|
115
127
|
|
|
116
128
|
new_pdfs = [
|
visual_parser/pipeline.py
CHANGED
|
@@ -26,7 +26,8 @@ from __future__ import annotations
|
|
|
26
26
|
|
|
27
27
|
import logging
|
|
28
28
|
import os
|
|
29
|
-
from
|
|
29
|
+
from datetime import datetime, timezone
|
|
30
|
+
from typing import Dict, List, Optional, Tuple
|
|
30
31
|
|
|
31
32
|
from visual_parser.config import ParserConfig
|
|
32
33
|
from visual_parser.figure_describer import describe_figures_for_new_pdfs
|
|
@@ -60,6 +61,41 @@ def _setup_logging(config: ParserConfig) -> None:
|
|
|
60
61
|
logging.getLogger().addHandler(console)
|
|
61
62
|
|
|
62
63
|
|
|
64
|
+
def _build_parser_block(
|
|
65
|
+
config: ParserConfig,
|
|
66
|
+
vision_model: str,
|
|
67
|
+
text_engine: str,
|
|
68
|
+
text_fallback: bool,
|
|
69
|
+
) -> Dict:
|
|
70
|
+
"""
|
|
71
|
+
Build the per-document ``parser`` provenance block attached to each
|
|
72
|
+
03_metadata_kb.jsonl row -- records which engine/model actually produced
|
|
73
|
+
this document's rows, since the configured settings aren't always what
|
|
74
|
+
ran (Nougat can fail and fall back to lightweight; Ollama can auto-select
|
|
75
|
+
a model at run time). New field only; does not change any existing KB
|
|
76
|
+
field or ID.
|
|
77
|
+
"""
|
|
78
|
+
from visual_parser import __version__
|
|
79
|
+
|
|
80
|
+
block = {
|
|
81
|
+
"visual_parser": __version__,
|
|
82
|
+
"parsed_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
83
|
+
"text_engine": text_engine,
|
|
84
|
+
"text_fallback": text_fallback,
|
|
85
|
+
"vision_provider": config.vision_provider,
|
|
86
|
+
"vision_model": vision_model,
|
|
87
|
+
"vision_detail": config.vision_detail,
|
|
88
|
+
"reasoning_effort": config.gpt_reasoning_effort,
|
|
89
|
+
"metadata_pages": config.metadata_pages,
|
|
90
|
+
"vision_context_pages": config.vision_context_pages,
|
|
91
|
+
"chunk_size": config.chunk_size,
|
|
92
|
+
"chunk_overlap": config.chunk_overlap,
|
|
93
|
+
}
|
|
94
|
+
if text_engine == "nougat":
|
|
95
|
+
block["nougat_model"] = config.nougat_model
|
|
96
|
+
return block
|
|
97
|
+
|
|
98
|
+
|
|
63
99
|
def _recover_stale_registry_entries(input_dir: str, output_dir: str, registry_path: str) -> int:
|
|
64
100
|
"""
|
|
65
101
|
Remove processed-PDF registry entries for files that have no chunk records.
|
|
@@ -134,6 +170,7 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
134
170
|
"redone_basenames": List[str],
|
|
135
171
|
"redo_names_not_found": List[str],
|
|
136
172
|
"processed_basenames": List[str],
|
|
173
|
+
"figure_model_mismatches": List[str],
|
|
137
174
|
}
|
|
138
175
|
"""
|
|
139
176
|
if config is None:
|
|
@@ -155,6 +192,7 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
155
192
|
"failed_basenames": [],
|
|
156
193
|
"redone_basenames": [],
|
|
157
194
|
"redo_names_not_found": [],
|
|
195
|
+
"figure_model_mismatches": [],
|
|
158
196
|
"status": "success",
|
|
159
197
|
}
|
|
160
198
|
|
|
@@ -174,7 +212,10 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
174
212
|
# -----------------------------------------------------------------------
|
|
175
213
|
redo_name_set = set(config.redo_names)
|
|
176
214
|
if config.rebuild:
|
|
177
|
-
redo_name_set |= {
|
|
215
|
+
redo_name_set |= {
|
|
216
|
+
os.path.basename(p)
|
|
217
|
+
for p in find_new_pdfs(config.input_dir, rebuild=True, registry_path=registry_path)
|
|
218
|
+
}
|
|
178
219
|
redo_name_set |= {
|
|
179
220
|
os.path.basename(p)
|
|
180
221
|
for p in find_new_images(config.input_dir, images_registry_path, rebuild=True)
|
|
@@ -384,7 +425,7 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
384
425
|
) if os.path.exists(figures_path) else 0
|
|
385
426
|
|
|
386
427
|
print(f"[Step 2] Describing figures for {len(all_pdfs)} PDF(s) (page-level resume active) …")
|
|
387
|
-
describe_figures_for_new_pdfs(
|
|
428
|
+
summary["figure_model_mismatches"] = describe_figures_for_new_pdfs(
|
|
388
429
|
new_pdf_paths = all_pdfs,
|
|
389
430
|
output_dir = output_dir,
|
|
390
431
|
vision_provider = config.vision_provider,
|
|
@@ -417,6 +458,7 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
417
458
|
row = {"source": source, "document_id": document_id}
|
|
418
459
|
if isinstance(meta, dict):
|
|
419
460
|
row.update(meta)
|
|
461
|
+
row["parser"] = _build_parser_block(config, _vision_model, "skipped", False)
|
|
420
462
|
metadata_rows.append(row)
|
|
421
463
|
|
|
422
464
|
if metadata_rows:
|
|
@@ -432,13 +474,13 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
432
474
|
else:
|
|
433
475
|
# ── Normal (full) pipeline ───────────────────────────────────────────
|
|
434
476
|
|
|
435
|
-
new_pdfs = find_new_pdfs(config.input_dir, rebuild=False)
|
|
477
|
+
new_pdfs = find_new_pdfs(config.input_dir, rebuild=False, registry_path=registry_path)
|
|
436
478
|
summary["new_pdfs_found"] = len(new_pdfs)
|
|
437
479
|
|
|
438
480
|
if not new_pdfs:
|
|
439
481
|
recovered = _recover_stale_registry_entries(config.input_dir, output_dir, registry_path)
|
|
440
482
|
if recovered:
|
|
441
|
-
new_pdfs = find_new_pdfs(config.input_dir, rebuild=False)
|
|
483
|
+
new_pdfs = find_new_pdfs(config.input_dir, rebuild=False, registry_path=registry_path)
|
|
442
484
|
summary["new_pdfs_found"] = len(new_pdfs)
|
|
443
485
|
|
|
444
486
|
if not new_pdfs and not images_to_process:
|
|
@@ -482,7 +524,11 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
482
524
|
# Step 1 — Text extraction and chunking. Guarded by `if new_pdfs:` so
|
|
483
525
|
# an image-only pass (possible now that images are tracked/processed
|
|
484
526
|
# independently of PDFs) never needlessly loads Nougat for zero PDFs.
|
|
527
|
+
# text_engine_by_basename records which engine actually produced each
|
|
528
|
+
# PDF's chunks (engine, text_fallback) -- the configured text_mode
|
|
529
|
+
# isn't always what ran, since Nougat can fail and fall back.
|
|
485
530
|
processor = model = device = None # only bound by the "nougat" branch below
|
|
531
|
+
text_engine_by_basename: Dict[str, Tuple[str, bool]] = {}
|
|
486
532
|
if not new_pdfs:
|
|
487
533
|
processed_basenames, failed_basenames, chunk_count = [], [], 0
|
|
488
534
|
elif config.text_mode == "nougat":
|
|
@@ -504,6 +550,8 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
504
550
|
nougat_crop_margin = config.nougat_crop_margin,
|
|
505
551
|
)
|
|
506
552
|
print(nougat_summary)
|
|
553
|
+
for _name in processed_basenames:
|
|
554
|
+
text_engine_by_basename[_name] = ("nougat", False)
|
|
507
555
|
except Exception as exc:
|
|
508
556
|
logger.error("Nougat initialization/extraction failed: %s", exc)
|
|
509
557
|
print(f"[Step 1] Nougat failed ({exc}). Falling back to lightweight extraction.")
|
|
@@ -515,6 +563,8 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
515
563
|
max_workers = config.max_workers,
|
|
516
564
|
)
|
|
517
565
|
print(lw_summary)
|
|
566
|
+
for _name in processed_basenames:
|
|
567
|
+
text_engine_by_basename[_name] = ("lightweight", True)
|
|
518
568
|
else:
|
|
519
569
|
if failed_basenames:
|
|
520
570
|
failed_set = set(failed_basenames)
|
|
@@ -534,6 +584,8 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
534
584
|
processed_basenames.extend(lw_processed)
|
|
535
585
|
failed_basenames = [name for name in failed_basenames if name not in set(lw_processed)]
|
|
536
586
|
chunk_count += lw_chunk_count
|
|
587
|
+
for _name in lw_processed:
|
|
588
|
+
text_engine_by_basename[_name] = ("lightweight", True)
|
|
537
589
|
|
|
538
590
|
else: # "lightweight"
|
|
539
591
|
print("[Step 1] Running lightweight (PyMuPDF) text extraction …")
|
|
@@ -547,6 +599,8 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
547
599
|
max_workers = config.max_workers,
|
|
548
600
|
)
|
|
549
601
|
print(lw_summary)
|
|
602
|
+
for _name in processed_basenames:
|
|
603
|
+
text_engine_by_basename[_name] = ("lightweight", False)
|
|
550
604
|
|
|
551
605
|
summary["processed_basenames"] = processed_basenames
|
|
552
606
|
summary["failed_basenames"] = failed_basenames
|
|
@@ -587,7 +641,7 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
587
641
|
|
|
588
642
|
if pdfs_for_figures:
|
|
589
643
|
print(f"[Step 2] Describing figures in {len(pdfs_for_figures)} PDF(s) …")
|
|
590
|
-
describe_figures_for_new_pdfs(
|
|
644
|
+
summary["figure_model_mismatches"] = describe_figures_for_new_pdfs(
|
|
591
645
|
new_pdf_paths = pdfs_for_figures,
|
|
592
646
|
output_dir = output_dir,
|
|
593
647
|
vision_provider = config.vision_provider,
|
|
@@ -621,6 +675,8 @@ def run_pipeline(config: Optional[ParserConfig] = None) -> Dict:
|
|
|
621
675
|
row = {"source": source, "document_id": document_id}
|
|
622
676
|
if isinstance(meta, dict):
|
|
623
677
|
row.update(meta)
|
|
678
|
+
text_engine, text_fallback = text_engine_by_basename.get(source, ("unknown", False))
|
|
679
|
+
row["parser"] = _build_parser_block(config, _vision_model, text_engine, text_fallback)
|
|
624
680
|
metadata_rows.append(row)
|
|
625
681
|
|
|
626
682
|
if metadata_rows:
|
visual_parser/text_extractor.py
CHANGED
|
@@ -34,7 +34,7 @@ import pymupdf as fitz # PyMuPDF -- "fitz" is the deprecated legacy import name
|
|
|
34
34
|
from PIL import Image
|
|
35
35
|
|
|
36
36
|
from visual_parser.jsonl_writer import append_to_jsonl, make_document_id
|
|
37
|
-
from visual_parser.pdf_tracker import load_processed_pdfs
|
|
37
|
+
from visual_parser.pdf_tracker import PROCESSED_REGISTRY, load_processed_pdfs
|
|
38
38
|
|
|
39
39
|
logger = logging.getLogger(__name__)
|
|
40
40
|
|
|
@@ -101,7 +101,7 @@ def nougat_extract_pdfs(
|
|
|
101
101
|
|
|
102
102
|
from visual_parser.nougat_engine import RasterizePaper, StoppingCriteriaScores
|
|
103
103
|
|
|
104
|
-
registry_path = os.path.join(output_dir,
|
|
104
|
+
registry_path = os.path.join(output_dir, PROCESSED_REGISTRY)
|
|
105
105
|
processed_set = set(load_processed_pdfs(registry_path))
|
|
106
106
|
pdfs_to_run = [p for p in only_process_these if os.path.basename(p) not in processed_set]
|
|
107
107
|
|
|
@@ -284,7 +284,7 @@ def lightweight_extract_pdfs(
|
|
|
284
284
|
"""
|
|
285
285
|
from io import BytesIO
|
|
286
286
|
|
|
287
|
-
registry_path = os.path.join(output_dir,
|
|
287
|
+
registry_path = os.path.join(output_dir, PROCESSED_REGISTRY)
|
|
288
288
|
processed_set = set(load_processed_pdfs(registry_path))
|
|
289
289
|
pdfs_to_run = [p for p in only_process_these if os.path.basename(p) not in processed_set]
|
|
290
290
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: visual-parser
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.4
|
|
4
4
|
Summary: Standalone Visual-RAG PDF & Image Parser - text extraction and Vision-LLM figure/image descriptions to JSONL
|
|
5
5
|
Author: Zavier N. Ndum
|
|
6
6
|
Author-email: zavier.ndum@tamu.edu
|
|
@@ -53,7 +53,7 @@ Dynamic: license-file
|
|
|
53
53
|
`visual-parser` converts PDFs (and standalone images alongside them) into a multi-modal JSONL knowledge base (text chunks, figure descriptions, standalone image descriptions, metadata). It was extracted from [RADIANT-LLM](https://github.com/SmartLabNuclear/RADIANT_LLM) as a standalone PDF-and-image ingestion tool, independent of any chatbot.
|
|
54
54
|
|
|
55
55
|
1) Run `visual-parser` on a directory of curated PDFs and/or standalone images to generate JSONL KB files.
|
|
56
|
-
2) Point any downstream RAG system at the generated KB — RADIANT-LLM, AutoSAM, and AutoFLUKA all consume the identical JSONL/registry format, so the same KB works with any of them without re-parsing.
|
|
56
|
+
2) Point any downstream RAG system at the generated KB — [RADIANT-LLM](https://github.com/SmartLabNuclear/RADIANT_LLM), [AutoSAM](https://github.com/SmartLabNuclear/AutoSAM), and [AutoFLUKA](https://github.com/SmartLabNuclear/AutoFLUKA) all consume the identical JSONL/registry format, so the same KB works with any of them without re-parsing.
|
|
57
57
|
|
|
58
58
|
## Table of Contents
|
|
59
59
|
|
|
@@ -78,6 +78,8 @@ By default, the pipeline writes:
|
|
|
78
78
|
|
|
79
79
|
Alongside the KB, the pipeline also writes bookkeeping files: `04_processed_pdfs.txt` / `parsed_images.txt` (track which PDFs/images have already been processed, so re-runs skip them unless `--rebuild` or `--redo NAME`) and `05_pipeline.log` (run log at the verbosity set by `--log-level`, default `ERROR`).
|
|
80
80
|
|
|
81
|
+
> **New in 2.2.4 (additive, no existing field or ID changed):** every `02_visuals_kb.jsonl` and `image_descriptions_kb.jsonl` row now carries `vision_provider`/`vision_model`, and every `03_metadata_kb.jsonl` row carries a `parser` block (`visual_parser` version, `parsed_at`, `text_engine`, `text_fallback`, the resolved `vision_model`, and the run's chunking/context settings) — recording which engine and model actually produced that document's rows, since the configured settings aren't always what ran (Nougat can fail and fall back to lightweight; a local Ollama model can be auto-selected at run time). Older rows simply lack these fields. Figure description also now actively guards against a mixed-model document: if a PDF already has figures described under a different `vision_model` than the current run's, its remaining pages are skipped entirely (not partially re-described under the new model) and its basename is reported back — reprocess it with `--redo NAME` to redescribe it consistently under the new model.
|
|
82
|
+
|
|
81
83
|
> **Breaking change in 2.2.3:** the standalone-image KB file was renamed from `image_descriptions.jsonl` to `image_descriptions_kb.jsonl` (matching the `NN_name_kb.jsonl` convention used by the other three KB files). Upgrading from an older version does **not** migrate existing data automatically — the resume check only looks at the new filename, so already-described images in an old `image_descriptions.jsonl` would otherwise appear "new" again and get re-described. If you have existing data, rename the file yourself before re-running: `image_descriptions.jsonl` → `image_descriptions_kb.jsonl`.
|
|
82
84
|
|
|
83
85
|
## API Keys (.env)
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
visual_parser/__init__.py,sha256=
|
|
1
|
+
visual_parser/__init__.py,sha256=L8CC_lJgEBX38SyjUNrq_n9-krO9OuAIlgZP316rlAw,1039
|
|
2
2
|
visual_parser/__main__.py,sha256=-p6rqjrnsjrtu71eFtmZdqFI1NgHXX9ASOnjuGqghNo,168
|
|
3
3
|
visual_parser/cli.py,sha256=28u8JxiFwc6cHf-CGk3TiVFn7llQ0k_3smSH_KFR9tQ,16077
|
|
4
4
|
visual_parser/cli_main.py,sha256=fZPCY6M_gMpp8rJ8U1F0-hJjiTRn3Ih03A-A_ZaPp1Q,11841
|
|
5
5
|
visual_parser/config.py,sha256=mu52rGN1keQtho70IUP7kPuS6Wykpcc-gv3z0irg70c,12140
|
|
6
|
-
visual_parser/figure_describer.py,sha256=
|
|
7
|
-
visual_parser/image_describer.py,sha256=
|
|
8
|
-
visual_parser/image_tracker.py,sha256=
|
|
6
|
+
visual_parser/figure_describer.py,sha256=PrJVAh7cmsEmm06XqJDnOKTMBRwUyzgY0CqJEsodgp0,15520
|
|
7
|
+
visual_parser/image_describer.py,sha256=BIR42ofCGEr7tHSqvOvqWg_APD3P1ihMVs8kAVo4p4Y,14063
|
|
8
|
+
visual_parser/image_tracker.py,sha256=mlEgAdTkheddG12P2xIL6P7dPAYwrygCSPu0qA2WX-w,2292
|
|
9
9
|
visual_parser/jsonl_writer.py,sha256=klkDl_KKdCRtiWWjTNqecN2B6zXPc0e9f704Dvj8Fh0,6215
|
|
10
10
|
visual_parser/kb_redo.py,sha256=5_FpHHaEYQCAXSAN-j-snaj6OlaL36Vqi97DMqPPROA,5897
|
|
11
11
|
visual_parser/metadata_extractor.py,sha256=Z0jnA8Goo9R3SBThx5PQMetZkyrbevNvYCq36WsoQuQ,6579
|
|
@@ -13,14 +13,14 @@ visual_parser/model_catalog.py,sha256=Voli3Wv0YYLWyocNCpiBCchzfCxX4IYFNR4eOIlFyI
|
|
|
13
13
|
visual_parser/nougat_engine.py,sha256=o9u9XTxOL1EAo63mVARVt4z1IxfDMpvjhMD4URfTx1k,8369
|
|
14
14
|
visual_parser/ollama_local.py,sha256=YQi1oT4OZcuxrp2yCVN5wo49jl9k7f6dvVFGt108v8g,13883
|
|
15
15
|
visual_parser/openai_gateway.py,sha256=XebNY3qOY1I1SdrLFqw9yeax2_IwOSZyVi_XYWfYXj0,3683
|
|
16
|
-
visual_parser/pdf_tracker.py,sha256=
|
|
17
|
-
visual_parser/pipeline.py,sha256=
|
|
16
|
+
visual_parser/pdf_tracker.py,sha256=LEyhZtD5H05-BrWW6opGpGk-qivoKS3Tkar8-1aT1gs,5378
|
|
17
|
+
visual_parser/pipeline.py,sha256=2g30AxNATec8XWztVuIDDQmnz-gTle2iXG5lQOLbRg0,33776
|
|
18
18
|
visual_parser/prompts.py,sha256=hdjDUZjF0A9eugm6qt4K5E1qnqMO1JE9_mTDAGS7sgg,11186
|
|
19
|
-
visual_parser/text_extractor.py,sha256=
|
|
19
|
+
visual_parser/text_extractor.py,sha256=UoipnlAMdgFthsTQUHAnbhOeVLc1BUR4nTEOaijFsUg,17501
|
|
20
20
|
visual_parser/vision_llm.py,sha256=TXiitsDAMIdo9fbzYVKqd-0sGt-hqiJryMU481eo1SM,17429
|
|
21
|
-
visual_parser-2.2.
|
|
22
|
-
visual_parser-2.2.
|
|
23
|
-
visual_parser-2.2.
|
|
24
|
-
visual_parser-2.2.
|
|
25
|
-
visual_parser-2.2.
|
|
26
|
-
visual_parser-2.2.
|
|
21
|
+
visual_parser-2.2.4.dist-info/licenses/LICENSE,sha256=OEJRtZkWILIqPizQxk1m2CjYi1czEKW15RKX6WgLVBE,11344
|
|
22
|
+
visual_parser-2.2.4.dist-info/METADATA,sha256=Bg8PQBKDh1WLyCeZt2tMN23kyrwu_VEGCncHvRRL47U,18291
|
|
23
|
+
visual_parser-2.2.4.dist-info/WHEEL,sha256=a-udL2m4V_6Yn8b-uKzWb12RjRxwwMo6ohYFM24tpaY,91
|
|
24
|
+
visual_parser-2.2.4.dist-info/entry_points.txt,sha256=yypghqnxA-nSiyoDyhlg99SK9vfOzloFwUiiwRw4EWI,62
|
|
25
|
+
visual_parser-2.2.4.dist-info/top_level.txt,sha256=_aaZ41uFRN_WX-616YvmcXIroNq4k4m6Ph6FGetxLYY,14
|
|
26
|
+
visual_parser-2.2.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|