visual-parser 2.2.0__tar.gz → 2.2.1__tar.gz

{visual_parser-2.2.0 → visual_parser-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: visual-parser
-Version: 2.2.0
+Version: 2.2.1
 Summary: Standalone Visual-RAG PDF Parser - text extraction and Vision-LLM figure descriptions to JSONL
 Author: Zavier N. Ndum
 Author-email: zavier.ndum@tamu.edu

{visual_parser-2.2.0 → visual_parser-2.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "visual-parser"
-version = "2.2.0"
+version = "2.2.1"
 description = "Standalone Visual-RAG PDF Parser - text extraction and Vision-LLM figure descriptions to JSONL"
 readme = "README.md"
 requires-python = ">=3.10"

{visual_parser-2.2.0 → visual_parser-2.2.1}/tests/test_cli_parity.py

@@ -34,6 +34,30 @@ def test_cli_and_cli_main_have_identical_flag_structure():
         assert a1.option_strings == a2.option_strings, f"--{dest}: flag spelling mismatch"
 
 
+def test_help_text_image_mentions_match():
+    """cli.py and cli_main.py deliberately use different unicode-vs-ASCII
+    characters in help text (that's the whole reason cli_main.py exists --
+    see its module docstring), so help strings aren't expected to be
+    byte-identical, and the structural parity test above doesn't compare
+    them at all. But one file's help text documenting a real capability
+    (e.g. standalone-image support) that the other omits entirely is a real
+    content drift, not a cosmetic one -- this is a regression test for
+    exactly that: --input-dir's help text in cli.py forgot to mention images
+    when --skip-images was added to both files' flag sets."""
+    actions1 = _actions_by_dest(cli._build_arg_parser())
+    actions2 = _actions_by_dest(cli_main._build_arg_parser())
+
+    for dest in actions1:
+        if dest not in actions2:
+            continue
+        mentions1 = "image" in (actions1[dest].help or "").lower()
+        mentions2 = "image" in (actions2[dest].help or "").lower()
+        assert mentions1 == mentions2, (
+            f"--{dest}: one file's help text mentions images, the other doesn't "
+            f"(cli={mentions1}, cli_main={mentions2})"
+        )
+
+
 @pytest.mark.parametrize("module_name", sorted(MODULES))
 def test_missing_input_dir_errors_unless_list_models(module_name):
     module = MODULES[module_name]

{visual_parser-2.2.0 → visual_parser-2.2.1}/tests/test_config.py

@@ -36,6 +36,29 @@ def test_from_env_reads_overrides(monkeypatch):
     assert config.vision_context_pages == 2
 
 
+def test_skip_images_defaults_to_false():
+    assert ParserConfig(input_dir=".").skip_images is False
+
+
+def test_from_env_reads_skip_images(monkeypatch):
+    monkeypatch.setenv("VISUAL_PARSER_SKIP_IMAGES", "true")
+    assert ParserConfig.from_env().skip_images is True
+
+
+def test_redo_names_defaults_to_empty_list():
+    assert ParserConfig(input_dir=".").redo_names == []
+
+
+def test_from_env_redo_names_defaults_to_empty_list(monkeypatch):
+    monkeypatch.delenv("VISUAL_PARSER_REDO_NAMES", raising=False)
+    assert ParserConfig.from_env().redo_names == []
+
+
+def test_from_env_reads_redo_names_comma_separated(monkeypatch):
+    monkeypatch.setenv("VISUAL_PARSER_REDO_NAMES", "doc1.pdf, diagram2.png ,doc3.pdf")
+    assert ParserConfig.from_env().redo_names == ["doc1.pdf", "diagram2.png", "doc3.pdf"]
+
+
 def test_vision_context_pages_defaults_to_zero(monkeypatch):
     monkeypatch.delenv("VISUAL_PARSER_VISION_CONTEXT_PAGES", raising=False)
     assert ParserConfig.from_env().vision_context_pages == 0

{visual_parser-2.2.0 → visual_parser-2.2.1}/tests/test_figure_describer.py

@@ -60,6 +60,33 @@ def test_context_pages_zero_sends_one_image_per_call(tmp_path, monkeypatch):
     assert {r["page"] for r in rows} == {1, 2, 3}
 
 
+def test_non_string_description_is_skipped_not_written(tmp_path, monkeypatch):
+    """Regression test: a weak/non-compliant model can return 'description'
+    as a nested object instead of the documented flat string. Before the
+    isinstance(description, str) fix, this silently wrote the dict straight
+    into 02_visuals_kb.jsonl's 'description' field with no warning at all --
+    unlike image_describer.py's sibling bug, this one never crashed, so it
+    needs its own explicit check that the row is actually dropped."""
+    pdf_path = _make_pdf(tmp_path / "doc.pdf", num_pages=1)
+
+    def fake(images, prompt, provider, api_key, model, detail, reasoning_effort):
+        return json.dumps([{"description": {"Subject": "nested, not flat"}}])
+
+    monkeypatch.setattr(figure_describer, "call_vision_llm", fake)
+
+    figure_describer.describe_figures_for_new_pdfs(
+        new_pdf_paths=[pdf_path],
+        output_dir=str(tmp_path),
+        vision_provider="gpt",
+        vision_api_key="sk-test",
+        vision_model="gpt-5.4",
+        max_workers=1,
+        context_pages=0,
+    )
+
+    assert _read_jsonl(tmp_path / "02_visuals_kb.jsonl") == []
+
+
 def test_context_pages_window_bounded_at_pdf_edges(tmp_path, monkeypatch):
     pdf_path = _make_pdf(tmp_path / "doc.pdf", num_pages=3)
     captured = {}  # page_number -> image_count

visual_parser-2.2.1/tests/test_image_describer.py

@@ -0,0 +1,328 @@
+import json
+
+import pytest
+from PIL import Image
+
+from visual_parser import image_describer
+
+
+def _make_png(path, color=(255, 0, 0)):
+    img = Image.new("RGB", (8, 8), color=color)
+    img.save(str(path), format="PNG")
+    return str(path)
+
+
+def _read_jsonl(path):
+    if not path.exists():
+        return []
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+# ---------------------------------------------------------------------------
+# _parse_single_image_response
+# ---------------------------------------------------------------------------
+
+def test_parse_well_formed_object():
+    raw = json.dumps({"description": "**Subject:** A Chart"})
+    parsed = image_describer._parse_single_image_response(raw, "img.png")
+    assert parsed == {"description": "**Subject:** A Chart"}
+
+
+def test_parse_prose_wrapped_object():
+    raw = 'Sure, here is the result:\n```json\n{"description": "**Subject:** X"}\n```\nHope that helps!'
+    parsed = image_describer._parse_single_image_response(raw, "img.png")
+    assert parsed == {"description": "**Subject:** X"}
+
+
+def test_parse_malformed_returns_none():
+    parsed = image_describer._parse_single_image_response("not json at all {{{", "img.png")
+    assert parsed is None
+
+
+def test_parse_unwraps_single_element_list():
+    """A model that ignores 'NOT a list' and returns [{...}] anyway should
+    still be recovered, mirroring figure_describer.py's own defensive
+    normalization for the opposite shape mismatch."""
+    raw = json.dumps([{"description": "**Subject:** Y"}])
+    parsed = image_describer._parse_single_image_response(raw, "img.png")
+    assert parsed == {"description": "**Subject:** Y"}
+
+
+def test_parse_recovers_real_answer_when_model_echoes_the_prompt_template():
+    """Regression test: a greedy regex would span from the first '{' to the
+    LAST '}' across both blocks, producing one invalid blob when a model
+    echoes the prompt's own OUTPUT FORMAT example alongside its real answer.
+    The real answer appears first here, so it must be the one recovered."""
+    raw = (
+        'Here is my answer:\n'
+        '{"description": "**Subject:** Real Answer"}\n'
+        'Reminder of the format: { "description": "**Subject:** [Title]..." }'
+    )
+    parsed = image_describer._parse_single_image_response(raw, "img.png")
+    assert parsed == {"description": "**Subject:** Real Answer"}
+
+
+def test_parse_handles_literal_brace_inside_description_text():
+    """A naive lazy regex would stop at the first '}' even if it's just a
+    character inside a string value (e.g. set notation in the description),
+    truncating the object early and failing to parse. A proper JSON-aware
+    scan must not have this problem. Wrapped in prose so json.loads(body)
+    fails directly and the fallback scan path is actually exercised."""
+    raw = 'Answer:\n{"description": "Shows the set {A, B, C} on the x-axis"}\nDone.'
+    parsed = image_describer._parse_single_image_response(raw, "img.png")
+    assert parsed == {"description": "Shows the set {A, B, C} on the x-axis"}
+
+
+# ---------------------------------------------------------------------------
+# _flatten_to_rgb
+# ---------------------------------------------------------------------------
+
+def test_flatten_composites_transparency_onto_white():
+    """Regression test: Image.convert('RGB') alone drops alpha without
+    compositing, commonly exposing (0,0,0) stored under fully-transparent
+    pixels -- a transparent-background PNG would silently turn black."""
+    img = Image.new("RGBA", (4, 4), (0, 0, 0, 0))  # fully transparent, black underneath
+    img.putpixel((0, 0), (10, 20, 30, 255))  # one opaque pixel
+
+    flattened = image_describer._flatten_to_rgb(img)
+
+    assert flattened.mode == "RGB"
+    assert flattened.getpixel((1, 1)) == (255, 255, 255)  # transparent area -> white, not black
+    assert flattened.getpixel((0, 0)) == (10, 20, 30)  # opaque pixel preserved
+
+
+def test_flatten_leaves_opaque_rgb_image_unchanged():
+    img = Image.new("RGB", (4, 4), (10, 20, 30))
+    flattened = image_describer._flatten_to_rgb(img)
+    assert flattened.mode == "RGB"
+    assert flattened.getpixel((0, 0)) == (10, 20, 30)
+
+
+# ---------------------------------------------------------------------------
+# _derive_title
+# ---------------------------------------------------------------------------
+
+def test_derive_title_extracts_subject_heading():
+    description = "**Subject:** Vertical Parabolic Gate Schematic\n**Verbatim Text Content:** ..."
+    assert image_describer._derive_title(description) == "Vertical Parabolic Gate Schematic"
+
+
+def test_derive_title_falls_back_to_first_sentence():
+    description = "A plot of decay heat over time. More detail follows here."
+    assert image_describer._derive_title(description) == "A plot of decay heat over time."
+
+
+def test_derive_title_hard_truncates_as_last_resort():
+    description = "x" * 300  # no sentence boundary, no Subject heading
+    title = image_describer._derive_title(description)
+    assert title == "x" * 100
+
+
+# ---------------------------------------------------------------------------
+# describe_images
+# ---------------------------------------------------------------------------
+
+def test_describe_images_skips_non_string_description(tmp_path, monkeypatch):
+    """Regression test: a weak/non-compliant model can return 'description'
+    as a nested object instead of the documented flat string (e.g. one field
+    per heading instead of folding them into one string). Before the
+    isinstance(description, str) fix, this crashed _derive_title()'s regex
+    search with 'TypeError: expected string or bytes-like object, got dict'
+    instead of being skipped like a missing field."""
+    img = _make_png(tmp_path / "a.png")
+
+    monkeypatch.setattr(
+        image_describer, "call_vision_llm",
+        lambda **kwargs: json.dumps({"description": {"Subject": "nested, not flat"}}),
+    )
+
+    written = image_describer.describe_images(
+        new_image_paths=[img],
+        output_dir=str(tmp_path),
+        vision_provider="gpt",
+        vision_api_key="sk-test",
+        vision_model="gpt-5.4",
+    )
+
+    assert written == []
+    assert _read_jsonl(tmp_path / "image_descriptions.jsonl") == []
+
+
+def test_describe_images_writes_rows_for_each_image(tmp_path, monkeypatch):
+    img1 = _make_png(tmp_path / "a.png")
+    img2 = _make_png(tmp_path / "b.png")
+
+    monkeypatch.setattr(
+        image_describer, "call_vision_llm",
+        lambda **kwargs: json.dumps({"description": "**Subject:** Something"}),
+    )
+
+    written = image_describer.describe_images(
+        new_image_paths=[img1, img2],
+        output_dir=str(tmp_path),
+        vision_provider="gpt",
+        vision_api_key="sk-test",
+        vision_model="gpt-5.4",
+    )
+
+    assert set(written) == {"a.png", "b.png"}
+    rows = _read_jsonl(tmp_path / "image_descriptions.jsonl")
+    assert {r["source"] for r in rows} == {"a.png", "b.png"}
+    for row in rows:
+        assert row["title"] == "Something"
+        assert len(row["image_id"]) == 16
+
+
+def test_failed_image_is_not_marked_written(tmp_path, monkeypatch):
+    """A failed vision call must not be reported as written -- the caller
+    uses the return value to decide what to mark processed, and a failed
+    image must stay eligible for retry on the next run."""
+    good = _make_png(tmp_path / "good.png")
+    corrupt = tmp_path / "corrupt.png"
+    corrupt.write_bytes(b"this is not a real image")
+
+    monkeypatch.setattr(
+        image_describer, "call_vision_llm",
+        lambda **kwargs: json.dumps({"description": "**Subject:** Fine"}),
+    )
+
+    written = image_describer.describe_images(
+        new_image_paths=[good, str(corrupt)],
+        output_dir=str(tmp_path),
+        vision_provider="gpt",
+        vision_api_key="sk-test",
+        vision_model="gpt-5.4",
+    )
+
+    assert written == ["good.png"]
+    rows = _read_jsonl(tmp_path / "image_descriptions.jsonl")
+    assert {r["source"] for r in rows} == {"good.png"}
+
+
+def test_vision_call_exception_is_not_marked_written(tmp_path, monkeypatch):
+    img1 = _make_png(tmp_path / "a.png")
+    img2 = _make_png(tmp_path / "b.png")
+
+    call_count = {"n": 0}
+
+    def fake(**kwargs):
+        call_count["n"] += 1
+        if call_count["n"] == 1:
+            raise RuntimeError("simulated failure")
+        return json.dumps({"description": "**Subject:** OK"})
+
+    monkeypatch.setattr(image_describer, "call_vision_llm", fake)
+
+    written = image_describer.describe_images(
+        new_image_paths=[img1, img2],
+        output_dir=str(tmp_path),
+        vision_provider="gpt",
+        vision_api_key="sk-test",
+        vision_model="gpt-5.4",
+        max_workers=1,  # deterministic call order
+    )
+
+    assert len(written) == 1
+    rows = _read_jsonl(tmp_path / "image_descriptions.jsonl")
+    assert len(rows) == 1
+
+
+def test_resume_skips_already_done_images(tmp_path, monkeypatch):
+    img1 = _make_png(tmp_path / "a.png")
+    img2 = _make_png(tmp_path / "b.png")
+
+    descriptions_path = tmp_path / "image_descriptions.jsonl"
+    descriptions_path.write_text(
+        json.dumps({"source": "a.png", "image_id": "x", "title": "Old", "description": "already done"}) + "\n",
+        encoding="utf-8",
+    )
+
+    called = []
+    monkeypatch.setattr(
+        image_describer, "call_vision_llm",
+        lambda **kwargs: called.append(True) or json.dumps({"description": "**Subject:** New"}),
+    )
+
+    written = image_describer.describe_images(
+        new_image_paths=[img1, img2],
+        output_dir=str(tmp_path),
+        vision_provider="gpt",
+        vision_api_key="sk-test",
+        vision_model="gpt-5.4",
+    )
+
+    assert len(called) == 1  # only b.png was actually called
+    assert written == ["b.png"]
+    rows = _read_jsonl(descriptions_path)
+    assert {r["source"] for r in rows} == {"a.png", "b.png"}
+    assert next(r for r in rows if r["source"] == "a.png")["description"] == "already done"
+
+
+
+
+def test_write_failure_is_not_marked_written(tmp_path, monkeypatch):
+    """Regression test: append_to_jsonl can fail silently (disk full,
+    permissions, transient network-share issue) -- describe_images() must
+    check its return value rather than assume the row landed, or the image
+    gets permanently marked processed with no row ever written and no retry
+    path short of --redo/--rebuild."""
+    img = _make_png(tmp_path / "a.png")
+
+    monkeypatch.setattr(
+        image_describer, "call_vision_llm",
+        lambda **kwargs: json.dumps({"description": "**Subject:** X"}),
+    )
+    monkeypatch.setattr(image_describer, "append_to_jsonl", lambda path, rows: False)
+
+    written = image_describer.describe_images(
+        new_image_paths=[img],
+        output_dir=str(tmp_path),
+        vision_provider="gpt",
+        vision_api_key="sk-test",
+        vision_model="gpt-5.4",
+    )
+
+    assert written == []
+
+
+def test_describe_images_uses_its_own_ollama_timeout(tmp_path, monkeypatch):
+    """Regression test: this path's timeout must be decoupled from
+    figure_describer.py's context-window-scaled formula -- calibrated
+    separately (900s) against a real measured baseline, since
+    IMAGE_DESCRIPTION_PROMPT's verbatim-text-transcription requirement can
+    demand much more output than FIGURE_PROMPT ever needs."""
+    img = _make_png(tmp_path / "a.png")
+    captured = {}
+
+    def fake(**kwargs):
+        captured.update(kwargs)
+        return json.dumps({"description": "**Subject:** X"})
+
+    monkeypatch.setattr(image_describer, "call_vision_llm", fake)
+
+    image_describer.describe_images(
+        new_image_paths=[img],
+        output_dir=str(tmp_path),
+        vision_provider="ollama",
+        vision_api_key="",
+        vision_model="qwen2.5-vl:32b",
+    )
+
+    assert captured["ollama_timeout_seconds"] == image_describer._OLLAMA_IMAGE_TIMEOUT_SECONDS
+    assert image_describer._OLLAMA_IMAGE_TIMEOUT_SECONDS == 900.0
+
+
+def test_no_new_images_returns_empty_list_without_calling_vision(tmp_path, monkeypatch):
+    def fail_if_called(**kwargs):
+        raise AssertionError("should not be called")
+
+    monkeypatch.setattr(image_describer, "call_vision_llm", fail_if_called)
+
+    written = image_describer.describe_images(
+        new_image_paths=[],
+        output_dir=str(tmp_path),
+        vision_provider="gpt",
+        vision_api_key="sk-test",
+        vision_model="gpt-5.4",
+    )
+    assert written == []

visual_parser-2.2.1/tests/test_image_tracker.py

@@ -0,0 +1,79 @@
+from visual_parser import image_tracker
+from visual_parser.image_tracker import find_new_images
+
+
+def test_finds_images_by_extension(tmp_path):
+    (tmp_path / "a.png").write_bytes(b"fake")
+    (tmp_path / "b.JPG").write_bytes(b"fake")  # case-insensitive
+    (tmp_path / "c.pdf").write_bytes(b"fake")  # not an image -- excluded
+    (tmp_path / "d.txt").write_bytes(b"fake")  # not an image -- excluded
+
+    registry_path = str(tmp_path / "parsed_images.txt")
+    found = find_new_images(str(tmp_path), registry_path)
+
+    basenames = {__import__("os").path.basename(p) for p in found}
+    assert basenames == {"a.png", "b.JPG"}
+
+
+def test_already_processed_images_are_excluded(tmp_path):
+    (tmp_path / "a.png").write_bytes(b"fake")
+    (tmp_path / "b.png").write_bytes(b"fake")
+
+    registry_path = tmp_path / "parsed_images.txt"
+    registry_path.write_text("a.png\n", encoding="utf-8")
+
+    found = find_new_images(str(tmp_path), str(registry_path))
+    assert [p for p in found if p.endswith("a.png")] == []
+    assert any(p.endswith("b.png") for p in found)
+
+
+def test_rebuild_ignores_registry(tmp_path):
+    (tmp_path / "a.png").write_bytes(b"fake")
+
+    registry_path = tmp_path / "parsed_images.txt"
+    registry_path.write_text("a.png\n", encoding="utf-8")
+
+    found = find_new_images(str(tmp_path), str(registry_path), rebuild=True)
+    assert any(p.endswith("a.png") for p in found)
+
+
+def test_no_images_returns_empty_list(tmp_path):
+    (tmp_path / "doc.pdf").write_bytes(b"fake")
+    found = find_new_images(str(tmp_path), str(tmp_path / "parsed_images.txt"))
+    assert found == []
+
+
+def test_load_processed_pdfs_is_monkeypatchable_at_module_level(tmp_path, monkeypatch):
+    """Regression test: load_processed_pdfs must be imported at module level
+    (not inside find_new_images' own body), or monkeypatch.setattr(
+    image_tracker, "load_processed_pdfs", ...) -- the idiomatic way tests in
+    this file stub registry-loading without touching the filesystem --
+    would silently have no effect, since a function-local import re-resolves
+    the real pdf_tracker.load_processed_pdfs fresh on every call."""
+    (tmp_path / "a.png").write_bytes(b"fake")
+
+    monkeypatch.setattr(image_tracker, "load_processed_pdfs", lambda registry_path: ["a.png"])
+
+    found = find_new_images(str(tmp_path), str(tmp_path / "parsed_images.txt"))
+    assert found == []  # "a.png" reported processed by the stub, correctly excluded
+
+
+def test_registry_path_is_explicit_not_joined_against_input_dir(tmp_path):
+    """Deliberate deviation from find_new_pdfs(): the registry lives wherever
+    the caller points it (e.g. output_dir), not auto-joined against input_dir
+    -- this sidesteps the pre-existing PDF-side input_dir/output_dir registry
+    mismatch bug."""
+    input_dir = tmp_path / "input"
+    output_dir = tmp_path / "output"
+    input_dir.mkdir()
+    output_dir.mkdir()
+    (input_dir / "a.png").write_bytes(b"fake")
+
+    registry_in_output = output_dir / "parsed_images.txt"
+    registry_in_output.write_text("a.png\n", encoding="utf-8")
+
+    # Registry says "a.png" is done, and it's correctly excluded even though
+    # the registry file lives in a completely different directory than the
+    # images themselves.
+    found = find_new_images(str(input_dir), str(registry_in_output))
+    assert found == []

visual_parser-2.2.1/tests/test_jsonl_writer.py

@@ -0,0 +1,61 @@
+import json
+
+from visual_parser.jsonl_writer import append_to_jsonl, atomic_rewrite_jsonl
+
+
+def test_append_returns_true_on_success(tmp_path):
+    path = tmp_path / "out.jsonl"
+    assert append_to_jsonl(str(path), [{"a": 1}]) is True
+    assert json.loads(path.read_text(encoding="utf-8").strip()) == {"a": 1}
+
+
+def test_append_returns_false_when_new_data_is_not_a_list(tmp_path):
+    path = tmp_path / "out.jsonl"
+    assert append_to_jsonl(str(path), {"a": 1}) is False  # type: ignore[arg-type]
+    assert not path.exists()
+
+
+def test_append_returns_false_on_filesystem_error(tmp_path, monkeypatch):
+    """Simulates a disk-full/permission-style failure: open() itself raises."""
+    path = tmp_path / "out.jsonl"
+
+    def raising_open(*a, **k):
+        raise OSError("disk full")
+
+    monkeypatch.setattr("builtins.open", raising_open)
+    assert append_to_jsonl(str(path), [{"a": 1}]) is False
+
+
+# ---------------------------------------------------------------------------
+# atomic_rewrite_jsonl
+# ---------------------------------------------------------------------------
+
+def _write_lines(path, rows):
+    path.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8")
+
+
+def test_atomic_rewrite_keeps_and_drops_rows(tmp_path):
+    path = tmp_path / "kb.jsonl"
+    _write_lines(path, [{"source": "a.pdf"}, {"source": "b.pdf"}, {"source": "a.pdf"}])
+
+    stats = atomic_rewrite_jsonl(str(path), lambda row: row["source"] != "a.pdf")
+
+    assert stats == {"removed": 2, "kept": 1, "malformed": 0}
+    remaining = [json.loads(l) for l in path.read_text(encoding="utf-8").splitlines() if l.strip()]
+    assert remaining == [{"source": "b.pdf"}]
+
+
+def test_atomic_rewrite_counts_malformed_lines(tmp_path):
+    path = tmp_path / "kb.jsonl"
+    path.write_text('{"source": "a.pdf"}\nnot json\n{"source": "b.pdf"}\n', encoding="utf-8")
+
+    stats = atomic_rewrite_jsonl(str(path), lambda row: True)
+
+    assert stats == {"removed": 0, "kept": 2, "malformed": 1}
+
+
+def test_atomic_rewrite_missing_file_is_a_graceful_noop(tmp_path):
+    path = tmp_path / "does_not_exist.jsonl"
+    stats = atomic_rewrite_jsonl(str(path), lambda row: True)
+    assert stats == {"removed": 0, "kept": 0, "malformed": 0}
+    assert not path.exists()  # not created by this call

visual_parser-2.2.1/tests/test_kb_redo.py

@@ -0,0 +1,85 @@
+import json
+
+from visual_parser.kb_redo import KB_JSONL_FILENAMES, REGISTRY_FILENAMES, redo_entries
+from visual_parser.pdf_tracker import load_processed_pdfs, mark_as_processed
+
+
+def _write_rows(path, rows):
+    path.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8")
+
+
+def test_removes_only_matching_rows_across_all_kb_files(tmp_path):
+    for filename in KB_JSONL_FILENAMES:
+        _write_rows(
+            tmp_path / filename,
+            [{"source": "a.pdf", "x": 1}, {"source": "b.pdf", "x": 2}, {"source": "a.pdf", "x": 3}],
+        )
+
+    result = redo_entries(str(tmp_path), ["a.pdf"])
+
+    assert result["totals"]["rows_removed"] == 2 * len(KB_JSONL_FILENAMES)
+    for filename in KB_JSONL_FILENAMES:
+        rows = [json.loads(l) for l in (tmp_path / filename).read_text(encoding="utf-8").splitlines() if l.strip()]
+        assert rows == [{"source": "b.pdf", "x": 2}]
+    assert result["cleared"] == ["a.pdf"]
+    assert result["not_found"] == []
+
+
+def test_backup_created_once_not_again_on_second_call(tmp_path):
+    path = tmp_path / "02_visuals_kb.jsonl"
+    _write_rows(path, [{"source": "a.pdf"}])
+    backup_path = tmp_path / "02_visuals_kb_backup_before_redo.jsonl"
+
+    redo_entries(str(tmp_path), ["a.pdf"])
+    assert backup_path.exists()
+    first_backup_content = backup_path.read_text(encoding="utf-8")
+
+    # Re-seed the file and call again -- backup must NOT be overwritten.
+    _write_rows(path, [{"source": "c.pdf"}])
+    redo_entries(str(tmp_path), ["c.pdf"])
+    assert backup_path.read_text(encoding="utf-8") == first_backup_content
+
+
+def test_not_found_separates_typo_from_real_name(tmp_path):
+    _write_rows(tmp_path / "02_visuals_kb.jsonl", [{"source": "real.pdf"}])
+
+    result = redo_entries(str(tmp_path), ["real.pdf", "typo.pdf"])
+
+    assert result["cleared"] == ["real.pdf"]
+    assert result["not_found"] == ["typo.pdf"]
+
+
+def test_registries_get_entries_stripped(tmp_path):
+    for filename in REGISTRY_FILENAMES:
+        mark_as_processed(str(tmp_path / filename), ["a.pdf", "b.png"])
+
+    result = redo_entries(str(tmp_path), ["a.pdf"])
+
+    for filename in REGISTRY_FILENAMES:
+        assert load_processed_pdfs(str(tmp_path / filename)) == ["b.png"]
+    assert result["totals"]["registry_entries_removed"] == len(REGISTRY_FILENAMES)
+    assert "a.pdf" in result["cleared"]
+
+
+def test_missing_files_handled_gracefully(tmp_path):
+    """No KB files, no registries at all -- must not crash."""
+    result = redo_entries(str(tmp_path), ["ghost.pdf"])
+    assert result["cleared"] == []
+    assert result["not_found"] == ["ghost.pdf"]
+    assert result["totals"] == {"rows_removed": 0, "registry_entries_removed": 0}
+
+
+def test_matches_by_basename_path_stripped(tmp_path):
+    """Both the requested name and the on-disk basename should be
+    path-stripped before comparison -- a full path passed to --redo should
+    still match a bare basename stored as 'source'."""
+    _write_rows(tmp_path / "02_visuals_kb.jsonl", [{"source": "a.pdf"}])
+
+    result = redo_entries(str(tmp_path), [r"C:\some\other\dir\a.pdf"])
+
+    assert result["cleared"] == ["a.pdf"]
+    rows = [
+        json.loads(l) for l in (tmp_path / "02_visuals_kb.jsonl").read_text(encoding="utf-8").splitlines()
+        if l.strip()
+    ]
+    assert rows == []