any2md 1.0.3__tar.gz → 1.0.5__tar.gz

{any2md-1.0.3 → any2md-1.0.5}/CHANGELOG.md

@@ -4,6 +4,89 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## [1.0.5] — 2026-04-27
+
+Patch release. Recovers DOCX content that Docling's msword backend
+silently drops on malformed list structures, by auto-retrying
+affected files through the mammoth lane and forwarding the
+captured Docling warnings into any2md's run-level warning bucket.
+Real-world impact observed during validation: a 12,740-word
+Docling output for a policy-template DOCX recovered to 15,566
+words (+22%) when re-rendered via mammoth.
+
+### Added
+- DOCX converter now auto-retries files through the mammoth lane
+  when Docling emits any warning from
+  `docling.backend.msword_backend` during conversion. Docling's
+  DOCX backend silently drops list items in a known
+  malformed-input path (`msword_backend.py:1377/1675` —
+  "Parent element of the list item is not a ListGroup. The list
+  item will be ignored."), so a warning means the Docling
+  Markdown is missing content. On fallback, `extracted_via` is
+  set to `"docling→mammoth (warning fallback)"`, the lane
+  switches to `text`, the captured Docling warnings are
+  forwarded into the run-level warning bucket (so `--strict`
+  still fails), and a `FALLBACK:` line is printed to stderr.
+  Default: enabled. Disable with `--no-docx-fallback-on-warn`
+  (e.g. when comparing backends explicitly).
+- New `PipelineOptions.docx_fallback_on_warn: bool = True`.
+- New CLI flag pair `--docx-fallback-on-warn` /
+  `--no-docx-fallback-on-warn`.
+
+### Tests
+- New `tests/integration/test_docx_fallback_on_warn.py` covers
+  the warning-capture context manager, the default-on fallback
+  path (output swap, `extracted_via`, captured warning forwarded
+  into `collected_warnings()`, user-visible message), the
+  `--no-docx-fallback-on-warn` opt-out, the no-warning
+  fast-path, and the explicit `--backend docling` interaction.
+
+## [1.0.4] — 2026-04-26
+
+Patch release. Two follow-ups deferred from v1.0.3 (#17): the T7
+`dedupe_toc_table` now normalizes leader-dot padding from TOC
+cells before matching against body headings, so Docling-rendered
+TOC tables (e.g., `Purpose..........Page`) get properly stripped;
+and the audit script's `leading-toc-table` check no longer
+false-flags documents whose only GFM table sits at the end (skip
+when the body contains no H2).
+
+### Changed
+- `pipeline/text.py::dedupe_toc_table` strips leader-dot padding
+  (`re.sub(r'\.{3,}.*$', '', cell)`) from each TOC cell before
+  lower-casing and matching against body H2/H3 titles. (The
+  text-block TOC variant already handles this via `_TOC_LINE_RE`'s
+  capture group — only the table-variant site needed the fix.)
+- `scripts/audit-outputs.py` `leading-toc-table` check is now
+  gated on `"\n## "` being present in the body — when no H2
+  exists, no "leading-TOC region" is defined, so the check is
+  skipped instead of matching the entire document.
+
+### Fixes
+- TOC tables with dotted-page-number padding (typical Docling
+  output for SafeBreach-style PDFs — e.g., `Backup.pdf`,
+  `Password.pdf`) now get stripped by T7 in aggressive/maximum
+  profiles. Previously they survived because the cell content
+  `"1.1. Purpose............."` failed string-equality with the
+  body heading `"1.1. Purpose"`, dropping overlap below the 70%
+  threshold.
+- Audit script no longer reports false-positive
+  `leading-toc-table` flags on documents whose only GFM table is
+  at the end and the body contains no `## ` headings.
+
+### Tests
+- New `test_strips_toc_table_with_leader_dot_padding` in
+  `tests/unit/pipeline/test_text_dedupe_toc_table.py` — TOC
+  table whose cells carry leader-dot padding mirroring later H2
+  headings still triggers the strip.
+- New `tests/unit/scripts/test_audit_outputs.py` covering the
+  audit-script `leading-toc-table` check: no-H2-with-trailing-
+  table is NOT flagged; leading-table-before-H2 IS still flagged.
+
+### Other
+- `scripts/audit-outputs.py` is now tracked in version control
+  (was previously local-only).
+
 ## [1.0.3] — 2026-04-26
 
 Patch release. Audit on the v1.0.2-regenerated corpus surfaced two

{any2md-1.0.3 → any2md-1.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: any2md
-Version: 1.0.3
+Version: 1.0.5
 Summary: Convert PDF, DOCX, HTML, and TXT files — or web pages by URL — to clean, LLM-optimized Markdown with YAML frontmatter.
 Project-URL: Homepage, https://github.com/rocklambros/any2md
 Project-URL: Issues, https://github.com/rocklambros/any2md/issues

{any2md-1.0.3 → any2md-1.0.5}/any2md/__init__.py

@@ -1,3 +1,3 @@
 """Convert PDF, DOCX, HTML, and TXT files to LLM-optimized Markdown."""
 
-__version__ = "1.0.3"
+__version__ = "1.0.5"

{any2md-1.0.3 → any2md-1.0.5}/any2md/cli.py

@@ -169,6 +169,16 @@ def main():
         action="store_true",
         help="Promote pipeline validation warnings to errors (exit 3).",
     )
+    parser.add_argument(
+        "--docx-fallback-on-warn",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="DOCX-only. When Docling emits any msword_backend warning "
+        "during conversion, auto-retry the file through mammoth and use "
+        "that output instead. Docling silently drops list items on "
+        "malformed input, so a warning means content is missing. "
+        "Default: enabled. Disable with --no-docx-fallback-on-warn.",
+    )
     parser.add_argument(
         "--quiet",
         "-q",
@@ -272,6 +282,7 @@ def main():
         frontmatter_overrides=overrides or None,
         backend=args.backend,
         arxiv_lookup=not args.no_arxiv_lookup,
+        docx_fallback_on_warn=args.docx_fallback_on_warn,
     )
 
     # CLI-only output controls (not part of PipelineOptions).

{any2md-1.0.3 → any2md-1.0.5}/any2md/converters/docx.py

@@ -9,6 +9,7 @@ produces the markdown body.
 
 from __future__ import annotations
 
+import logging
 import sys
 import xml.etree.ElementTree as ET
 import zipfile
@@ -26,6 +27,46 @@ from any2md.heuristics import filter_organization
 from any2md.pipeline import PipelineOptions
 from any2md.utils import sanitize_filename
 
+
+_DOCLING_MSWORD_LOGGER = "docling.backend.msword_backend"
+
+
+class _DoclingMswordWarningCapture:
+    """Buffer WARNING+ records from Docling's DOCX backend logger.
+
+    Used as a context manager around a Docling DOCX conversion to detect
+    upstream's silent list-item-drop guard (msword_backend.py:1377/1675).
+    Any captured records signal that Docling's Markdown output is
+    incomplete; ``convert_docx`` consults this to decide whether to
+    auto-retry with the mammoth lane.
+    """
+
+    def __init__(self) -> None:
+        self._records: list[logging.LogRecord] = []
+        self._handler: logging.Handler | None = None
+
+    def __enter__(self) -> _DoclingMswordWarningCapture:
+        records = self._records
+
+        class _BufferHandler(logging.Handler):
+            def emit(self, record: logging.LogRecord) -> None:
+                records.append(record)
+
+        handler = _BufferHandler(level=logging.WARNING)
+        logging.getLogger(_DOCLING_MSWORD_LOGGER).addHandler(handler)
+        self._handler = handler
+        return self
+
+    def __exit__(self, *_exc: object) -> None:
+        if self._handler is not None:
+            logging.getLogger(_DOCLING_MSWORD_LOGGER).removeHandler(self._handler)
+            self._handler = None
+
+    @property
+    def messages(self) -> list[str]:
+        return [r.getMessage() for r in self._records]
+
+
 _NS_CORE = {
     "dc": "http://purl.org/dc/elements/1.1/",
     "cp": "http://schemas.openxmlformats.org/package/2006/metadata/core-properties",
@@ -90,13 +131,19 @@ def _read_docx_metadata(docx_path: Path) -> dict[str, object]:
     return out
 
 
-def _extract_via_docling(docx_path: Path) -> tuple[str, str]:
-    """Returns (markdown, 'docling'). Raises on Docling errors."""
+def _extract_via_docling(docx_path: Path) -> tuple[str, str, list[str]]:
+    """Returns (markdown, 'docling', captured_warnings).
+
+    ``captured_warnings`` is the list of WARNING+ messages emitted by
+    Docling's ``msword_backend`` logger during the conversion. It is
+    empty for clean runs. Raises on Docling errors.
+    """
     from docling.document_converter import DocumentConverter
 
     converter = DocumentConverter()
-    result = converter.convert(str(docx_path))
-    return result.document.export_to_markdown(), "docling"
+    with _DoclingMswordWarningCapture() as cap:
+        result = converter.convert(str(docx_path))
+    return result.document.export_to_markdown(), "docling", list(cap.messages)
 
 
 def _extract_via_mammoth(docx_path: Path, options: PipelineOptions) -> tuple[str, str]:
@@ -139,15 +186,18 @@ def convert_docx(
             )
             return False
 
+        docling_warnings: list[str] = []
         if options.backend == "mammoth":
             md_text, extracted_via = _extract_via_mammoth(docx_path, options)
             lane = "text"
         elif options.backend == "docling":
-            md_text, extracted_via = _extract_via_docling(docx_path)
+            md_text, extracted_via, docling_warnings = _extract_via_docling(docx_path)
             lane = "structured"
         elif has_docling():
             try:
-                md_text, extracted_via = _extract_via_docling(docx_path)
+                md_text, extracted_via, docling_warnings = _extract_via_docling(
+                    docx_path
+                )
                 lane = "structured"
             except Exception as e:  # noqa: BLE001 — fall back rather than fail
                 print(
@@ -157,10 +207,34 @@ def convert_docx(
                 )
                 md_text, extracted_via = _extract_via_mammoth(docx_path, options)
                 lane = "text"
+                docling_warnings = []
         else:
             md_text, extracted_via = _extract_via_mammoth(docx_path, options)
             lane = "text"
 
+        # v1.0.5: Docling's DOCX backend silently drops list items in a
+        # known malformed-input path. When any msword_backend warning
+        # fires, swap the Docling output for the mammoth lane (different
+        # parser, generally more permissive). Captured warnings are
+        # forwarded into the run-level warning bucket so --strict still
+        # fails on them.
+        if (
+            docling_warnings
+            and options.docx_fallback_on_warn
+            and extracted_via == "docling"
+        ):
+            print(
+                f"  FALLBACK: {docx_path.name} -- Docling emitted "
+                f"{len(docling_warnings)} msword_backend warning(s); "
+                f"re-running via mammoth (content may have been dropped "
+                f"by Docling).",
+                file=sys.stderr,
+            )
+            md_text, _ = _extract_via_mammoth(docx_path, options)
+            extracted_via = "docling→mammoth (warning fallback)"
+            lane = "text"
+            add_warnings([f"docling.msword_backend: {m}" for m in docling_warnings])
+
         md_text, warnings = pipeline.run(md_text, lane, options)
         add_warnings(warnings)
 

{any2md-1.0.3 → any2md-1.0.5}/any2md/pipeline/__init__.py

@@ -49,6 +49,15 @@ class PipelineOptions:
     # ``--no-arxiv-lookup`` to disable the network call (airgapped envs
     # or when offline behavior is required).
     arxiv_lookup: bool = True
+    # v1.0.5: DOCX-only. When True (default) and Docling emits any
+    # WARNING from ``docling.backend.msword_backend`` during conversion,
+    # any2md re-runs the file through the mammoth+markdownify lane and
+    # uses that output. Docling silently drops list items in a known
+    # malformed-input path (msword_backend.py:1377/1675), so a Docling
+    # warning means the Markdown is missing content. Disable via
+    # ``--no-docx-fallback-on-warn`` to keep Docling output even when
+    # warnings fire (e.g. when comparing backends explicitly).
+    docx_fallback_on_warn: bool = True
 
 
 Stage = Callable[[str, PipelineOptions], str]

{any2md-1.0.3 → any2md-1.0.5}/any2md/pipeline/text.py

@@ -199,6 +199,7 @@ def dedupe_toc_block(text: str, options: "PipelineOptions") -> str:
     return "\n".join(lines[end:]).lstrip("\n")
 
 
+_LEADER_DOT_RE = re.compile(r"\.{3,}.*$")
 _TABLE_ROW_RE = re.compile(r"^\s*\|.*\|\s*$")
 _TABLE_SEP_RE = re.compile(r"^\s*\|[\s\-:|]+\|\s*$")
 
@@ -260,12 +261,16 @@ def dedupe_toc_table(text: str, options: "PipelineOptions") -> str:
                 continue
 
             # Extract title text from non-numeric cells.
+            # Strip leader-dot padding ('Purpose............') before matching —
+            # Docling renders TOC entries with dotted page-number padding and
+            # the body-heading equivalent does not contain the dots.
             toc_titles: set[str] = set()
             for row in entry_rows:
                 cells = _split_table_cells(row)
                 for cell in cells:
-                    if cell and not cell.replace(".", "").isdigit():
-                        toc_titles.add(cell.lower())
+                    normalized = _LEADER_DOT_RE.sub("", cell).strip()
+                    if normalized and not normalized.replace(".", "").isdigit():
+                        toc_titles.add(normalized.lower())
             if not toc_titles:
                 i = tbl_end if tbl_end > i else i + 1
                 continue