any2md 1.0.4__tar.gz → 1.0.5__tar.gz

{any2md-1.0.4 → any2md-1.0.5}/CHANGELOG.md

@@ -4,6 +4,43 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## [1.0.5] — 2026-04-27
+
+Patch release. Recovers DOCX content that Docling's msword backend
+silently drops on malformed list structures, by auto-retrying
+affected files through the mammoth lane and forwarding the
+captured Docling warnings into any2md's run-level warning bucket.
+Real-world impact observed during validation: a 12,740-word
+Docling output for a policy-template DOCX recovered to 15,566
+words (+22%) when re-rendered via mammoth.
+
+### Added
+- DOCX converter now auto-retries files through the mammoth lane
+  when Docling emits any warning from
+  `docling.backend.msword_backend` during conversion. Docling's
+  DOCX backend silently drops list items in a known
+  malformed-input path (`msword_backend.py:1377/1675` —
+  "Parent element of the list item is not a ListGroup. The list
+  item will be ignored."), so a warning means the Docling
+  Markdown is missing content. On fallback, `extracted_via` is
+  set to `"docling→mammoth (warning fallback)"`, the lane
+  switches to `text`, the captured Docling warnings are
+  forwarded into the run-level warning bucket (so `--strict`
+  still fails), and a `FALLBACK:` line is printed to stderr.
+  Default: enabled. Disable with `--no-docx-fallback-on-warn`
+  (e.g. when comparing backends explicitly).
+- New `PipelineOptions.docx_fallback_on_warn: bool = True`.
+- New CLI flag pair `--docx-fallback-on-warn` /
+  `--no-docx-fallback-on-warn`.
+
+### Tests
+- New `tests/integration/test_docx_fallback_on_warn.py` covers
+  the warning-capture context manager, the default-on fallback
+  path (output swap, `extracted_via`, captured warning forwarded
+  into `collected_warnings()`, user-visible message), the
+  `--no-docx-fallback-on-warn` opt-out, the no-warning
+  fast-path, and the explicit `--backend docling` interaction.
+
 ## [1.0.4] — 2026-04-26
 
 Patch release. Two follow-ups deferred from v1.0.3 (#17): the T7

{any2md-1.0.4 → any2md-1.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: any2md
-Version: 1.0.4
+Version: 1.0.5
 Summary: Convert PDF, DOCX, HTML, and TXT files — or web pages by URL — to clean, LLM-optimized Markdown with YAML frontmatter.
 Project-URL: Homepage, https://github.com/rocklambros/any2md
 Project-URL: Issues, https://github.com/rocklambros/any2md/issues

{any2md-1.0.4 → any2md-1.0.5}/any2md/__init__.py

@@ -1,3 +1,3 @@
 """Convert PDF, DOCX, HTML, and TXT files to LLM-optimized Markdown."""
 
-__version__ = "1.0.4"
+__version__ = "1.0.5"

{any2md-1.0.4 → any2md-1.0.5}/any2md/cli.py

@@ -169,6 +169,16 @@ def main():
         action="store_true",
         help="Promote pipeline validation warnings to errors (exit 3).",
     )
+    parser.add_argument(
+        "--docx-fallback-on-warn",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="DOCX-only. When Docling emits any msword_backend warning "
+        "during conversion, auto-retry the file through mammoth and use "
+        "that output instead. Docling silently drops list items on "
+        "malformed input, so a warning means content is missing. "
+        "Default: enabled. Disable with --no-docx-fallback-on-warn.",
+    )
     parser.add_argument(
         "--quiet",
         "-q",
@@ -272,6 +282,7 @@ def main():
         frontmatter_overrides=overrides or None,
         backend=args.backend,
         arxiv_lookup=not args.no_arxiv_lookup,
+        docx_fallback_on_warn=args.docx_fallback_on_warn,
     )
 
     # CLI-only output controls (not part of PipelineOptions).

{any2md-1.0.4 → any2md-1.0.5}/any2md/converters/docx.py

@@ -9,6 +9,7 @@ produces the markdown body.
 
 from __future__ import annotations
 
+import logging
 import sys
 import xml.etree.ElementTree as ET
 import zipfile
@@ -26,6 +27,46 @@ from any2md.heuristics import filter_organization
 from any2md.pipeline import PipelineOptions
 from any2md.utils import sanitize_filename
 
+
+_DOCLING_MSWORD_LOGGER = "docling.backend.msword_backend"
+
+
+class _DoclingMswordWarningCapture:
+    """Buffer WARNING+ records from Docling's DOCX backend logger.
+
+    Used as a context manager around a Docling DOCX conversion to detect
+    upstream's silent list-item-drop guard (msword_backend.py:1377/1675).
+    Any captured records signal that Docling's Markdown output is
+    incomplete; ``convert_docx`` consults this to decide whether to
+    auto-retry with the mammoth lane.
+    """
+
+    def __init__(self) -> None:
+        self._records: list[logging.LogRecord] = []
+        self._handler: logging.Handler | None = None
+
+    def __enter__(self) -> _DoclingMswordWarningCapture:
+        records = self._records
+
+        class _BufferHandler(logging.Handler):
+            def emit(self, record: logging.LogRecord) -> None:
+                records.append(record)
+
+        handler = _BufferHandler(level=logging.WARNING)
+        logging.getLogger(_DOCLING_MSWORD_LOGGER).addHandler(handler)
+        self._handler = handler
+        return self
+
+    def __exit__(self, *_exc: object) -> None:
+        if self._handler is not None:
+            logging.getLogger(_DOCLING_MSWORD_LOGGER).removeHandler(self._handler)
+            self._handler = None
+
+    @property
+    def messages(self) -> list[str]:
+        return [r.getMessage() for r in self._records]
+
+
 _NS_CORE = {
     "dc": "http://purl.org/dc/elements/1.1/",
     "cp": "http://schemas.openxmlformats.org/package/2006/metadata/core-properties",
@@ -90,13 +131,19 @@ def _read_docx_metadata(docx_path: Path) -> dict[str, object]:
     return out
 
 
-def _extract_via_docling(docx_path: Path) -> tuple[str, str]:
-    """Returns (markdown, 'docling'). Raises on Docling errors."""
+def _extract_via_docling(docx_path: Path) -> tuple[str, str, list[str]]:
+    """Returns (markdown, 'docling', captured_warnings).
+
+    ``captured_warnings`` is the list of WARNING+ messages emitted by
+    Docling's ``msword_backend`` logger during the conversion. It is
+    empty for clean runs. Raises on Docling errors.
+    """
     from docling.document_converter import DocumentConverter
 
     converter = DocumentConverter()
-    result = converter.convert(str(docx_path))
-    return result.document.export_to_markdown(), "docling"
+    with _DoclingMswordWarningCapture() as cap:
+        result = converter.convert(str(docx_path))
+    return result.document.export_to_markdown(), "docling", list(cap.messages)
 
 
 def _extract_via_mammoth(docx_path: Path, options: PipelineOptions) -> tuple[str, str]:
@@ -139,15 +186,18 @@ def convert_docx(
             )
             return False
 
+        docling_warnings: list[str] = []
         if options.backend == "mammoth":
             md_text, extracted_via = _extract_via_mammoth(docx_path, options)
             lane = "text"
         elif options.backend == "docling":
-            md_text, extracted_via = _extract_via_docling(docx_path)
+            md_text, extracted_via, docling_warnings = _extract_via_docling(docx_path)
             lane = "structured"
         elif has_docling():
             try:
-                md_text, extracted_via = _extract_via_docling(docx_path)
+                md_text, extracted_via, docling_warnings = _extract_via_docling(
+                    docx_path
+                )
                 lane = "structured"
             except Exception as e:  # noqa: BLE001 — fall back rather than fail
                 print(
@@ -157,10 +207,34 @@ def convert_docx(
                 )
                 md_text, extracted_via = _extract_via_mammoth(docx_path, options)
                 lane = "text"
+                docling_warnings = []
         else:
             md_text, extracted_via = _extract_via_mammoth(docx_path, options)
             lane = "text"
 
+        # v1.0.5: Docling's DOCX backend silently drops list items in a
+        # known malformed-input path. When any msword_backend warning
+        # fires, swap the Docling output for the mammoth lane (different
+        # parser, generally more permissive). Captured warnings are
+        # forwarded into the run-level warning bucket so --strict still
+        # fails on them.
+        if (
+            docling_warnings
+            and options.docx_fallback_on_warn
+            and extracted_via == "docling"
+        ):
+            print(
+                f"  FALLBACK: {docx_path.name} -- Docling emitted "
+                f"{len(docling_warnings)} msword_backend warning(s); "
+                f"re-running via mammoth (content may have been dropped "
+                f"by Docling).",
+                file=sys.stderr,
+            )
+            md_text, _ = _extract_via_mammoth(docx_path, options)
+            extracted_via = "docling→mammoth (warning fallback)"
+            lane = "text"
+            add_warnings([f"docling.msword_backend: {m}" for m in docling_warnings])
+
         md_text, warnings = pipeline.run(md_text, lane, options)
         add_warnings(warnings)
 

{any2md-1.0.4 → any2md-1.0.5}/any2md/pipeline/__init__.py

@@ -49,6 +49,15 @@ class PipelineOptions:
     # ``--no-arxiv-lookup`` to disable the network call (airgapped envs
     # or when offline behavior is required).
     arxiv_lookup: bool = True
+    # v1.0.5: DOCX-only. When True (default) and Docling emits any
+    # WARNING from ``docling.backend.msword_backend`` during conversion,
+    # any2md re-runs the file through the mammoth+markdownify lane and
+    # uses that output. Docling silently drops list items in a known
+    # malformed-input path (msword_backend.py:1377/1675), so a Docling
+    # warning means the Markdown is missing content. Disable via
+    # ``--no-docx-fallback-on-warn`` to keep Docling output even when
+    # warnings fire (e.g. when comparing backends explicitly).
+    docx_fallback_on_warn: bool = True
 
 
 Stage = Callable[[str, PipelineOptions], str]

any2md-1.0.5/tests/integration/test_docx_fallback_on_warn.py

@@ -0,0 +1,254 @@
+"""Integration tests for the Docling→mammoth auto-retry on msword warnings.
+
+Docling's DOCX backend silently drops list items when their tracked
+parent isn't a `ListGroup` (msword_backend.py:1377/1675). When that
+warning fires we want any2md to auto-retry the file with mammoth, which
+uses a different parsing path. These tests stub `_extract_via_docling`
+so they don't depend on a real DOCX that reproduces the upstream bug.
+"""
+
+from __future__ import annotations
+
+import logging
+import zipfile
+from pathlib import Path
+
+import yaml
+
+import any2md.converters.docx as docx_mod
+from any2md.converters import collected_warnings, reset_warnings
+from any2md.converters.docx import (
+    _DoclingMswordWarningCapture,
+    convert_docx,
+)
+from any2md.pipeline import PipelineOptions
+
+
+_CONTENT_TYPES = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
+  <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
+  <Default Extension="xml" ContentType="application/xml"/>
+  <Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
+  <Override PartName="/docProps/core.xml" ContentType="application/vnd.openxmlformats-package.core-properties+xml"/>
+  <Override PartName="/docProps/app.xml" ContentType="application/vnd.openxmlformats-officedocument.extended-properties+xml"/>
+</Types>
+"""
+
+_RELS = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+  <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
+  <Relationship Id="rId2" Type="http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties" Target="docProps/core.xml"/>
+  <Relationship Id="rId3" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/extended-properties" Target="docProps/app.xml"/>
+</Relationships>
+"""
+
+_CORE = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/metadata/core-properties"
+                   xmlns:dc="http://purl.org/dc/elements/1.1/"
+                   xmlns:dcterms="http://purl.org/dc/terms/">
+  <dc:title>Listy Doc</dc:title>
+  <dc:creator>Test Author</dc:creator>
+  <dcterms:modified xsi:type="dcterms:W3CDTF" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">2026-04-26T00:00:00Z</dcterms:modified>
+</cp:coreProperties>
+"""
+
+_APP = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Properties xmlns="http://schemas.openxmlformats.org/officeDocument/2006/extended-properties">
+</Properties>
+"""
+
+# Minimal document with one heading, one paragraph, one (mammoth-parseable)
+# bullet list — enough that mammoth produces a non-trivial Markdown body.
+_DOCUMENT = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
+  <w:body>
+    <w:p><w:pPr><w:pStyle w:val="Heading1"/></w:pPr><w:r><w:t>Listy Doc</w:t></w:r></w:p>
+    <w:p><w:r><w:t>Body before list.</w:t></w:r></w:p>
+    <w:p><w:pPr><w:numPr><w:ilvl w:val="0"/><w:numId w:val="1"/></w:numPr></w:pPr><w:r><w:t>Item one</w:t></w:r></w:p>
+    <w:p><w:pPr><w:numPr><w:ilvl w:val="0"/><w:numId w:val="1"/></w:numPr></w:pPr><w:r><w:t>Item two</w:t></w:r></w:p>
+  </w:body>
+</w:document>
+"""
+
+
+def _build_listy_docx(out_path: Path) -> None:
+    with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED) as z:
+        z.writestr("[Content_Types].xml", _CONTENT_TYPES)
+        z.writestr("_rels/.rels", _RELS)
+        z.writestr("docProps/core.xml", _CORE)
+        z.writestr("docProps/app.xml", _APP)
+        z.writestr("word/document.xml", _DOCUMENT)
+
+
+# ---------- _DoclingMswordWarningCapture (unit-ish behavior) -------------
+
+
+def test_capture_collects_warning_records():
+    cap = _DoclingMswordWarningCapture()
+    log = logging.getLogger("docling.backend.msword_backend")
+    with cap:
+        log.warning("Parent element of the list item is not a ListGroup. Ignored.")
+        log.info("not captured (below WARNING)")
+    assert len(cap.messages) == 1
+    assert "ListGroup" in cap.messages[0]
+
+
+def test_capture_handler_detached_after_exit():
+    cap = _DoclingMswordWarningCapture()
+    log = logging.getLogger("docling.backend.msword_backend")
+    before = list(log.handlers)
+    with cap:
+        pass
+    assert log.handlers == before
+
+
+def test_capture_records_from_child_loggers():
+    """Child-logger records propagate up; capture must see them."""
+    cap = _DoclingMswordWarningCapture()
+    child = logging.getLogger("docling.backend.msword_backend.sub")
+    with cap:
+        child.warning("nested warning")
+    assert any("nested warning" in m for m in cap.messages)
+
+
+# ---------- convert_docx fallback behavior --------------------------------
+
+
+def _stub_docling_with_warning(
+    monkeypatch,
+    message: str = "Parent element of the list item is not a ListGroup. The list item will be ignored.",
+):
+    """Make `_extract_via_docling` emit a docling msword warning and return stub MD."""
+    log = logging.getLogger("docling.backend.msword_backend")
+
+    def _fake(docx_path):
+        # Use the real capture machinery: emit through the same logger
+        # the capture is attached to.
+        log.warning(message)
+        return "# Docling output\n", "docling", [message]
+
+    monkeypatch.setattr(docx_mod, "_extract_via_docling", _fake)
+
+
+def test_fallback_fires_by_default(tmp_path, tmp_output_dir, monkeypatch, capsys):
+    monkeypatch.setattr(docx_mod, "has_docling", lambda: True)
+
+    def _fake(docx_path):
+        return (
+            "# Docling output\n",
+            "docling",
+            ["Parent element of the list item is not a ListGroup. Ignored."],
+        )
+
+    monkeypatch.setattr(docx_mod, "_extract_via_docling", _fake)
+
+    docx = tmp_path / "listy.docx"
+    _build_listy_docx(docx)
+
+    reset_warnings()
+    ok = convert_docx(docx, tmp_output_dir, options=PipelineOptions(), force=True)
+    assert ok
+
+    out = next(tmp_output_dir.glob("*.md")).read_text(encoding="utf-8")
+    end = out.index("\n---\n", 4)
+    fm = yaml.safe_load(out[4:end])
+    body = out[end + 5 :]
+
+    # Frontmatter reflects the fallback explicitly.
+    assert fm["extracted_via"] == "docling→mammoth (warning fallback)"
+    # Mammoth output reached the body — Docling's "# Docling output" is gone.
+    assert "Docling output" not in body
+    assert "Listy Doc" in body or "Item" in body
+
+    # Warning surfaced into collected_warnings so --strict catches it.
+    warns = collected_warnings()
+    assert any("ListGroup" in w for w in warns)
+
+    # Stderr (or stdout via OK line suffix) carries a user-visible note.
+    captured = capsys.readouterr()
+    combined = captured.out + captured.err
+    assert "fallback" in combined.lower() or "fell back" in combined.lower()
+
+
+def test_fallback_disabled_keeps_docling(tmp_path, tmp_output_dir, monkeypatch):
+    monkeypatch.setattr(docx_mod, "has_docling", lambda: True)
+
+    def _fake(docx_path):
+        return (
+            "# Docling output\n",
+            "docling",
+            ["Parent element of the list item is not a ListGroup. Ignored."],
+        )
+
+    monkeypatch.setattr(docx_mod, "_extract_via_docling", _fake)
+
+    docx = tmp_path / "listy.docx"
+    _build_listy_docx(docx)
+
+    reset_warnings()
+    options = PipelineOptions(docx_fallback_on_warn=False)
+    ok = convert_docx(docx, tmp_output_dir, options=options, force=True)
+    assert ok
+
+    out = next(tmp_output_dir.glob("*.md")).read_text(encoding="utf-8")
+    end = out.index("\n---\n", 4)
+    fm = yaml.safe_load(out[4:end])
+    body = out[end + 5 :]
+    # Fallback disabled: Docling output retained, no fallback marker.
+    assert fm["extracted_via"] == "docling"
+    assert "Docling output" in body
+
+
+def test_no_warnings_no_fallback(tmp_path, tmp_output_dir, monkeypatch):
+    monkeypatch.setattr(docx_mod, "has_docling", lambda: True)
+
+    def _fake(docx_path):
+        return "# Clean Docling output\n", "docling", []
+
+    monkeypatch.setattr(docx_mod, "_extract_via_docling", _fake)
+
+    docx = tmp_path / "listy.docx"
+    _build_listy_docx(docx)
+
+    reset_warnings()
+    ok = convert_docx(docx, tmp_output_dir, options=PipelineOptions(), force=True)
+    assert ok
+
+    out = next(tmp_output_dir.glob("*.md")).read_text(encoding="utf-8")
+    end = out.index("\n---\n", 4)
+    fm = yaml.safe_load(out[4:end])
+    body = out[end + 5 :]
+    assert fm["extracted_via"] == "docling"
+    assert "Clean Docling output" in body
+
+
+def test_explicit_docling_backend_still_falls_back(
+    tmp_path, tmp_output_dir, monkeypatch
+):
+    """User passed --backend docling but Docling emitted warnings: still retry.
+
+    The fallback flag is independent of backend selection; explicit
+    `--backend docling` doesn't pin the user to a lossy result.
+    """
+    monkeypatch.setattr(docx_mod, "has_docling", lambda: True)
+
+    def _fake(docx_path):
+        return (
+            "# Docling output\n",
+            "docling",
+            ["Parent element of the list item is not a ListGroup. Ignored."],
+        )
+
+    monkeypatch.setattr(docx_mod, "_extract_via_docling", _fake)
+
+    docx = tmp_path / "listy.docx"
+    _build_listy_docx(docx)
+
+    options = PipelineOptions(backend="docling")
+    reset_warnings()
+    ok = convert_docx(docx, tmp_output_dir, options=options, force=True)
+    assert ok
+    out = next(tmp_output_dir.glob("*.md")).read_text(encoding="utf-8")
+    end = out.index("\n---\n", 4)
+    fm = yaml.safe_load(out[4:end])
+    assert fm["extracted_via"] == "docling→mammoth (warning fallback)"