any2md 1.2.0__tar.gz → 1.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {any2md-1.2.0 → any2md-1.2.1}/CHANGELOG.md +111 -0
- {any2md-1.2.0 → any2md-1.2.1}/CONTRIBUTING.md +7 -8
- {any2md-1.2.0 → any2md-1.2.1}/PKG-INFO +3 -3
- {any2md-1.2.0 → any2md-1.2.1}/SECURITY.md +2 -3
- {any2md-1.2.0 → any2md-1.2.1}/any2md/__init__.py +1 -1
- {any2md-1.2.0 → any2md-1.2.1}/any2md/_http.py +52 -1
- {any2md-1.2.0 → any2md-1.2.1}/any2md/cli.py +72 -22
- {any2md-1.2.0 → any2md-1.2.1}/any2md/converters/docx.py +62 -16
- {any2md-1.2.0 → any2md-1.2.1}/any2md/converters/html.py +29 -37
- {any2md-1.2.0 → any2md-1.2.1}/any2md/converters/pdf.py +14 -12
- {any2md-1.2.0 → any2md-1.2.1}/any2md/converters/txt.py +9 -4
- {any2md-1.2.0 → any2md-1.2.1}/any2md/frontmatter.py +36 -15
- {any2md-1.2.0 → any2md-1.2.1}/any2md/heuristics.py +44 -10
- {any2md-1.2.0 → any2md-1.2.1}/any2md/pipeline/cleanup.py +53 -3
- any2md-1.2.1/any2md/pipeline/structured.py +639 -0
- {any2md-1.2.0 → any2md-1.2.1}/any2md/pipeline/text.py +146 -37
- {any2md-1.2.0 → any2md-1.2.1}/any2md/utils.py +0 -10
- {any2md-1.2.0 → any2md-1.2.1}/docs/architecture.md +7 -6
- {any2md-1.2.0 → any2md-1.2.1}/docs/cli-reference.md +6 -1
- {any2md-1.2.0 → any2md-1.2.1}/pyproject.toml +1 -1
- any2md-1.2.1/tests/cli/test_cli_existing_flags.py +65 -0
- any2md-1.2.1/tests/cli/test_cli_meta.py +96 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/cli/test_cli_ocr_save_images.py +11 -4
- any2md-1.2.1/tests/cli/test_cli_symlinks.py +197 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/snapshots/table_heavy.fallback.md +0 -2
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_docx_mammoth_fallback.py +16 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_html_trafilatura.py +13 -0
- any2md-1.2.1/tests/unit/pipeline/test_cleanup_strip_links.py +69 -0
- any2md-1.2.1/tests/unit/pipeline/test_fenced_lines.py +305 -0
- any2md-1.2.1/tests/unit/pipeline/test_structured_heading_hierarchy.py +195 -0
- any2md-1.2.1/tests/unit/pipeline/test_text_dehyphenate.py +117 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_text_repair_line_wraps.py +15 -0
- any2md-1.2.1/tests/unit/test_converter_emission_errors.py +42 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_docling_cache.py +2 -2
- any2md-1.2.1/tests/unit/test_docling_local_paths.py +86 -0
- any2md-1.2.1/tests/unit/test_docx_external_images.py +212 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_html_heading_levels.py +56 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_html_title_h1.py +17 -0
- any2md-1.2.1/tests/unit/test_http_ipv6_embedded.py +86 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_meta_overrides.py +59 -0
- any2md-1.2.1/tests/unit/test_redos_equivalence.py +138 -0
- any2md-1.2.1/tests/unit/test_redos_hardening.py +147 -0
- any2md-1.2.1/tests/unit/test_redos_review_followups.py +143 -0
- any2md-1.2.1/tests/unit/test_scan_directory_case.py +28 -0
- any2md-1.2.1/tests/unit/test_terminal_escape_output.py +217 -0
- any2md-1.2.0/any2md/pipeline/structured.py +0 -172
- any2md-1.2.0/tests/cli/test_cli_existing_flags.py +0 -52
- any2md-1.2.0/tests/cli/test_cli_meta.py +0 -36
- any2md-1.2.0/tests/unit/pipeline/test_structured_heading_hierarchy.py +0 -34
- any2md-1.2.0/tests/unit/pipeline/test_text_dehyphenate.py +0 -29
- {any2md-1.2.0 → any2md-1.2.1}/.gitignore +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/CODE_OF_CONDUCT.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/LICENSE +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/README.md +1 -1
- {any2md-1.2.0 → any2md-1.2.1}/any2md/__main__.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/any2md/_docling.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/any2md/_docling_cache.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/any2md/_logging.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/any2md/config.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/any2md/converters/__init__.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/any2md/pipeline/__init__.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/any2md/validators.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/assets/logo-original.jpeg +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/assets/logo.png +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/assets/social-preview.jpg +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/output-format.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-any2md-v1-phase1-foundation.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-any2md-v1-phase2-docling-backend.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-any2md-v1-phase3-figures-ocr-text-lane.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-any2md-v1-phase4-polish-config-docs.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-any2md-v1.0.2-implementation.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-issue-17-toc-followups.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/plans/2026-04-27-v1.0.6-security-hardening.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/plans/2026-05-04-v1.1.0-persistent-converter.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/plans/2026-05-06-v1.1.1-security-hardening.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/specs/2026-04-26-any2md-v1-design.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/specs/2026-04-26-any2md-v1.0.2-design.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/specs/2026-04-26-issue-17-toc-followups-design.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/specs/2026-04-26-v1.0.3-empty-title-orphan-punct-design.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/specs/2026-04-27-v1.0.6-security-hardening-design.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/specs/2026-05-04-v1.1.0-persistent-converter-design.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/superpowers/specs/2026-05-06-v1.1.1-security-design.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/troubleshooting.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/docs/upgrading-from-0.7.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/requirements.txt +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/scripts/audit-outputs.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/scripts/bench_docling_cache.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/scripts/validate-phase1.sh +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/__init__.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/cli/__init__.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/cli/test_cli_backend.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/cli/test_cli_exit_codes.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/cli/test_cli_high_fidelity.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/cli/test_cli_no_arxiv_lookup.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/cli/test_cli_quiet_verbose.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/cli/test_cli_strict.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/conftest.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/__init__.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/docs/ligatures_and_softhyphens.txt +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/docs/multi_column.pdf +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/docs/table_heavy.docx +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/docs/web_bold_title_first_line.html +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/docs/web_h1_in_hero.html +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/docs/web_h3_sections.html +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/docs/web_no_title.html +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/docs/web_page.html +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/make_fixtures.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/snapshots/ligatures_and_softhyphens.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/snapshots/multi_column.docling.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/snapshots/multi_column.fallback.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/snapshots/table_heavy.docling.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/fixtures/snapshots/web_page.md +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/__init__.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_docling_persistence.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_docx_backend_override.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_docx_docling.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_docx_fallback_on_warn.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_html_h1.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_pdf_backend_override.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_pdf_docling.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_pdf_pymupdf_fallback.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_save_images.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_snapshots.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/integration/test_txt.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/__init__.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/__init__.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_footnote_markers.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_html_entities.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_html_entities_double_encoded.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_ligatures.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_nfc.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_quotes_dashes.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_soft_hyphens.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_validate.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_whitespace.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_runner.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_strip_orphan_punctuation.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_structured_body_cleanup_stages.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_structured_compact_tables.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_structured_lift_figures.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_structured_normalize_cites.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_text_dedupe_paragraphs.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_text_dedupe_toc.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_text_dedupe_toc_table.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_text_restore_lists_code.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_text_strip_cover_artifacts.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_text_strip_headers_footers.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_text_strip_repeated_byline.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/pipeline/test_text_strip_web_fragments.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/scripts/__init__.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/scripts/test_audit_outputs.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_atomic_write.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_atomic_write_symlink_race.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_auto_id.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_config.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_config_toml_warn.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_config_walkup.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_content_hash.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_docling_helper.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_frontmatter_compose.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_frontmatter_helpers.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_frontmatter_key_injection.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_heuristics.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_heuristics_wikipedia_host.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_http_ip_allowlist.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_http_safe.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_http_ssrf_pin.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_http_user_agent.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_lockfile_freshness.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_log_sanitizer.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_logging_module.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_meta_reserved_keys.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_pdf_complexity.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_pdf_image_symlink.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_safe_dir_name.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_security_error_strings.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_url_creds_scrubbed.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_validators.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_workflow_permissions.py +0 -0
- {any2md-1.2.0 → any2md-1.2.1}/tests/unit/test_zipbomb_guard.py +0 -0
|
@@ -4,6 +4,117 @@ All notable changes to this project will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
6
6
|
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [1.2.1] — 2026-11-10
|
|
10
|
+
|
|
11
|
+
Security fixes from a full-repository scan (ReDoS, SSRF, symlink
|
|
12
|
+
following, terminal escape injection) and fixes for nested `--meta`,
|
|
13
|
+
text-lane dehyphenation, `--strip-links`, and structured-lane headings.
|
|
14
|
+
|
|
15
|
+
### Security
|
|
16
|
+
|
|
17
|
+
- **Regex denial of service on crafted documents (CWE-1333, CWE-407).**
|
|
18
|
+
Several patterns that run on converted text backtracked in quadratic,
|
|
19
|
+
cubic, or exponential time on hostile input, so a document of a few KB
|
|
20
|
+
to a few MB could hang a conversion for hours and stall the rest of a
|
|
21
|
+
batch. Fixed sites: the academic byline (exponential), the first-H1
|
|
22
|
+
title, TOC-line and TOC-hint detection, body-heading collection, the
|
|
23
|
+
`Contact:` byline check, trailing-whitespace trimming in C5, inline-link
|
|
24
|
+
stripping in C9 and in abstract cleanup, the S1 image and `<figure>`
|
|
25
|
+
handlers, the S4 heading matcher, and a blank-line rescan in the TXT
|
|
26
|
+
structurizer. Each now runs in linear time, through procedural parsing
|
|
27
|
+
or unambiguous, bounded patterns, plus length caps where real input is
|
|
28
|
+
short (TOC lines and author lists, which are cut at a comma rather than
|
|
29
|
+
dropped). Escaped brackets in link text are kept, and backslash parity
|
|
30
|
+
decides whether a bracket opens a link. Behavior on normal
|
|
31
|
+
input is unchanged: a fuzz test compares every rewrite against the
|
|
32
|
+
regex it replaced. Known differences are limited to degenerate input.
|
|
33
|
+
A bare `#` line no longer turns the next line into the H1, empty or
|
|
34
|
+
dots-only TOC titles no longer count, and 2-letter byline names
|
|
35
|
+
written with a space before the comma are no longer matched.
|
|
36
|
+
|
|
37
|
+
- **SSRF guard accepted IPv6 forms that embed an internal IPv4 address
|
|
38
|
+
(CWE-918).** NAT64 (`64:ff9b::/96`), 6to4 (`2002::/16`), SIIT, and on
|
|
39
|
+
some Python builds IPv4-mapped addresses passed the URL guard even when
|
|
40
|
+
the embedded IPv4 was private, loopback, or a cloud metadata address.
|
|
41
|
+
The guard now extracts the embedded address and applies the IPv4 policy,
|
|
42
|
+
with the same verdict on Python 3.11 through 3.13. Teredo, NAT64
|
|
43
|
+
local-use, and deprecated IPv4-compatible addresses are rejected.
|
|
44
|
+
- **DOCX image links could make mammoth read local files or fetch URLs
|
|
45
|
+
(CWE-918, CVE-2025-11849).** The minimum mammoth version is now 1.11.0,
|
|
46
|
+
the mammoth lane passes `external_file_access=False`, and image bytes
|
|
47
|
+
are not read at all unless `--save-images` is set.
|
|
48
|
+
- **Docling treated local paths as URLs (CWE-918).** A file named like
|
|
49
|
+
`http:host#.pdf` was resolved as a URL. Both Docling call sites now pass
|
|
50
|
+
a `Path`, which Docling reads from disk.
|
|
51
|
+
- **Directory scans followed symlinks (CWE-59, CWE-61).** A link such as
|
|
52
|
+
`report.txt -> ~/.ssh/id_rsa` in a scanned tree copied its target into
|
|
53
|
+
the output. Scans now skip symlinked files, never enter symlinked
|
|
54
|
+
directories, and skip FIFOs and device nodes, each with a `SKIP` line.
|
|
55
|
+
A symlink named on the command line still converts. Extension matching
|
|
56
|
+
keeps the platform's case rules. Known limit: a file swapped for a link
|
|
57
|
+
between the scan and the read is not caught.
|
|
58
|
+
- **Terminal escape injection through filenames and error text (CWE-117,
|
|
59
|
+
CWE-150).** Filenames, output paths, and Docling exception messages in
|
|
60
|
+
`OK`, `SKIP`, `FAIL`, `WARN`, and `FALLBACK` lines reached the terminal
|
|
61
|
+
raw, so a crafted name could emit ANSI or OSC sequences, bidi
|
|
62
|
+
overrides, or forge extra status lines. Every such line now goes
|
|
63
|
+
through `safe_oneline`.
|
|
64
|
+
|
|
65
|
+
### Fixed
|
|
66
|
+
|
|
67
|
+
- **Text-lane dehyphenation never ran.** T1 `repair_line_wraps` ran before
|
|
68
|
+
T2 `dehyphenate` and joined `co-\noperation` into `co- operation`, so T2
|
|
69
|
+
never saw a line-end hyphen. T2 now runs first and decides from evidence
|
|
70
|
+
in the same document. It merges the word when the joined form appears
|
|
71
|
+
elsewhere (`cooperation`), keeps the hyphen when the hyphenated form
|
|
72
|
+
appears elsewhere (`co-pilot`), and leaves suspended hyphens
|
|
73
|
+
(`first- and second-order`) and breaks with no evidence as they were.
|
|
74
|
+
It also tolerates trailing spaces after the hyphen, handles non-ASCII
|
|
75
|
+
letters and capitalized words, and no longer rewrites fenced code.
|
|
76
|
+
- **T1 joined setext underlines and rules onto the line above.** A `---`,
|
|
77
|
+
`===`, `***`, or `___` line under text came out as `Heading ---`. These
|
|
78
|
+
lines are now structural, so T1 leaves them alone.
|
|
79
|
+
- **`--strip-links` and `--profile maximum` left links in place.** The
|
|
80
|
+
flag reached `PipelineOptions`, but no stage read it, so link URLs
|
|
81
|
+
always survived. A new shared cleanup stage, C9 `strip_links`, runs
|
|
82
|
+
after entity decoding in both lanes. It replaces `[text](url)` with
|
|
83
|
+
`text` and keeps images, fenced code, and inline code. The unused
|
|
84
|
+
`any2md.utils.strip_links()` helper is removed.
|
|
85
|
+
- **Tables lost their header row on the DOCX mammoth fallback and the
|
|
86
|
+
HTML bs4 fallback.** mammoth never emits `<thead>`, and many web tables
|
|
87
|
+
omit it. markdownify 1.x then wrote an empty header row (`| | |`) and
|
|
88
|
+
demoted the real header to a data row. Both call sites now pass
|
|
89
|
+
`table_infer_header=True`, so the first row stays the header. The
|
|
90
|
+
stale `table_heavy.fallback.md` snapshot is regenerated (the table
|
|
91
|
+
matches the original snapshot again, and two stray blank lines are gone).
|
|
92
|
+
|
|
93
|
+
- **Nested `--meta` lists crashed the batch.** `--meta a.b=x,y` (and any
|
|
94
|
+
list or table below the top level) raised `TypeError` in the
|
|
95
|
+
frontmatter emitter, and the txt, docx, and pdf converters did not catch
|
|
96
|
+
it, so one flag ended the whole run. Nested values now render like
|
|
97
|
+
top-level ones, an empty table renders as `{}`, and an unsupported value
|
|
98
|
+
fails only its own file with a `FAIL:` line.
|
|
99
|
+
- **Structured-lane S4 turned sibling headings into parent and children.**
|
|
100
|
+
`# T`, `### A`, `### B` came out as `## A`, `### B`. S4 now ranks
|
|
101
|
+
headings with the same stack the HTML lane uses, so siblings stay
|
|
102
|
+
siblings and nesting is kept. Both lanes share one block scanner
|
|
103
|
+
that follows CommonMark block rules (fence delimiter character and
|
|
104
|
+
length, indentation and tab stops, list items, paragraph interruption,
|
|
105
|
+
thematic breaks, and raw HTML blocks), so headings inside fenced code
|
|
106
|
+
or raw HTML blocks such as `<pre>` are never rewritten, and real
|
|
107
|
+
headings after them are never skipped.
|
|
108
|
+
|
|
109
|
+
### Changed
|
|
110
|
+
|
|
111
|
+
- **CI runs the full test suite** (`pytest -q`: unit, CLI, and
|
|
112
|
+
integration) instead of `tests/unit` only. `test_flags_compose` now
|
|
113
|
+
asserts the documented pre-flight exit 1 when `--save-images` /
|
|
114
|
+
`--ocr-figures` are given without Docling, instead of exit 0.
|
|
115
|
+
CONTRIBUTING drops a stale test count and a reference to a
|
|
116
|
+
network test file that does not exist.
|
|
117
|
+
|
|
7
118
|
## [1.2.0] — 2026-11-09
|
|
8
119
|
|
|
9
120
|
Drops Python 3.10 to take the Pillow 13 security fixes, and fixes
|
|
@@ -33,7 +33,8 @@ Python 3.11 or later is required.
|
|
|
33
33
|
|
|
34
34
|
## Running tests
|
|
35
35
|
|
|
36
|
-
The default invocation runs the whole suite
|
|
36
|
+
The default invocation runs the whole suite. CI runs the same command on
|
|
37
|
+
Python 3.11, 3.12, and 3.13:
|
|
37
38
|
|
|
38
39
|
```bash
|
|
39
40
|
pytest -v
|
|
@@ -59,13 +60,11 @@ UPDATE_SNAPSHOTS=1 pytest tests/integration/test_snapshots.py
|
|
|
59
60
|
Then inspect the diff (`git diff tests/fixtures/snapshots/`) before
|
|
60
61
|
committing — the snapshots are review material, not just bytes.
|
|
61
62
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
pytest tests/integration/test_url_wikipedia.py -v
|
|
68
|
-
```
|
|
63
|
+
The suite makes no network calls. URL and SSRF behavior is tested with
|
|
64
|
+
stubbed DNS and a fake opener (`tests/unit/test_http_safe.py`,
|
|
65
|
+
`tests/integration/test_html_trafilatura.py`), and HTML conversion with
|
|
66
|
+
saved fixtures under `tests/fixtures/docs/`. When you change
|
|
67
|
+
`converters/html.py`, also spot-check a few live pages by hand.
|
|
69
68
|
|
|
70
69
|
## Adding a new converter
|
|
71
70
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: any2md
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.1
|
|
4
4
|
Summary: Convert PDF, DOCX, HTML, and TXT files — or web pages by URL — to clean, LLM-optimized Markdown with YAML frontmatter.
|
|
5
5
|
Project-URL: Homepage, https://github.com/rocklambros/any2md
|
|
6
6
|
Project-URL: Issues, https://github.com/rocklambros/any2md/issues
|
|
@@ -15,7 +15,7 @@ Requires-Python: >=3.11
|
|
|
15
15
|
Requires-Dist: beautifulsoup4<5,>=4.12.0
|
|
16
16
|
Requires-Dist: defusedxml<1,>=0.7.1
|
|
17
17
|
Requires-Dist: lxml<7,>=6.1.0
|
|
18
|
-
Requires-Dist: mammoth<2,>=1.
|
|
18
|
+
Requires-Dist: mammoth<2,>=1.11.0
|
|
19
19
|
Requires-Dist: markdownify<2,>=0.13.0
|
|
20
20
|
Requires-Dist: pillow<14,>=13.0.0
|
|
21
21
|
Requires-Dist: pymupdf4llm<1,>=0.0.17
|
|
@@ -516,8 +516,8 @@ any2md uses a two-lane post-processing pipeline: one lane for layout-trustworthy
|
|
|
516
516
|
│
|
|
517
517
|
┌────────────┴────────────┐
|
|
518
518
|
│ pipeline/text.py │ text lane
|
|
519
|
-
│ T1 line-wrap repair │
|
|
520
519
|
│ T2 dehyphenate │
|
|
520
|
+
│ T1 line-wrap repair │
|
|
521
521
|
│ T9 strip repeated │ ◀ NEW (E1)
|
|
522
522
|
│ byline │
|
|
523
523
|
│ T3 dedupe paragraphs │
|
|
@@ -4,9 +4,8 @@
|
|
|
4
4
|
|
|
5
5
|
| Version | Status |
|
|
6
6
|
|---|---|
|
|
7
|
-
| 1.
|
|
8
|
-
|
|
|
9
|
-
| < 0.7 | ❌ Not supported. Please upgrade. |
|
|
7
|
+
| 1.2.x | ✅ Supported (current stable). Patch releases for security and quality fixes are published as needed. |
|
|
8
|
+
| < 1.2 | ❌ Not supported. Please upgrade to the latest 1.2.x release. |
|
|
10
9
|
|
|
11
10
|
## Reporting a vulnerability
|
|
12
11
|
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
Defenses against:
|
|
4
4
|
- non-http(s) schemes
|
|
5
5
|
- URLs resolving to non-public addresses (private, multicast, CGNAT, etc.)
|
|
6
|
+
- IPv6 forms that embed an internal IPv4 target (IPv4-mapped, NAT64,
|
|
7
|
+
6to4, Teredo, IPv4-compatible)
|
|
6
8
|
- DNS rebinding via redirect (per-hop revalidation)
|
|
7
9
|
- DNS rebinding INTRA-hop (IP pinned through to socket.create_connection,
|
|
8
10
|
so the IP we validated is the IP we connect to)
|
|
@@ -28,8 +30,57 @@ _MAX_RESPONSE_BYTES = 20 * 1024 * 1024 # 20 MB cap on body read
|
|
|
28
30
|
_USER_AGENT = f"any2md/{__version__}"
|
|
29
31
|
|
|
30
32
|
|
|
33
|
+
# Shared address space (RFC 6598). Cloud providers host metadata services
|
|
34
|
+
# here, for example Alibaba's 100.100.100.200. ``is_global`` already
|
|
35
|
+
# excludes it, but the explicit check keeps the rule visible and stable.
|
|
36
|
+
_CGNAT_V4 = ipaddress.IPv4Network("100.64.0.0/10")
|
|
37
|
+
|
|
38
|
+
# IPv6 prefixes whose low 32 bits carry an IPv4 destination. A socket or
|
|
39
|
+
# translator delivers the packet to that IPv4 address, so the IPv4 policy
|
|
40
|
+
# decides. Python's ``is_global`` does not unwrap NAT64 or IPv4-translated
|
|
41
|
+
# forms on any version, and unwraps IPv4-mapped forms only on builds with
|
|
42
|
+
# the CVE-2024-4032 fix, so the guard extracts the address itself.
|
|
43
|
+
_V4_IN_LOW_32 = (
|
|
44
|
+
ipaddress.IPv6Network("::ffff:0:0/96"), # IPv4-mapped
|
|
45
|
+
ipaddress.IPv6Network("::ffff:0:0:0/96"), # SIIT IPv4-translated
|
|
46
|
+
ipaddress.IPv6Network("64:ff9b::/96"), # NAT64 well-known prefix
|
|
47
|
+
)
|
|
48
|
+
_SIXTOFOUR = ipaddress.IPv6Network("2002::/16")
|
|
49
|
+
|
|
50
|
+
# Rejected outright. Teredo (2001::/32) tunnels to an obfuscated IPv4
|
|
51
|
+
# client through a relay. NAT64 local-use (64:ff9b:1::/48) maps into a
|
|
52
|
+
# site's own IPv4 space by definition. IPv4-compatible (::/96) is
|
|
53
|
+
# deprecated by RFC 4291 and has no legitimate use in a DNS answer.
|
|
54
|
+
_V6_DENY = (
|
|
55
|
+
ipaddress.IPv6Network("2001::/32"),
|
|
56
|
+
ipaddress.IPv6Network("64:ff9b:1::/48"),
|
|
57
|
+
ipaddress.IPv6Network("::/96"),
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _v4_allowed(addr: ipaddress.IPv4Address) -> bool:
|
|
62
|
+
return (
|
|
63
|
+
bool(addr.is_global)
|
|
64
|
+
and addr not in _CGNAT_V4
|
|
65
|
+
and not addr.is_multicast
|
|
66
|
+
and not addr.is_unspecified
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
31
70
|
def _addr_allowed(addr) -> bool:
|
|
32
|
-
"""Allowlist policy: globally routable, non-multicast, non-unspecified.
|
|
71
|
+
"""Allowlist policy: globally routable, non-multicast, non-unspecified.
|
|
72
|
+
|
|
73
|
+
IPv6 addresses that embed an IPv4 destination are judged by the IPv4
|
|
74
|
+
policy alone, so the verdict does not vary with the Python version.
|
|
75
|
+
"""
|
|
76
|
+
if addr.version == 4:
|
|
77
|
+
return _v4_allowed(addr)
|
|
78
|
+
if any(addr in net for net in _V6_DENY):
|
|
79
|
+
return False
|
|
80
|
+
if any(addr in net for net in _V4_IN_LOW_32):
|
|
81
|
+
return _v4_allowed(ipaddress.IPv4Address(int(addr) & 0xFFFFFFFF))
|
|
82
|
+
if addr in _SIXTOFOUR:
|
|
83
|
+
return _v4_allowed(ipaddress.IPv4Address((int(addr) >> 80) & 0xFFFFFFFF))
|
|
33
84
|
return bool(addr.is_global) and not addr.is_multicast and not addr.is_unspecified
|
|
34
85
|
|
|
35
86
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""CLI entry point for any2md."""
|
|
2
2
|
|
|
3
3
|
import argparse
|
|
4
|
+
import os
|
|
4
5
|
import sys
|
|
5
6
|
import time
|
|
6
7
|
from pathlib import Path
|
|
@@ -74,6 +75,63 @@ def _deep_merge_overrides(
|
|
|
74
75
|
return out
|
|
75
76
|
|
|
76
77
|
|
|
78
|
+
def _scan_directory(root: Path, recursive: bool) -> list[Path]:
|
|
79
|
+
"""Return supported regular files under ``root``, sorted.
|
|
80
|
+
|
|
81
|
+
Never follows symlinks below ``root``. An untrusted document tree can
|
|
82
|
+
hold a link such as ``report.txt -> ~/.ssh/id_rsa``, and following it
|
|
83
|
+
would copy the target into the output (F14 CWE-59, F22 CWE-61).
|
|
84
|
+
``Path.glob`` and ``Path.rglob`` return file links, and their handling
|
|
85
|
+
of directory links has changed across Python releases, so the walk
|
|
86
|
+
uses ``os.scandir`` with ``follow_symlinks=False`` to get one answer
|
|
87
|
+
on every supported version. Skipping directory links also rules out
|
|
88
|
+
symlink loops. ``root`` itself is followed when it is a link,
|
|
89
|
+
because the user named it.
|
|
90
|
+
|
|
91
|
+
Suffixes are compared after ``os.path.normcase``, which matches the
|
|
92
|
+
old glob semantics: case-insensitive on Windows, case-sensitive on
|
|
93
|
+
POSIX.
|
|
94
|
+
"""
|
|
95
|
+
suffixes = tuple(os.path.normcase(ext) for ext in SUPPORTED_EXTENSIONS)
|
|
96
|
+
found: list[Path] = []
|
|
97
|
+
# An explicit stack instead of recursion keeps deep trees clear of
|
|
98
|
+
# the interpreter recursion limit.
|
|
99
|
+
pending = [root]
|
|
100
|
+
while pending:
|
|
101
|
+
current = pending.pop()
|
|
102
|
+
try:
|
|
103
|
+
with os.scandir(current) as entries:
|
|
104
|
+
children = list(entries)
|
|
105
|
+
except OSError as e:
|
|
106
|
+
_logging.warn(f"{current} -- {e}", prefix="SKIP (unreadable)")
|
|
107
|
+
continue
|
|
108
|
+
for entry in children:
|
|
109
|
+
path = Path(entry.path)
|
|
110
|
+
try:
|
|
111
|
+
if entry.is_symlink():
|
|
112
|
+
# Report only links the old scan would have picked up,
|
|
113
|
+
# so unrelated links in a tree do not flood stderr.
|
|
114
|
+
wanted = os.path.normcase(entry.name).endswith(suffixes)
|
|
115
|
+
if wanted or (recursive and entry.is_dir()):
|
|
116
|
+
_logging.warn(str(path), prefix="SKIP (symlink)")
|
|
117
|
+
continue
|
|
118
|
+
if entry.is_dir(follow_symlinks=False):
|
|
119
|
+
if recursive:
|
|
120
|
+
pending.append(path)
|
|
121
|
+
continue
|
|
122
|
+
if not os.path.normcase(entry.name).endswith(suffixes):
|
|
123
|
+
continue
|
|
124
|
+
# FIFOs and device nodes can hang the read or never end.
|
|
125
|
+
if not entry.is_file(follow_symlinks=False):
|
|
126
|
+
_logging.warn(str(path), prefix="SKIP (not a regular file)")
|
|
127
|
+
continue
|
|
128
|
+
except OSError as e:
|
|
129
|
+
_logging.warn(f"{path} -- {e}", prefix="SKIP (unreadable)")
|
|
130
|
+
continue
|
|
131
|
+
found.append(path)
|
|
132
|
+
return sorted(found)
|
|
133
|
+
|
|
134
|
+
|
|
77
135
|
def main():
|
|
78
136
|
script_dir = Path.cwd()
|
|
79
137
|
default_output_dir = script_dir / "Text"
|
|
@@ -325,14 +383,7 @@ def main():
|
|
|
325
383
|
_logging.fail(f"{f}", prefix="NOT FOUND")
|
|
326
384
|
continue
|
|
327
385
|
if p.is_dir():
|
|
328
|
-
|
|
329
|
-
file_paths.extend(
|
|
330
|
-
sorted(
|
|
331
|
-
fp
|
|
332
|
-
for ext in SUPPORTED_EXTENSIONS
|
|
333
|
-
for fp in glob_method(f"*{ext}")
|
|
334
|
-
)
|
|
335
|
-
)
|
|
386
|
+
file_paths.extend(_scan_directory(p, args.recursive))
|
|
336
387
|
continue
|
|
337
388
|
if p.suffix.lower() not in SUPPORTED_EXTENSIONS:
|
|
338
389
|
_logging.fail(f"{f}", prefix="UNSUPPORTED FORMAT")
|
|
@@ -344,22 +395,18 @@ def main():
|
|
|
344
395
|
f"Error: not a directory: {_logging.safe_oneline(str(args.input_dir))}\n"
|
|
345
396
|
)
|
|
346
397
|
sys.exit(1)
|
|
347
|
-
|
|
348
|
-
file_paths = sorted(
|
|
349
|
-
p for ext in SUPPORTED_EXTENSIONS for p in glob_method(f"*{ext}")
|
|
350
|
-
)
|
|
398
|
+
file_paths = _scan_directory(args.input_dir, args.recursive)
|
|
351
399
|
else:
|
|
352
|
-
|
|
353
|
-
file_paths = sorted(
|
|
354
|
-
p for ext in SUPPORTED_EXTENSIONS for p in glob_method(f"*{ext}")
|
|
355
|
-
)
|
|
400
|
+
file_paths = _scan_directory(script_dir, args.recursive)
|
|
356
401
|
|
|
357
402
|
if not file_paths and not urls:
|
|
358
403
|
print("No supported files to process.")
|
|
359
404
|
sys.exit(0)
|
|
360
405
|
|
|
361
406
|
total = len(file_paths) + len(urls)
|
|
362
|
-
print(
|
|
407
|
+
print(
|
|
408
|
+
f"Processing {total} file(s) → {_logging.safe_oneline(str(args.output_dir))}/\n"
|
|
409
|
+
)
|
|
363
410
|
start = time.time()
|
|
364
411
|
ok = 0
|
|
365
412
|
fail = 0
|
|
@@ -370,7 +417,7 @@ def main():
|
|
|
370
417
|
out_name = url_to_filename(url)
|
|
371
418
|
out_path = args.output_dir / out_name
|
|
372
419
|
if out_path.exists() and not args.force:
|
|
373
|
-
print(f" SKIP (exists): {out_name}")
|
|
420
|
+
print(f" SKIP (exists): {_logging.safe_oneline(out_name)}")
|
|
374
421
|
skip += 1
|
|
375
422
|
continue
|
|
376
423
|
|
|
@@ -387,10 +434,13 @@ def main():
|
|
|
387
434
|
|
|
388
435
|
# Process local files
|
|
389
436
|
for file_path in file_paths:
|
|
437
|
+
# sanitize_filename strips only C0 and DEL, so C1 controls and bidi
|
|
438
|
+
# overrides survive into out_name. Every status line that prints it
|
|
439
|
+
# goes through safe_oneline as well.
|
|
390
440
|
out_name = sanitize_filename(file_path.name)
|
|
391
441
|
out_path = args.output_dir / out_name
|
|
392
442
|
if out_path.exists() and not args.force:
|
|
393
|
-
print(f" SKIP (exists): {out_name}")
|
|
443
|
+
print(f" SKIP (exists): {_logging.safe_oneline(out_name)}")
|
|
394
444
|
skip += 1
|
|
395
445
|
continue
|
|
396
446
|
|
|
@@ -403,9 +453,9 @@ def main():
|
|
|
403
453
|
continue
|
|
404
454
|
|
|
405
455
|
if file_size > args.max_file_size:
|
|
406
|
-
|
|
407
|
-
f"
|
|
408
|
-
|
|
456
|
+
_logging.warn(
|
|
457
|
+
f"{file_path.name} ({file_size} bytes, max {args.max_file_size})",
|
|
458
|
+
prefix="SKIP (too large)",
|
|
409
459
|
)
|
|
410
460
|
skip += 1
|
|
411
461
|
continue
|
|
@@ -9,8 +9,9 @@ produces the markdown body.
|
|
|
9
9
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
|
+
import importlib.metadata
|
|
12
13
|
import logging
|
|
13
|
-
import
|
|
14
|
+
import re
|
|
14
15
|
import zipfile
|
|
15
16
|
|
|
16
17
|
from defusedxml.ElementTree import ParseError as _XmlParseError
|
|
@@ -174,21 +175,65 @@ def _extract_via_docling(docx_path: Path) -> tuple[str, str, list[str]]:
|
|
|
174
175
|
converter = get_docx_converter()
|
|
175
176
|
with _DoclingMswordWarningCapture() as cap:
|
|
176
177
|
try:
|
|
177
|
-
|
|
178
|
+
# Docling resolves a str source as a URL first, so a file named
|
|
179
|
+
# ``http:host#.docx`` would be fetched. A Path is read from disk.
|
|
180
|
+
result = converter.convert(Path(docx_path))
|
|
178
181
|
except Exception:
|
|
179
182
|
evict_on_convert_failure("docx", None)
|
|
180
183
|
raise
|
|
181
184
|
return result.document.export_to_markdown(), "docling", list(cap.messages)
|
|
182
185
|
|
|
183
186
|
|
|
187
|
+
# mammoth 1.11.0 added ``external_file_access`` and turned it off by
|
|
188
|
+
# default (CVE-2025-11849). Earlier releases open a linked image's target
|
|
189
|
+
# with ``urlopen`` or ``open`` and accept no switch to stop it.
|
|
190
|
+
_MAMMOTH_SAFE_VERSION = (1, 11)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _installed_mammoth_version() -> str:
|
|
194
|
+
try:
|
|
195
|
+
return importlib.metadata.version("mammoth")
|
|
196
|
+
except importlib.metadata.PackageNotFoundError:
|
|
197
|
+
return ""
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _mammoth_blocks_external_files() -> bool:
|
|
201
|
+
match = re.match(r"(\d+)\.(\d+)", _installed_mammoth_version())
|
|
202
|
+
if match is None:
|
|
203
|
+
# Unknown version: assume the unsafe behavior.
|
|
204
|
+
return False
|
|
205
|
+
return (int(match.group(1)), int(match.group(2))) >= _MAMMOTH_SAFE_VERSION
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
# Emits an <img> with alt text and no src, so mammoth never calls
|
|
209
|
+
# ``image.open()``. markdownify strips the tag when images are not saved.
|
|
210
|
+
_skip_image_bytes = mammoth.images.img_element(lambda image: {})
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _mammoth_kwargs(save_images: bool) -> dict:
|
|
214
|
+
if not _mammoth_blocks_external_files():
|
|
215
|
+
# mammoth's image object does not say whether bytes come from the
|
|
216
|
+
# DOCX zip or from a link, so on an unsafe release no image is read.
|
|
217
|
+
return {"convert_image": _skip_image_bytes}
|
|
218
|
+
kwargs: dict = {"external_file_access": False}
|
|
219
|
+
if not save_images:
|
|
220
|
+
# The image is stripped from the output anyway, so reading and
|
|
221
|
+
# base64-encoding its bytes is wasted work.
|
|
222
|
+
kwargs["convert_image"] = _skip_image_bytes
|
|
223
|
+
return kwargs
|
|
224
|
+
|
|
225
|
+
|
|
184
226
|
def _extract_via_mammoth(docx_path: Path, options: PipelineOptions) -> tuple[str, str]:
|
|
185
227
|
with open(docx_path, "rb") as f:
|
|
186
|
-
html_result = mammoth.convert_to_html(f)
|
|
228
|
+
html_result = mammoth.convert_to_html(f, **_mammoth_kwargs(options.save_images))
|
|
187
229
|
md = markdownify.markdownify(
|
|
188
230
|
html_result.value,
|
|
189
231
|
heading_style="ATX",
|
|
190
232
|
strip=["img"] if not options.save_images else [],
|
|
191
233
|
bullets="-",
|
|
234
|
+
# mammoth never emits <thead>; without inference markdownify 1.x
|
|
235
|
+
# writes an empty header row and demotes the real header.
|
|
236
|
+
table_infer_header=True,
|
|
192
237
|
)
|
|
193
238
|
return md, "mammoth+markdownify"
|
|
194
239
|
|
|
@@ -206,7 +251,7 @@ def convert_docx(
|
|
|
206
251
|
out_name = sanitize_filename(docx_path.name)
|
|
207
252
|
out_path = output_dir / out_name
|
|
208
253
|
if out_path.exists() and not force:
|
|
209
|
-
print(f" SKIP (exists): {out_name}")
|
|
254
|
+
print(f" SKIP (exists): {_logging.safe_oneline(out_name)}")
|
|
210
255
|
return True
|
|
211
256
|
|
|
212
257
|
try:
|
|
@@ -214,10 +259,9 @@ def convert_docx(
|
|
|
214
259
|
# caller's choice instead of the auto-select. ``pymupdf4llm`` is
|
|
215
260
|
# invalid for DOCX — fail the file with a clear error.
|
|
216
261
|
if options.backend == "pymupdf4llm":
|
|
217
|
-
|
|
218
|
-
f"
|
|
219
|
-
f"valid for DOCX input (pymupdf4llm processes PDF only)."
|
|
220
|
-
file=sys.stderr,
|
|
262
|
+
_logging.fail(
|
|
263
|
+
f"{docx_path.name} -- backend 'pymupdf4llm' is not "
|
|
264
|
+
f"valid for DOCX input (pymupdf4llm processes PDF only)."
|
|
221
265
|
)
|
|
222
266
|
return False
|
|
223
267
|
|
|
@@ -235,10 +279,9 @@ def convert_docx(
|
|
|
235
279
|
)
|
|
236
280
|
lane = "structured"
|
|
237
281
|
except Exception as e: # noqa: BLE001 — fall back rather than fail
|
|
238
|
-
|
|
239
|
-
f"
|
|
240
|
-
f"falling back to mammoth."
|
|
241
|
-
file=sys.stderr,
|
|
282
|
+
_logging.warn(
|
|
283
|
+
f"Docling extraction failed for {docx_path.name}: {e}; "
|
|
284
|
+
f"falling back to mammoth."
|
|
242
285
|
)
|
|
243
286
|
md_text, extracted_via = _extract_via_mammoth(docx_path, options)
|
|
244
287
|
lane = "text"
|
|
@@ -258,12 +301,12 @@ def convert_docx(
|
|
|
258
301
|
and options.docx_fallback_on_warn
|
|
259
302
|
and extracted_via == "docling"
|
|
260
303
|
):
|
|
261
|
-
|
|
262
|
-
f"
|
|
304
|
+
_logging.warn(
|
|
305
|
+
f"{docx_path.name} -- Docling emitted "
|
|
263
306
|
f"{len(docling_warnings)} msword_backend warning(s); "
|
|
264
307
|
f"re-running via mammoth (content may have been dropped "
|
|
265
308
|
f"by Docling).",
|
|
266
|
-
|
|
309
|
+
prefix="FALLBACK",
|
|
267
310
|
)
|
|
268
311
|
md_text, _ = _extract_via_mammoth(docx_path, options)
|
|
269
312
|
extracted_via = "docling→mammoth (warning fallback)"
|
|
@@ -304,7 +347,10 @@ def convert_docx(
|
|
|
304
347
|
wc = meta.word_count or 0
|
|
305
348
|
suffix = f", {len(warnings)} warning(s)" if warnings else ""
|
|
306
349
|
if not is_quiet():
|
|
307
|
-
print(
|
|
350
|
+
print(
|
|
351
|
+
f" OK: {_logging.safe_oneline(out_name)} "
|
|
352
|
+
f"({wc} words, via {extracted_via}{suffix})"
|
|
353
|
+
)
|
|
308
354
|
return True
|
|
309
355
|
|
|
310
356
|
except (OSError, ValueError) as e:
|