any2md 1.0.4__tar.gz → 1.0.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. {any2md-1.0.4 → any2md-1.0.6}/CHANGELOG.md +120 -0
  2. {any2md-1.0.4 → any2md-1.0.6}/PKG-INFO +11 -8
  3. {any2md-1.0.4 → any2md-1.0.6}/any2md/__init__.py +1 -1
  4. any2md-1.0.6/any2md/_http.py +106 -0
  5. {any2md-1.0.4 → any2md-1.0.6}/any2md/cli.py +21 -1
  6. {any2md-1.0.4 → any2md-1.0.6}/any2md/config.py +16 -2
  7. {any2md-1.0.4 → any2md-1.0.6}/any2md/converters/docx.py +129 -14
  8. {any2md-1.0.4 → any2md-1.0.6}/any2md/converters/html.py +26 -65
  9. {any2md-1.0.4 → any2md-1.0.6}/any2md/converters/pdf.py +9 -3
  10. {any2md-1.0.4 → any2md-1.0.6}/any2md/converters/txt.py +2 -2
  11. {any2md-1.0.4 → any2md-1.0.6}/any2md/frontmatter.py +41 -0
  12. {any2md-1.0.4 → any2md-1.0.6}/any2md/heuristics.py +14 -22
  13. {any2md-1.0.4 → any2md-1.0.6}/any2md/pipeline/__init__.py +9 -0
  14. {any2md-1.0.4 → any2md-1.0.6}/any2md/utils.py +44 -0
  15. any2md-1.0.6/docs/superpowers/plans/2026-04-27-v1.0.6-security-hardening.md +1867 -0
  16. any2md-1.0.6/docs/superpowers/specs/2026-04-27-v1.0.6-security-hardening-design.md +477 -0
  17. {any2md-1.0.4 → any2md-1.0.6}/pyproject.toml +10 -7
  18. any2md-1.0.6/requirements.txt +10 -0
  19. any2md-1.0.6/tests/integration/test_docx_fallback_on_warn.py +254 -0
  20. any2md-1.0.6/tests/integration/test_html_trafilatura.py +105 -0
  21. any2md-1.0.6/tests/unit/test_atomic_write.py +52 -0
  22. any2md-1.0.6/tests/unit/test_config_walkup.py +53 -0
  23. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/test_heuristics.py +21 -27
  24. any2md-1.0.6/tests/unit/test_http_safe.py +202 -0
  25. any2md-1.0.6/tests/unit/test_log_sanitizer.py +27 -0
  26. any2md-1.0.6/tests/unit/test_meta_reserved_keys.py +37 -0
  27. any2md-1.0.6/tests/unit/test_safe_dir_name.py +25 -0
  28. any2md-1.0.6/tests/unit/test_zipbomb_guard.py +47 -0
  29. any2md-1.0.4/requirements.txt +0 -7
  30. any2md-1.0.4/tests/integration/test_html_trafilatura.py +0 -29
  31. {any2md-1.0.4 → any2md-1.0.6}/.gitignore +0 -0
  32. {any2md-1.0.4 → any2md-1.0.6}/CODE_OF_CONDUCT.md +0 -0
  33. {any2md-1.0.4 → any2md-1.0.6}/CONTRIBUTING.md +0 -0
  34. {any2md-1.0.4 → any2md-1.0.6}/LICENSE +0 -0
  35. {any2md-1.0.4 → any2md-1.0.6}/README.md +0 -0
  36. {any2md-1.0.4 → any2md-1.0.6}/SECURITY.md +0 -0
  37. {any2md-1.0.4 → any2md-1.0.6}/any2md/__main__.py +0 -0
  38. {any2md-1.0.4 → any2md-1.0.6}/any2md/_docling.py +0 -0
  39. {any2md-1.0.4 → any2md-1.0.6}/any2md/converters/__init__.py +0 -0
  40. {any2md-1.0.4 → any2md-1.0.6}/any2md/pipeline/cleanup.py +0 -0
  41. {any2md-1.0.4 → any2md-1.0.6}/any2md/pipeline/structured.py +0 -0
  42. {any2md-1.0.4 → any2md-1.0.6}/any2md/pipeline/text.py +0 -0
  43. {any2md-1.0.4 → any2md-1.0.6}/any2md/validators.py +0 -0
  44. {any2md-1.0.4 → any2md-1.0.6}/assets/logo-original.jpeg +0 -0
  45. {any2md-1.0.4 → any2md-1.0.6}/assets/logo.png +0 -0
  46. {any2md-1.0.4 → any2md-1.0.6}/assets/social-preview.jpg +0 -0
  47. {any2md-1.0.4 → any2md-1.0.6}/docs/architecture.md +0 -0
  48. {any2md-1.0.4 → any2md-1.0.6}/docs/cli-reference.md +0 -0
  49. {any2md-1.0.4 → any2md-1.0.6}/docs/output-format.md +0 -0
  50. {any2md-1.0.4 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase1-foundation.md +0 -0
  51. {any2md-1.0.4 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase2-docling-backend.md +0 -0
  52. {any2md-1.0.4 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase3-figures-ocr-text-lane.md +0 -0
  53. {any2md-1.0.4 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase4-polish-config-docs.md +0 -0
  54. {any2md-1.0.4 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1.0.2-implementation.md +0 -0
  55. {any2md-1.0.4 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-issue-17-toc-followups.md +0 -0
  56. {any2md-1.0.4 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-any2md-v1-design.md +0 -0
  57. {any2md-1.0.4 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-any2md-v1.0.2-design.md +0 -0
  58. {any2md-1.0.4 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-issue-17-toc-followups-design.md +0 -0
  59. {any2md-1.0.4 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-v1.0.3-empty-title-orphan-punct-design.md +0 -0
  60. {any2md-1.0.4 → any2md-1.0.6}/docs/troubleshooting.md +0 -0
  61. {any2md-1.0.4 → any2md-1.0.6}/docs/upgrading-from-0.7.md +0 -0
  62. {any2md-1.0.4 → any2md-1.0.6}/scripts/audit-outputs.py +0 -0
  63. {any2md-1.0.4 → any2md-1.0.6}/scripts/validate-phase1.sh +0 -0
  64. {any2md-1.0.4 → any2md-1.0.6}/tests/__init__.py +0 -0
  65. {any2md-1.0.4 → any2md-1.0.6}/tests/cli/__init__.py +0 -0
  66. {any2md-1.0.4 → any2md-1.0.6}/tests/cli/test_cli_backend.py +0 -0
  67. {any2md-1.0.4 → any2md-1.0.6}/tests/cli/test_cli_existing_flags.py +0 -0
  68. {any2md-1.0.4 → any2md-1.0.6}/tests/cli/test_cli_exit_codes.py +0 -0
  69. {any2md-1.0.4 → any2md-1.0.6}/tests/cli/test_cli_high_fidelity.py +0 -0
  70. {any2md-1.0.4 → any2md-1.0.6}/tests/cli/test_cli_meta.py +0 -0
  71. {any2md-1.0.4 → any2md-1.0.6}/tests/cli/test_cli_no_arxiv_lookup.py +0 -0
  72. {any2md-1.0.4 → any2md-1.0.6}/tests/cli/test_cli_ocr_save_images.py +0 -0
  73. {any2md-1.0.4 → any2md-1.0.6}/tests/cli/test_cli_quiet_verbose.py +0 -0
  74. {any2md-1.0.4 → any2md-1.0.6}/tests/cli/test_cli_strict.py +0 -0
  75. {any2md-1.0.4 → any2md-1.0.6}/tests/conftest.py +0 -0
  76. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/__init__.py +0 -0
  77. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/docs/ligatures_and_softhyphens.txt +0 -0
  78. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/docs/multi_column.pdf +0 -0
  79. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/docs/table_heavy.docx +0 -0
  80. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/docs/web_page.html +0 -0
  81. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/make_fixtures.py +0 -0
  82. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/snapshots/ligatures_and_softhyphens.md +0 -0
  83. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/snapshots/multi_column.docling.md +0 -0
  84. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/snapshots/multi_column.fallback.md +0 -0
  85. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/snapshots/table_heavy.docling.md +0 -0
  86. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/snapshots/table_heavy.fallback.md +0 -0
  87. {any2md-1.0.4 → any2md-1.0.6}/tests/fixtures/snapshots/web_page.md +0 -0
  88. {any2md-1.0.4 → any2md-1.0.6}/tests/integration/__init__.py +0 -0
  89. {any2md-1.0.4 → any2md-1.0.6}/tests/integration/test_docx_backend_override.py +0 -0
  90. {any2md-1.0.4 → any2md-1.0.6}/tests/integration/test_docx_docling.py +0 -0
  91. {any2md-1.0.4 → any2md-1.0.6}/tests/integration/test_docx_mammoth_fallback.py +0 -0
  92. {any2md-1.0.4 → any2md-1.0.6}/tests/integration/test_pdf_backend_override.py +0 -0
  93. {any2md-1.0.4 → any2md-1.0.6}/tests/integration/test_pdf_docling.py +0 -0
  94. {any2md-1.0.4 → any2md-1.0.6}/tests/integration/test_pdf_pymupdf_fallback.py +0 -0
  95. {any2md-1.0.4 → any2md-1.0.6}/tests/integration/test_save_images.py +0 -0
  96. {any2md-1.0.4 → any2md-1.0.6}/tests/integration/test_snapshots.py +0 -0
  97. {any2md-1.0.4 → any2md-1.0.6}/tests/integration/test_txt.py +0 -0
  98. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/__init__.py +0 -0
  99. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/__init__.py +0 -0
  100. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_footnote_markers.py +0 -0
  101. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_html_entities.py +0 -0
  102. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_html_entities_double_encoded.py +0 -0
  103. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_ligatures.py +0 -0
  104. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_nfc.py +0 -0
  105. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_quotes_dashes.py +0 -0
  106. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_soft_hyphens.py +0 -0
  107. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_validate.py +0 -0
  108. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_whitespace.py +0 -0
  109. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_runner.py +0 -0
  110. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_strip_orphan_punctuation.py +0 -0
  111. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_structured_body_cleanup_stages.py +0 -0
  112. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_structured_compact_tables.py +0 -0
  113. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_structured_heading_hierarchy.py +0 -0
  114. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_structured_lift_figures.py +0 -0
  115. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_structured_normalize_cites.py +0 -0
  116. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_text_dedupe_paragraphs.py +0 -0
  117. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_text_dedupe_toc.py +0 -0
  118. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_text_dedupe_toc_table.py +0 -0
  119. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_text_dehyphenate.py +0 -0
  120. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_text_repair_line_wraps.py +0 -0
  121. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_text_restore_lists_code.py +0 -0
  122. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_cover_artifacts.py +0 -0
  123. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_headers_footers.py +0 -0
  124. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_repeated_byline.py +0 -0
  125. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_web_fragments.py +0 -0
  126. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/scripts/__init__.py +0 -0
  127. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/scripts/test_audit_outputs.py +0 -0
  128. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/test_auto_id.py +0 -0
  129. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/test_config.py +0 -0
  130. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/test_content_hash.py +0 -0
  131. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/test_docling_helper.py +0 -0
  132. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/test_frontmatter_compose.py +0 -0
  133. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/test_frontmatter_helpers.py +0 -0
  134. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/test_meta_overrides.py +0 -0
  135. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/test_pdf_complexity.py +0 -0
  136. {any2md-1.0.4 → any2md-1.0.6}/tests/unit/test_validators.py +0 -0
@@ -4,6 +4,126 @@ All notable changes to this project will be documented in this file.
4
4
 
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
6
6
 
7
+ ## [1.0.6] — 2026-04-27
8
+
9
+ Security hardening release. Closes 8 actionable findings from the
10
+ 2026-04-27 multi-tool audit (semgrep + trivy + gitleaks + bandit +
11
+ pip-audit + manual review by the security-engineer agent + Opus 4.7).
12
+ No public API changes; behavior unchanged for legitimate inputs.
13
+
14
+ ### Security
15
+
16
+ - **F1 (HIGH)** SSRF: replaced `trafilatura.fetch_url` and the bare
17
+ `urllib.request.urlopen` paths with a new `any2md/_http.py` module
18
+ that does manual redirect walking with per-hop revalidation. Defends
19
+ against DNS-rebinding and redirect-based bypass of the IP-class
20
+ check. New `validate_url` rejects non-http(s) schemes and
21
+ private/loopback/link-local/reserved targets. 20 MB body cap, 3-hop
22
+ redirect cap, redirect-loop detection. Applied to `convert_url`,
23
+ `_http_last_modified`, and `arxiv_lookup`.
24
+ - **F5 (MED)** `--meta` integrity: `content_hash`, `extracted_via`,
25
+ `source_file`, `lane`, `token_estimate`, `recommended_chunk_level`
26
+ are now **reserved** and silently filtered from `--meta` /
27
+ `--meta-file` / `.any2md.toml` overrides with a stderr `WARN` per
28
+ drop. Defense-in-depth filter also runs inside `frontmatter.compose`.
29
+ - **F3 (MED)** TOML auto-discovery now stops at the first ancestor
30
+ containing `.git` / `pyproject.toml` / `setup.py` / `setup.cfg` /
31
+ `package.json` / `.any2md.toml.boundary`. New `--no-config` flag
32
+ fully disables discovery.
33
+ - **F4 (MED)** DOCX zip-bomb: `_safe_zip_open` enforces a 1 MB cap on
34
+ the declared uncompressed size of `core.xml` and `app.xml` before
35
+ parsing. Oversized members raise `ValueError` → clean per-file FAIL.
36
+ - **F-CVE (MED)** Dependency floors bumped: `lxml>=6.1.0,<7`
37
+ (CVE-2026-41066), `pillow>=12.2.0,<13` (CVE-2026-25990,
38
+ CVE-2026-40192), `urllib3>=2.6.3,<3` (CVE-2026-21441). All ranges
39
+ acquire upper bounds (compat envelope). `requirements.txt` becomes
40
+ a development/CI lockfile with exact pins.
41
+ - **F6 (LOW)** Atomic writes: new `any2md.utils.atomic_write_text`
42
+ writes via `tempfile.mkstemp` + `os.replace`. Refuses to overwrite
43
+ a pre-existing symlink at the output path. Wired through
44
+ `pdf/docx/html/txt` converters; PDF image extraction adds a symlink
45
+ check before `pil_image.save`.
46
+ - **F2 (LOW)** Control-char sanitizer strips C0/C1 controls (preserves
47
+ `\t`, `\n`) from captured Docling `msword_backend` warnings before
48
+ forwarding them to `collected_warnings()` and stderr. Defends
49
+ terminals from ANSI-escape spoofing.
50
+ - **F11 (LOW)** PDF image-dir name now goes through `safe_dir_name`
51
+ (alphanumeric/`_`/`-` only). Closes a `Path.stem == ".."` corner
52
+ case that co-mingled images with markdown outputs.
53
+ - **XXE defense-in-depth**: migrated `xml.etree.ElementTree` →
54
+ `defusedxml.ElementTree` in `docx.py` and `heuristics.py`. Stdlib
55
+ was already safe in Python 3.7+; this silences bandit B405/B314
56
+ permanently and protects against future stdlib regressions.
57
+
58
+ ### Added
59
+
60
+ - New module `any2md/_http.py` (`validate_url`, `safe_fetch`).
61
+ - New CLI flag `--no-config`.
62
+ - New helpers `any2md.utils.atomic_write_text` and `safe_dir_name`.
63
+ - New runtime dependency: `defusedxml>=0.7.1,<1`.
64
+
65
+ ### Tests
66
+
67
+ - 7 new unit-test files (40 new tests): `test_http_safe.py`,
68
+ `test_meta_reserved_keys.py`, `test_config_walkup.py`,
69
+ `test_zipbomb_guard.py`, `test_atomic_write.py`,
70
+ `test_safe_dir_name.py`, `test_log_sanitizer.py`.
71
+ - Existing 290 tests continue to pass.
72
+ - Bandit audit went from **9 findings** (3 medium, 6 low) to **1 low**
73
+ (legitimate `try/except/pass` in arxiv `_warn`).
74
+ - 5-DOCX regression run reproduces v1.0.5 word counts exactly
75
+ (978/6726/15566/7811/1477).
76
+
77
+ ### Audit summary
78
+
79
+ | ID | Title | Severity |
80
+ |----|-------|----------|
81
+ | F1 | SSRF: DNS rebind + redirect bypass | High |
82
+ | F5 | `--meta` clobbers reserved frontmatter | Medium |
83
+ | F3 | TOML discovery walks above project root | Medium |
84
+ | F4 | DOCX zip-bomb amplification | Medium |
85
+ | F-CVE | Transitive deps with known CVEs | Medium |
86
+ | F6 | Symlink-following on output writes | Low |
87
+ | F2 | Control-char passthrough in Docling logs | Low |
88
+ | F11 | PDF stem `..` corner case | Low |
89
+
90
+ ## [1.0.5] — 2026-04-27
91
+
92
+ Patch release. Recovers DOCX content that Docling's msword backend
93
+ silently drops on malformed list structures, by auto-retrying
94
+ affected files through the mammoth lane and forwarding the
95
+ captured Docling warnings into any2md's run-level warning bucket.
96
+ Real-world impact observed during validation: a 12,740-word
97
+ Docling output for a policy-template DOCX recovered to 15,566
98
+ words (+22%) when re-rendered via mammoth.
99
+
100
+ ### Added
101
+ - DOCX converter now auto-retries files through the mammoth lane
102
+ when Docling emits any warning from
103
+ `docling.backend.msword_backend` during conversion. Docling's
104
+ DOCX backend silently drops list items in a known
105
+ malformed-input path (`msword_backend.py:1377/1675` —
106
+ "Parent element of the list item is not a ListGroup. The list
107
+ item will be ignored."), so a warning means the Docling
108
+ Markdown is missing content. On fallback, `extracted_via` is
109
+ set to `"docling→mammoth (warning fallback)"`, the lane
110
+ switches to `text`, the captured Docling warnings are
111
+ forwarded into the run-level warning bucket (so `--strict`
112
+ still fails), and a `FALLBACK:` line is printed to stderr.
113
+ Default: enabled. Disable with `--no-docx-fallback-on-warn`
114
+ (e.g. when comparing backends explicitly).
115
+ - New `PipelineOptions.docx_fallback_on_warn: bool = True`.
116
+ - New CLI flag pair `--docx-fallback-on-warn` /
117
+ `--no-docx-fallback-on-warn`.
118
+
119
+ ### Tests
120
+ - New `tests/integration/test_docx_fallback_on_warn.py` covers
121
+ the warning-capture context manager, the default-on fallback
122
+ path (output swap, `extracted_via`, captured warning forwarded
123
+ into `collected_warnings()`, user-visible message), the
124
+ `--no-docx-fallback-on-warn` opt-out, the no-warning
125
+ fast-path, and the explicit `--backend docling` interaction.
126
+
7
127
  ## [1.0.4] — 2026-04-26
8
128
 
9
129
  Patch release. Two follow-ups deferred from v1.0.3 (#17): the T7
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: any2md
3
- Version: 1.0.4
3
+ Version: 1.0.6
4
4
  Summary: Convert PDF, DOCX, HTML, and TXT files — or web pages by URL — to clean, LLM-optimized Markdown with YAML frontmatter.
5
5
  Project-URL: Homepage, https://github.com/rocklambros/any2md
6
6
  Project-URL: Issues, https://github.com/rocklambros/any2md/issues
@@ -12,13 +12,16 @@ Classifier: Operating System :: OS Independent
12
12
  Classifier: Programming Language :: Python :: 3
13
13
  Classifier: Topic :: Text Processing :: Markup :: Markdown
14
14
  Requires-Python: >=3.10
15
- Requires-Dist: beautifulsoup4>=4.12.0
16
- Requires-Dist: lxml>=5.0.0
17
- Requires-Dist: mammoth>=1.6.0
18
- Requires-Dist: markdownify>=0.13.0
19
- Requires-Dist: pymupdf4llm>=0.0.17
20
- Requires-Dist: pymupdf>=1.24.0
21
- Requires-Dist: trafilatura>=1.12.0
15
+ Requires-Dist: beautifulsoup4<5,>=4.12.0
16
+ Requires-Dist: defusedxml<1,>=0.7.1
17
+ Requires-Dist: lxml<7,>=6.1.0
18
+ Requires-Dist: mammoth<2,>=1.6.0
19
+ Requires-Dist: markdownify<2,>=0.13.0
20
+ Requires-Dist: pillow<13,>=12.2.0
21
+ Requires-Dist: pymupdf4llm<1,>=0.0.17
22
+ Requires-Dist: pymupdf<2,>=1.24.0
23
+ Requires-Dist: trafilatura<3,>=1.12.0
24
+ Requires-Dist: urllib3<3,>=2.6.3
22
25
  Provides-Extra: dev
23
26
  Requires-Dist: pytest-snapshot>=0.9.0; extra == 'dev'
24
27
  Requires-Dist: pytest>=8.0.0; extra == 'dev'
@@ -1,3 +1,3 @@
1
1
  """Convert PDF, DOCX, HTML, and TXT files to LLM-optimized Markdown."""
2
2
 
3
- __version__ = "1.0.4"
3
+ __version__ = "1.0.6"
@@ -0,0 +1,106 @@
1
+ """SSRF-safe HTTP fetching utilities for any2md.
2
+
3
+ Centralizes URL scheme + IP validation and provides a fetcher that
4
+ walks redirects manually and revalidates the host on each hop. This
5
+ defends against:
6
+ - DNS rebinding (host -> public IP for validator, private IP for fetch)
7
+ - Redirect-based SSRF (public landing page -> 302 to 169.254.169.254)
8
+ - Non-http(s) schemes (file://, gopher://, etc.)
9
+ - Decompression / huge-response DoS (response-size cap)
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import ipaddress
15
+ import socket
16
+ import urllib.error
17
+ import urllib.parse
18
+ import urllib.request
19
+
20
+ _MAX_REDIRECT_HOPS = 3
21
+ _FETCH_TIMEOUT = 15 # seconds
22
+ _MAX_RESPONSE_BYTES = 20 * 1024 * 1024 # 20 MB cap on body read
23
+ _USER_AGENT = "any2md/1.0.6"
24
+
25
+
26
+ def validate_url(url: str) -> str | None:
27
+ """Validate URL scheme + host. Returns an error message or None.
28
+
29
+ One-shot DNS lookup; rebind protection requires re-checking on
30
+ every redirect hop (handled by ``safe_fetch``).
31
+ """
32
+ parsed = urllib.parse.urlparse(url)
33
+ if parsed.scheme not in ("http", "https"):
34
+ return f"unsupported scheme: {parsed.scheme!r}"
35
+ if not parsed.hostname:
36
+ return f"no hostname in URL: {url}"
37
+ try:
38
+ infos = socket.getaddrinfo(parsed.hostname, None, type=socket.SOCK_STREAM)
39
+ except socket.gaierror:
40
+ return f"cannot resolve host: {parsed.hostname}"
41
+ for *_, sockaddr in infos:
42
+ try:
43
+ addr = ipaddress.ip_address(sockaddr[0])
44
+ except ValueError:
45
+ continue
46
+ if (
47
+ addr.is_private
48
+ or addr.is_reserved
49
+ or addr.is_loopback
50
+ or addr.is_link_local
51
+ ):
52
+ return f"URL resolves to disallowed address: {addr}"
53
+ return None
54
+
55
+
56
+ class _NoFollowRedirect(urllib.request.HTTPRedirectHandler):
57
+ """Disable urllib's automatic redirect following."""
58
+
59
+ def redirect_request(self, *args, **kwargs): # noqa: ARG002
60
+ return None
61
+
62
+
63
+ def safe_fetch(
64
+ url: str,
65
+ *,
66
+ method: str = "GET",
67
+ max_hops: int = _MAX_REDIRECT_HOPS,
68
+ ) -> tuple[bytes | None, dict | None, str | None]:
69
+ """Fetch ``url`` with manual redirect walking + per-hop revalidation.
70
+
71
+ Returns ``(body, headers, None)`` on 2xx, or ``(None, None, error)``
72
+ on rejection / failure. ``headers`` is the response headers as a
73
+ plain ``dict``.
74
+ """
75
+ visited: list[str] = []
76
+ current = url
77
+ opener = urllib.request.build_opener(_NoFollowRedirect())
78
+ for hop in range(max_hops + 1):
79
+ if current in visited:
80
+ return None, None, "redirect loop"
81
+ visited.append(current)
82
+ err = validate_url(current)
83
+ if err:
84
+ return None, None, err
85
+ req = urllib.request.Request(
86
+ current, method=method, headers={"User-Agent": _USER_AGENT}
87
+ )
88
+ try:
89
+ resp = opener.open(req, timeout=_FETCH_TIMEOUT)
90
+ except urllib.error.HTTPError as e:
91
+ if e.code in (301, 302, 303, 307, 308):
92
+ if hop >= max_hops:
93
+ return None, None, f"too many redirects (>{max_hops})"
94
+ location = e.headers.get("Location") if e.headers else None
95
+ if not location:
96
+ return None, None, f"HTTP {e.code} without Location"
97
+ current = urllib.parse.urljoin(current, location)
98
+ continue
99
+ return None, None, f"HTTP {e.code}"
100
+ except (urllib.error.URLError, OSError, TimeoutError) as e:
101
+ return None, None, f"fetch error: {e}"
102
+ body = resp.read(_MAX_RESPONSE_BYTES + 1)
103
+ if len(body) > _MAX_RESPONSE_BYTES:
104
+ return None, None, f"response exceeds {_MAX_RESPONSE_BYTES} bytes"
105
+ return body, dict(resp.headers), None
106
+ return None, None, f"too many redirects (>{max_hops})"
@@ -20,6 +20,7 @@ from any2md.converters import (
20
20
  set_output_mode,
21
21
  )
22
22
  from any2md.converters.html import convert_url
23
+ from any2md.frontmatter import filter_reserved_overrides
23
24
  from any2md.pipeline import PipelineOptions
24
25
  from any2md.utils import sanitize_filename, url_to_filename
25
26
 
@@ -169,6 +170,21 @@ def main():
169
170
  action="store_true",
170
171
  help="Promote pipeline validation warnings to errors (exit 3).",
171
172
  )
173
+ parser.add_argument(
174
+ "--no-config",
175
+ action="store_true",
176
+ help="Skip .any2md.toml auto-discovery (overrides default walk-up).",
177
+ )
178
+ parser.add_argument(
179
+ "--docx-fallback-on-warn",
180
+ action=argparse.BooleanOptionalAction,
181
+ default=True,
182
+ help="DOCX-only. When Docling emits any msword_backend warning "
183
+ "during conversion, auto-retry the file through mammoth and use "
184
+ "that output instead. Docling silently drops list items on "
185
+ "malformed input, so a warning means content is missing. "
186
+ "Default: enabled. Disable with --no-docx-fallback-on-warn.",
187
+ )
172
188
  parser.add_argument(
173
189
  "--quiet",
174
190
  "-q",
@@ -212,7 +228,7 @@ def main():
212
228
  overrides: dict[str, Any] = {}
213
229
  auto_id_prefix, auto_id_type_code = "LOCAL", "DOC"
214
230
 
215
- discovered = discover_config()
231
+ discovered = None if args.no_config else discover_config()
216
232
  if discovered is not None:
217
233
  cfg = load_toml(discovered)
218
234
  overrides = _deep_merge_overrides(overrides, extract_meta_overrides(cfg))
@@ -239,6 +255,9 @@ def main():
239
255
  print(f"Error: {e}", file=sys.stderr)
240
256
  sys.exit(1)
241
257
  overrides = _deep_merge_overrides(overrides, cli_meta)
258
+ overrides = filter_reserved_overrides(
259
+ overrides, source_label="--meta/--meta-file/.any2md.toml"
260
+ )
242
261
 
243
262
  if (
244
263
  args.high_fidelity
@@ -272,6 +291,7 @@ def main():
272
291
  frontmatter_overrides=overrides or None,
273
292
  backend=args.backend,
274
293
  arxiv_lookup=not args.no_arxiv_lookup,
294
+ docx_fallback_on_warn=args.docx_fallback_on_warn,
275
295
  )
276
296
 
277
297
  # CLI-only output controls (not part of PipelineOptions).
@@ -42,17 +42,31 @@ def load_toml(path: Path) -> dict[str, Any]:
42
42
  return {}
43
43
 
44
44
 
45
+ _PROJECT_BOUNDARY_MARKERS = (
46
+ ".git",
47
+ "pyproject.toml",
48
+ "setup.py",
49
+ "setup.cfg",
50
+ "package.json",
51
+ ".any2md.toml.boundary",
52
+ )
53
+
54
+
45
55
  def discover_config(start: Path | None = None) -> Path | None:
46
56
  """Walk up from ``start`` (default: cwd) looking for ``.any2md.toml``.
47
57
 
48
- Returns the first match found or ``None`` if no config exists in
49
- any ancestor directory.
58
+ Stops at the first ancestor containing a project boundary marker
59
+ (``.git``, ``pyproject.toml``, ``setup.py``, ``setup.cfg``,
60
+ ``package.json``, or ``.any2md.toml.boundary``) so config in
61
+ unrelated parent directories cannot leak into a project's run.
50
62
  """
51
63
  cur = (start or Path.cwd()).resolve()
52
64
  while True:
53
65
  candidate = cur / ".any2md.toml"
54
66
  if candidate.is_file():
55
67
  return candidate
68
+ if any((cur / m).exists() for m in _PROJECT_BOUNDARY_MARKERS):
69
+ return None
56
70
  if cur.parent == cur:
57
71
  return None
58
72
  cur = cur.parent
@@ -9,9 +9,13 @@ produces the markdown body.
9
9
 
10
10
  from __future__ import annotations
11
11
 
12
+ import logging
13
+ import re
12
14
  import sys
13
- import xml.etree.ElementTree as ET
14
15
  import zipfile
16
+
17
+ from defusedxml.ElementTree import ParseError as _XmlParseError
18
+ from defusedxml.ElementTree import parse as _xml_parse
15
19
  from datetime import date
16
20
  from pathlib import Path
17
21
 
@@ -24,7 +28,80 @@ from any2md.converters import add_warnings, is_quiet
24
28
  from any2md.frontmatter import SourceMeta, compose
25
29
  from any2md.heuristics import filter_organization
26
30
  from any2md.pipeline import PipelineOptions
27
- from any2md.utils import sanitize_filename
31
+ from any2md.utils import atomic_write_text, sanitize_filename
32
+
33
+
34
+ _DOCLING_MSWORD_LOGGER = "docling.backend.msword_backend"
35
+
36
+ _LOG_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")
37
+
38
+
39
+ def _sanitize_log_text(s: str) -> str:
40
+ """Strip ASCII C0/C1 control chars from a log message before re-emission.
41
+
42
+ Defends terminals from ANSI-escape spoofing planted in malicious DOCX
43
+ content that surfaces as a Docling msword_backend warning. ``\\t``
44
+ and ``\\n`` are preserved (not in the strip class).
45
+ """
46
+ return _LOG_CONTROL_CHARS_RE.sub("", s)
47
+
48
+
49
+ _MAX_DOCX_METADATA_SIZE = 1 * 1024 * 1024 # 1 MB; core.xml/app.xml are typically <10 KB
50
+
51
+
52
+ def _safe_zip_open(
53
+ z: zipfile.ZipFile, name: str, max_size: int = _MAX_DOCX_METADATA_SIZE
54
+ ):
55
+ """Open a fixed-name DOCX member after a declared-size sanity check.
56
+
57
+ Defends against zip-bomb amplification: a malicious DOCX can declare
58
+ an uncompressed_size of 10 GB for ``core.xml`` and bomb the XML
59
+ parser. Raises ``ValueError`` (caught by ``convert_docx``'s existing
60
+ handler -> file fails cleanly).
61
+ """
62
+ info = z.getinfo(name)
63
+ if info.file_size > max_size:
64
+ raise ValueError(
65
+ f"{name} declared size {info.file_size} exceeds {max_size}-byte limit"
66
+ )
67
+ return z.open(name)
68
+
69
+
70
+ class _DoclingMswordWarningCapture:
71
+ """Buffer WARNING+ records from Docling's DOCX backend logger.
72
+
73
+ Used as a context manager around a Docling DOCX conversion to detect
74
+ upstream's silent list-item-drop guard (msword_backend.py:1377/1675).
75
+ Any captured records signal that Docling's Markdown output is
76
+ incomplete; ``convert_docx`` consults this to decide whether to
77
+ auto-retry with the mammoth lane.
78
+ """
79
+
80
+ def __init__(self) -> None:
81
+ self._records: list[logging.LogRecord] = []
82
+ self._handler: logging.Handler | None = None
83
+
84
+ def __enter__(self) -> _DoclingMswordWarningCapture:
85
+ records = self._records
86
+
87
+ class _BufferHandler(logging.Handler):
88
+ def emit(self, record: logging.LogRecord) -> None:
89
+ records.append(record)
90
+
91
+ handler = _BufferHandler(level=logging.WARNING)
92
+ logging.getLogger(_DOCLING_MSWORD_LOGGER).addHandler(handler)
93
+ self._handler = handler
94
+ return self
95
+
96
+ def __exit__(self, *_exc: object) -> None:
97
+ if self._handler is not None:
98
+ logging.getLogger(_DOCLING_MSWORD_LOGGER).removeHandler(self._handler)
99
+ self._handler = None
100
+
101
+ @property
102
+ def messages(self) -> list[str]:
103
+ return [r.getMessage() for r in self._records]
104
+
28
105
 
29
106
  _NS_CORE = {
30
107
  "dc": "http://purl.org/dc/elements/1.1/",
@@ -48,8 +125,8 @@ def _read_docx_metadata(docx_path: Path) -> dict[str, object]:
48
125
  try:
49
126
  with zipfile.ZipFile(docx_path) as z:
50
127
  try:
51
- with z.open("docProps/core.xml") as f:
52
- root = ET.parse(f).getroot()
128
+ with _safe_zip_open(z, "docProps/core.xml") as f:
129
+ root = _xml_parse(f).getroot()
53
130
  title = root.findtext("dc:title", namespaces=_NS_CORE)
54
131
  if title:
55
132
  out["title_hint"] = title.strip()
@@ -64,8 +141,8 @@ def _read_docx_metadata(docx_path: Path) -> dict[str, object]:
64
141
  except KeyError:
65
142
  pass
66
143
  try:
67
- with z.open("docProps/app.xml") as f:
68
- root = ET.parse(f).getroot()
144
+ with _safe_zip_open(z, "docProps/app.xml") as f:
145
+ root = _xml_parse(f).getroot()
69
146
  company = root.findtext("ext:Company", namespaces=_NS_APP)
70
147
  application = root.findtext("ext:Application", namespaces=_NS_APP)
71
148
  # v1.0.2: Company takes priority for `organization` (real
@@ -85,18 +162,24 @@ def _read_docx_metadata(docx_path: Path) -> dict[str, object]:
85
162
  out["produced_by"] = app_result.produced_by
86
163
  except KeyError:
87
164
  pass
88
- except (zipfile.BadZipFile, ET.ParseError):
165
+ except (zipfile.BadZipFile, _XmlParseError):
89
166
  pass
90
167
  return out
91
168
 
92
169
 
93
- def _extract_via_docling(docx_path: Path) -> tuple[str, str]:
94
- """Returns (markdown, 'docling'). Raises on Docling errors."""
170
+ def _extract_via_docling(docx_path: Path) -> tuple[str, str, list[str]]:
171
+ """Returns (markdown, 'docling', captured_warnings).
172
+
173
+ ``captured_warnings`` is the list of WARNING+ messages emitted by
174
+ Docling's ``msword_backend`` logger during the conversion. It is
175
+ empty for clean runs. Raises on Docling errors.
176
+ """
95
177
  from docling.document_converter import DocumentConverter
96
178
 
97
179
  converter = DocumentConverter()
98
- result = converter.convert(str(docx_path))
99
- return result.document.export_to_markdown(), "docling"
180
+ with _DoclingMswordWarningCapture() as cap:
181
+ result = converter.convert(str(docx_path))
182
+ return result.document.export_to_markdown(), "docling", list(cap.messages)
100
183
 
101
184
 
102
185
  def _extract_via_mammoth(docx_path: Path, options: PipelineOptions) -> tuple[str, str]:
@@ -139,15 +222,18 @@ def convert_docx(
139
222
  )
140
223
  return False
141
224
 
225
+ docling_warnings: list[str] = []
142
226
  if options.backend == "mammoth":
143
227
  md_text, extracted_via = _extract_via_mammoth(docx_path, options)
144
228
  lane = "text"
145
229
  elif options.backend == "docling":
146
- md_text, extracted_via = _extract_via_docling(docx_path)
230
+ md_text, extracted_via, docling_warnings = _extract_via_docling(docx_path)
147
231
  lane = "structured"
148
232
  elif has_docling():
149
233
  try:
150
- md_text, extracted_via = _extract_via_docling(docx_path)
234
+ md_text, extracted_via, docling_warnings = _extract_via_docling(
235
+ docx_path
236
+ )
151
237
  lane = "structured"
152
238
  except Exception as e: # noqa: BLE001 — fall back rather than fail
153
239
  print(
@@ -157,10 +243,39 @@ def convert_docx(
157
243
  )
158
244
  md_text, extracted_via = _extract_via_mammoth(docx_path, options)
159
245
  lane = "text"
246
+ docling_warnings = []
160
247
  else:
161
248
  md_text, extracted_via = _extract_via_mammoth(docx_path, options)
162
249
  lane = "text"
163
250
 
251
+ # v1.0.5: Docling's DOCX backend silently drops list items in a
252
+ # known malformed-input path. When any msword_backend warning
253
+ # fires, swap the Docling output for the mammoth lane (different
254
+ # parser, generally more permissive). Captured warnings are
255
+ # forwarded into the run-level warning bucket so --strict still
256
+ # fails on them.
257
+ if (
258
+ docling_warnings
259
+ and options.docx_fallback_on_warn
260
+ and extracted_via == "docling"
261
+ ):
262
+ print(
263
+ f" FALLBACK: {docx_path.name} -- Docling emitted "
264
+ f"{len(docling_warnings)} msword_backend warning(s); "
265
+ f"re-running via mammoth (content may have been dropped "
266
+ f"by Docling).",
267
+ file=sys.stderr,
268
+ )
269
+ md_text, _ = _extract_via_mammoth(docx_path, options)
270
+ extracted_via = "docling→mammoth (warning fallback)"
271
+ lane = "text"
272
+ add_warnings(
273
+ [
274
+ f"docling.msword_backend: {_sanitize_log_text(m)}"
275
+ for m in docling_warnings
276
+ ]
277
+ )
278
+
164
279
  md_text, warnings = pipeline.run(md_text, lane, options)
165
280
  add_warnings(warnings)
166
281
 
@@ -186,7 +301,7 @@ def convert_docx(
186
301
  full = compose(md_text, meta, options, overrides=options.frontmatter_overrides)
187
302
 
188
303
  output_dir.mkdir(parents=True, exist_ok=True)
189
- out_path.write_text(full, encoding="utf-8", newline="\n")
304
+ atomic_write_text(out_path, full)
190
305
  wc = meta.word_count or 0
191
306
  suffix = f", {len(warnings)} warning(s)" if warnings else ""
192
307
  if not is_quiet():