any2md 1.0.5__tar.gz → 1.0.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. {any2md-1.0.5 → any2md-1.0.6}/CHANGELOG.md +83 -0
  2. {any2md-1.0.5 → any2md-1.0.6}/PKG-INFO +11 -8
  3. {any2md-1.0.5 → any2md-1.0.6}/any2md/__init__.py +1 -1
  4. any2md-1.0.6/any2md/_http.py +106 -0
  5. {any2md-1.0.5 → any2md-1.0.6}/any2md/cli.py +10 -1
  6. {any2md-1.0.5 → any2md-1.0.6}/any2md/config.py +16 -2
  7. {any2md-1.0.5 → any2md-1.0.6}/any2md/converters/docx.py +50 -9
  8. {any2md-1.0.5 → any2md-1.0.6}/any2md/converters/html.py +26 -65
  9. {any2md-1.0.5 → any2md-1.0.6}/any2md/converters/pdf.py +9 -3
  10. {any2md-1.0.5 → any2md-1.0.6}/any2md/converters/txt.py +2 -2
  11. {any2md-1.0.5 → any2md-1.0.6}/any2md/frontmatter.py +41 -0
  12. {any2md-1.0.5 → any2md-1.0.6}/any2md/heuristics.py +14 -22
  13. {any2md-1.0.5 → any2md-1.0.6}/any2md/utils.py +44 -0
  14. any2md-1.0.6/docs/superpowers/plans/2026-04-27-v1.0.6-security-hardening.md +1867 -0
  15. any2md-1.0.6/docs/superpowers/specs/2026-04-27-v1.0.6-security-hardening-design.md +477 -0
  16. {any2md-1.0.5 → any2md-1.0.6}/pyproject.toml +10 -7
  17. any2md-1.0.6/requirements.txt +10 -0
  18. any2md-1.0.6/tests/integration/test_html_trafilatura.py +105 -0
  19. any2md-1.0.6/tests/unit/test_atomic_write.py +52 -0
  20. any2md-1.0.6/tests/unit/test_config_walkup.py +53 -0
  21. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_heuristics.py +21 -27
  22. any2md-1.0.6/tests/unit/test_http_safe.py +202 -0
  23. any2md-1.0.6/tests/unit/test_log_sanitizer.py +27 -0
  24. any2md-1.0.6/tests/unit/test_meta_reserved_keys.py +37 -0
  25. any2md-1.0.6/tests/unit/test_safe_dir_name.py +25 -0
  26. any2md-1.0.6/tests/unit/test_zipbomb_guard.py +47 -0
  27. any2md-1.0.5/requirements.txt +0 -7
  28. any2md-1.0.5/tests/integration/test_html_trafilatura.py +0 -29
  29. {any2md-1.0.5 → any2md-1.0.6}/.gitignore +0 -0
  30. {any2md-1.0.5 → any2md-1.0.6}/CODE_OF_CONDUCT.md +0 -0
  31. {any2md-1.0.5 → any2md-1.0.6}/CONTRIBUTING.md +0 -0
  32. {any2md-1.0.5 → any2md-1.0.6}/LICENSE +0 -0
  33. {any2md-1.0.5 → any2md-1.0.6}/README.md +0 -0
  34. {any2md-1.0.5 → any2md-1.0.6}/SECURITY.md +0 -0
  35. {any2md-1.0.5 → any2md-1.0.6}/any2md/__main__.py +0 -0
  36. {any2md-1.0.5 → any2md-1.0.6}/any2md/_docling.py +0 -0
  37. {any2md-1.0.5 → any2md-1.0.6}/any2md/converters/__init__.py +0 -0
  38. {any2md-1.0.5 → any2md-1.0.6}/any2md/pipeline/__init__.py +0 -0
  39. {any2md-1.0.5 → any2md-1.0.6}/any2md/pipeline/cleanup.py +0 -0
  40. {any2md-1.0.5 → any2md-1.0.6}/any2md/pipeline/structured.py +0 -0
  41. {any2md-1.0.5 → any2md-1.0.6}/any2md/pipeline/text.py +0 -0
  42. {any2md-1.0.5 → any2md-1.0.6}/any2md/validators.py +0 -0
  43. {any2md-1.0.5 → any2md-1.0.6}/assets/logo-original.jpeg +0 -0
  44. {any2md-1.0.5 → any2md-1.0.6}/assets/logo.png +0 -0
  45. {any2md-1.0.5 → any2md-1.0.6}/assets/social-preview.jpg +0 -0
  46. {any2md-1.0.5 → any2md-1.0.6}/docs/architecture.md +0 -0
  47. {any2md-1.0.5 → any2md-1.0.6}/docs/cli-reference.md +0 -0
  48. {any2md-1.0.5 → any2md-1.0.6}/docs/output-format.md +0 -0
  49. {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase1-foundation.md +0 -0
  50. {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase2-docling-backend.md +0 -0
  51. {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase3-figures-ocr-text-lane.md +0 -0
  52. {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase4-polish-config-docs.md +0 -0
  53. {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1.0.2-implementation.md +0 -0
  54. {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-issue-17-toc-followups.md +0 -0
  55. {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-any2md-v1-design.md +0 -0
  56. {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-any2md-v1.0.2-design.md +0 -0
  57. {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-issue-17-toc-followups-design.md +0 -0
  58. {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-v1.0.3-empty-title-orphan-punct-design.md +0 -0
  59. {any2md-1.0.5 → any2md-1.0.6}/docs/troubleshooting.md +0 -0
  60. {any2md-1.0.5 → any2md-1.0.6}/docs/upgrading-from-0.7.md +0 -0
  61. {any2md-1.0.5 → any2md-1.0.6}/scripts/audit-outputs.py +0 -0
  62. {any2md-1.0.5 → any2md-1.0.6}/scripts/validate-phase1.sh +0 -0
  63. {any2md-1.0.5 → any2md-1.0.6}/tests/__init__.py +0 -0
  64. {any2md-1.0.5 → any2md-1.0.6}/tests/cli/__init__.py +0 -0
  65. {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_backend.py +0 -0
  66. {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_existing_flags.py +0 -0
  67. {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_exit_codes.py +0 -0
  68. {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_high_fidelity.py +0 -0
  69. {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_meta.py +0 -0
  70. {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_no_arxiv_lookup.py +0 -0
  71. {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_ocr_save_images.py +0 -0
  72. {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_quiet_verbose.py +0 -0
  73. {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_strict.py +0 -0
  74. {any2md-1.0.5 → any2md-1.0.6}/tests/conftest.py +0 -0
  75. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/__init__.py +0 -0
  76. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/docs/ligatures_and_softhyphens.txt +0 -0
  77. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/docs/multi_column.pdf +0 -0
  78. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/docs/table_heavy.docx +0 -0
  79. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/docs/web_page.html +0 -0
  80. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/make_fixtures.py +0 -0
  81. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/ligatures_and_softhyphens.md +0 -0
  82. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/multi_column.docling.md +0 -0
  83. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/multi_column.fallback.md +0 -0
  84. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/table_heavy.docling.md +0 -0
  85. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/table_heavy.fallback.md +0 -0
  86. {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/web_page.md +0 -0
  87. {any2md-1.0.5 → any2md-1.0.6}/tests/integration/__init__.py +0 -0
  88. {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_docx_backend_override.py +0 -0
  89. {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_docx_docling.py +0 -0
  90. {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_docx_fallback_on_warn.py +0 -0
  91. {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_docx_mammoth_fallback.py +0 -0
  92. {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_pdf_backend_override.py +0 -0
  93. {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_pdf_docling.py +0 -0
  94. {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_pdf_pymupdf_fallback.py +0 -0
  95. {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_save_images.py +0 -0
  96. {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_snapshots.py +0 -0
  97. {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_txt.py +0 -0
  98. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/__init__.py +0 -0
  99. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/__init__.py +0 -0
  100. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_footnote_markers.py +0 -0
  101. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_html_entities.py +0 -0
  102. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_html_entities_double_encoded.py +0 -0
  103. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_ligatures.py +0 -0
  104. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_nfc.py +0 -0
  105. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_quotes_dashes.py +0 -0
  106. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_soft_hyphens.py +0 -0
  107. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_validate.py +0 -0
  108. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_whitespace.py +0 -0
  109. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_runner.py +0 -0
  110. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_strip_orphan_punctuation.py +0 -0
  111. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_structured_body_cleanup_stages.py +0 -0
  112. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_structured_compact_tables.py +0 -0
  113. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_structured_heading_hierarchy.py +0 -0
  114. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_structured_lift_figures.py +0 -0
  115. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_structured_normalize_cites.py +0 -0
  116. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_dedupe_paragraphs.py +0 -0
  117. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_dedupe_toc.py +0 -0
  118. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_dedupe_toc_table.py +0 -0
  119. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_dehyphenate.py +0 -0
  120. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_repair_line_wraps.py +0 -0
  121. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_restore_lists_code.py +0 -0
  122. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_cover_artifacts.py +0 -0
  123. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_headers_footers.py +0 -0
  124. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_repeated_byline.py +0 -0
  125. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_web_fragments.py +0 -0
  126. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/scripts/__init__.py +0 -0
  127. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/scripts/test_audit_outputs.py +0 -0
  128. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_auto_id.py +0 -0
  129. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_config.py +0 -0
  130. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_content_hash.py +0 -0
  131. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_docling_helper.py +0 -0
  132. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_frontmatter_compose.py +0 -0
  133. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_frontmatter_helpers.py +0 -0
  134. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_meta_overrides.py +0 -0
  135. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_pdf_complexity.py +0 -0
  136. {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_validators.py +0 -0
@@ -4,6 +4,89 @@ All notable changes to this project will be documented in this file.
4
4
 
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
6
6
 
7
+ ## [1.0.6] — 2026-04-27
8
+
9
+ Security hardening release. Closes 8 actionable findings from the
10
+ 2026-04-27 multi-tool audit (semgrep + trivy + gitleaks + bandit +
11
+ pip-audit + manual review by the security-engineer agent + Opus 4.7).
12
+ No public API changes; behavior unchanged for legitimate inputs.
13
+
14
+ ### Security
15
+
16
+ - **F1 (HIGH)** SSRF: replaced `trafilatura.fetch_url` and the bare
17
+ `urllib.request.urlopen` paths with a new `any2md/_http.py` module
18
+ that does manual redirect walking with per-hop revalidation. Defends
19
+ against DNS-rebinding and redirect-based bypass of the IP-class
20
+ check. New `validate_url` rejects non-http(s) schemes and
21
+ private/loopback/link-local/reserved targets. 20 MB body cap, 3-hop
22
+ redirect cap, redirect-loop detection. Applied to `convert_url`,
23
+ `_http_last_modified`, and `arxiv_lookup`.
24
+ - **F5 (MED)** `--meta` integrity: `content_hash`, `extracted_via`,
25
+ `source_file`, `lane`, `token_estimate`, `recommended_chunk_level`
26
+ are now **reserved** and silently filtered from `--meta` /
27
+ `--meta-file` / `.any2md.toml` overrides with a stderr `WARN` per
28
+ drop. Defense-in-depth filter also runs inside `frontmatter.compose`.
29
+ - **F3 (MED)** TOML auto-discovery now stops at the first ancestor
30
+ containing `.git` / `pyproject.toml` / `setup.py` / `setup.cfg` /
31
+ `package.json` / `.any2md.toml.boundary`. New `--no-config` flag
32
+ fully disables discovery.
33
+ - **F4 (MED)** DOCX zip-bomb: `_safe_zip_open` enforces a 1 MB cap on
34
+ the declared uncompressed size of `core.xml` and `app.xml` before
35
+ parsing. Oversized members raise `ValueError` → clean per-file FAIL.
36
+ - **F-CVE (MED)** Dependency floors bumped: `lxml>=6.1.0,<7`
37
+ (CVE-2026-41066), `pillow>=12.2.0,<13` (CVE-2026-25990,
38
+ CVE-2026-40192), `urllib3>=2.6.3,<3` (CVE-2026-21441). All ranges
39
+ acquire upper bounds (compat envelope). `requirements.txt` becomes
40
+ a development/CI lockfile with exact pins.
41
+ - **F6 (LOW)** Atomic writes: new `any2md.utils.atomic_write_text`
42
+ writes via `tempfile.mkstemp` + `os.replace`. Refuses to overwrite
43
+ a pre-existing symlink at the output path. Wired through
44
+ `pdf/docx/html/txt` converters; PDF image extraction adds a symlink
45
+ check before `pil_image.save`.
46
+ - **F2 (LOW)** Control-char sanitizer strips C0/C1 controls (preserves
47
+ `\t`, `\n`) from captured Docling `msword_backend` warnings before
48
+ forwarding them to `collected_warnings()` and stderr. Defends
49
+ terminals from ANSI-escape spoofing.
50
+ - **F11 (LOW)** PDF image-dir name now goes through `safe_dir_name`
51
+ (alphanumeric/`_`/`-` only). Closes a `Path.stem == ".."` corner
52
+ case that co-mingled images with markdown outputs.
53
+ - **XXE defense-in-depth**: migrated `xml.etree.ElementTree` →
54
+ `defusedxml.ElementTree` in `docx.py` and `heuristics.py`. Stdlib
55
+ was already safe in Python 3.7+; this silences bandit B405/B314
56
+ permanently and protects against future stdlib regressions.
57
+
58
+ ### Added
59
+
60
+ - New module `any2md/_http.py` (`validate_url`, `safe_fetch`).
61
+ - New CLI flag `--no-config`.
62
+ - New helpers `any2md.utils.atomic_write_text` and `safe_dir_name`.
63
+ - New runtime dependency: `defusedxml>=0.7.1,<1`.
64
+
65
+ ### Tests
66
+
67
+ - 7 new unit-test files (40 new tests): `test_http_safe.py`,
68
+ `test_meta_reserved_keys.py`, `test_config_walkup.py`,
69
+ `test_zipbomb_guard.py`, `test_atomic_write.py`,
70
+ `test_safe_dir_name.py`, `test_log_sanitizer.py`.
71
+ - Existing 290 tests continue to pass.
72
+ - Bandit audit went from **9 findings** (3 medium, 6 low) to **1 low**
73
+ (legitimate `try/except/pass` in arxiv `_warn`).
74
+ - 5-DOCX regression run reproduces v1.0.5 word counts exactly
75
+ (978/6726/15566/7811/1477).
76
+
77
+ ### Audit summary
78
+
79
+ | ID | Title | Severity |
80
+ |----|-------|----------|
81
+ | F1 | SSRF: DNS rebind + redirect bypass | High |
82
+ | F5 | `--meta` clobbers reserved frontmatter | Medium |
83
+ | F3 | TOML discovery walks above project root | Medium |
84
+ | F4 | DOCX zip-bomb amplification | Medium |
85
+ | F-CVE | Transitive deps with known CVEs | Medium |
86
+ | F6 | Symlink-following on output writes | Low |
87
+ | F2 | Control-char passthrough in Docling logs | Low |
88
+ | F11 | PDF stem `..` corner case | Low |
89
+
7
90
  ## [1.0.5] — 2026-04-27
8
91
 
9
92
  Patch release. Recovers DOCX content that Docling's msword backend
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: any2md
3
- Version: 1.0.5
3
+ Version: 1.0.6
4
4
  Summary: Convert PDF, DOCX, HTML, and TXT files — or web pages by URL — to clean, LLM-optimized Markdown with YAML frontmatter.
5
5
  Project-URL: Homepage, https://github.com/rocklambros/any2md
6
6
  Project-URL: Issues, https://github.com/rocklambros/any2md/issues
@@ -12,13 +12,16 @@ Classifier: Operating System :: OS Independent
12
12
  Classifier: Programming Language :: Python :: 3
13
13
  Classifier: Topic :: Text Processing :: Markup :: Markdown
14
14
  Requires-Python: >=3.10
15
- Requires-Dist: beautifulsoup4>=4.12.0
16
- Requires-Dist: lxml>=5.0.0
17
- Requires-Dist: mammoth>=1.6.0
18
- Requires-Dist: markdownify>=0.13.0
19
- Requires-Dist: pymupdf4llm>=0.0.17
20
- Requires-Dist: pymupdf>=1.24.0
21
- Requires-Dist: trafilatura>=1.12.0
15
+ Requires-Dist: beautifulsoup4<5,>=4.12.0
16
+ Requires-Dist: defusedxml<1,>=0.7.1
17
+ Requires-Dist: lxml<7,>=6.1.0
18
+ Requires-Dist: mammoth<2,>=1.6.0
19
+ Requires-Dist: markdownify<2,>=0.13.0
20
+ Requires-Dist: pillow<13,>=12.2.0
21
+ Requires-Dist: pymupdf4llm<1,>=0.0.17
22
+ Requires-Dist: pymupdf<2,>=1.24.0
23
+ Requires-Dist: trafilatura<3,>=1.12.0
24
+ Requires-Dist: urllib3<3,>=2.6.3
22
25
  Provides-Extra: dev
23
26
  Requires-Dist: pytest-snapshot>=0.9.0; extra == 'dev'
24
27
  Requires-Dist: pytest>=8.0.0; extra == 'dev'
@@ -1,3 +1,3 @@
1
1
  """Convert PDF, DOCX, HTML, and TXT files to LLM-optimized Markdown."""
2
2
 
3
- __version__ = "1.0.5"
3
+ __version__ = "1.0.6"
@@ -0,0 +1,106 @@
1
+ """SSRF-safe HTTP fetching utilities for any2md.
2
+
3
+ Centralizes URL scheme + IP validation and provides a fetcher that
4
+ walks redirects manually and revalidates the host on each hop. This
5
+ defends against:
6
+ - DNS rebinding (host -> public IP for validator, private IP for fetch)
7
+ - Redirect-based SSRF (public landing page -> 302 to 169.254.169.254)
8
+ - Non-http(s) schemes (file://, gopher://, etc.)
9
+ - Decompression / huge-response DoS (response-size cap)
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import ipaddress
15
+ import socket
16
+ import urllib.error
17
+ import urllib.parse
18
+ import urllib.request
19
+
20
+ _MAX_REDIRECT_HOPS = 3
21
+ _FETCH_TIMEOUT = 15 # seconds
22
+ _MAX_RESPONSE_BYTES = 20 * 1024 * 1024 # 20 MB cap on body read
23
+ _USER_AGENT = "any2md/1.0.6"
24
+
25
+
26
+ def validate_url(url: str) -> str | None:
27
+ """Validate URL scheme + host. Returns an error message or None.
28
+
29
+ One-shot DNS lookup; rebind protection requires re-checking on
30
+ every redirect hop (handled by ``safe_fetch``).
31
+ """
32
+ parsed = urllib.parse.urlparse(url)
33
+ if parsed.scheme not in ("http", "https"):
34
+ return f"unsupported scheme: {parsed.scheme!r}"
35
+ if not parsed.hostname:
36
+ return f"no hostname in URL: {url}"
37
+ try:
38
+ infos = socket.getaddrinfo(parsed.hostname, None, type=socket.SOCK_STREAM)
39
+ except socket.gaierror:
40
+ return f"cannot resolve host: {parsed.hostname}"
41
+ for *_, sockaddr in infos:
42
+ try:
43
+ addr = ipaddress.ip_address(sockaddr[0])
44
+ except ValueError:
45
+ continue
46
+ if (
47
+ addr.is_private
48
+ or addr.is_reserved
49
+ or addr.is_loopback
50
+ or addr.is_link_local
51
+ ):
52
+ return f"URL resolves to disallowed address: {addr}"
53
+ return None
54
+
55
+
56
+ class _NoFollowRedirect(urllib.request.HTTPRedirectHandler):
57
+ """Disable urllib's automatic redirect following."""
58
+
59
+ def redirect_request(self, *args, **kwargs): # noqa: ARG002
60
+ return None
61
+
62
+
63
+ def safe_fetch(
64
+ url: str,
65
+ *,
66
+ method: str = "GET",
67
+ max_hops: int = _MAX_REDIRECT_HOPS,
68
+ ) -> tuple[bytes | None, dict | None, str | None]:
69
+ """Fetch ``url`` with manual redirect walking + per-hop revalidation.
70
+
71
+ Returns ``(body, headers, None)`` on 2xx, or ``(None, None, error)``
72
+ on rejection / failure. ``headers`` is the response headers as a
73
+ plain ``dict``.
74
+ """
75
+ visited: list[str] = []
76
+ current = url
77
+ opener = urllib.request.build_opener(_NoFollowRedirect())
78
+ for hop in range(max_hops + 1):
79
+ if current in visited:
80
+ return None, None, "redirect loop"
81
+ visited.append(current)
82
+ err = validate_url(current)
83
+ if err:
84
+ return None, None, err
85
+ req = urllib.request.Request(
86
+ current, method=method, headers={"User-Agent": _USER_AGENT}
87
+ )
88
+ try:
89
+ resp = opener.open(req, timeout=_FETCH_TIMEOUT)
90
+ except urllib.error.HTTPError as e:
91
+ if e.code in (301, 302, 303, 307, 308):
92
+ if hop >= max_hops:
93
+ return None, None, f"too many redirects (>{max_hops})"
94
+ location = e.headers.get("Location") if e.headers else None
95
+ if not location:
96
+ return None, None, f"HTTP {e.code} without Location"
97
+ current = urllib.parse.urljoin(current, location)
98
+ continue
99
+ return None, None, f"HTTP {e.code}"
100
+ except (urllib.error.URLError, OSError, TimeoutError) as e:
101
+ return None, None, f"fetch error: {e}"
102
+ body = resp.read(_MAX_RESPONSE_BYTES + 1)
103
+ if len(body) > _MAX_RESPONSE_BYTES:
104
+ return None, None, f"response exceeds {_MAX_RESPONSE_BYTES} bytes"
105
+ return body, dict(resp.headers), None
106
+ return None, None, f"too many redirects (>{max_hops})"
@@ -20,6 +20,7 @@ from any2md.converters import (
20
20
  set_output_mode,
21
21
  )
22
22
  from any2md.converters.html import convert_url
23
+ from any2md.frontmatter import filter_reserved_overrides
23
24
  from any2md.pipeline import PipelineOptions
24
25
  from any2md.utils import sanitize_filename, url_to_filename
25
26
 
@@ -169,6 +170,11 @@ def main():
169
170
  action="store_true",
170
171
  help="Promote pipeline validation warnings to errors (exit 3).",
171
172
  )
173
+ parser.add_argument(
174
+ "--no-config",
175
+ action="store_true",
176
+ help="Skip .any2md.toml auto-discovery (overrides default walk-up).",
177
+ )
172
178
  parser.add_argument(
173
179
  "--docx-fallback-on-warn",
174
180
  action=argparse.BooleanOptionalAction,
@@ -222,7 +228,7 @@ def main():
222
228
  overrides: dict[str, Any] = {}
223
229
  auto_id_prefix, auto_id_type_code = "LOCAL", "DOC"
224
230
 
225
- discovered = discover_config()
231
+ discovered = None if args.no_config else discover_config()
226
232
  if discovered is not None:
227
233
  cfg = load_toml(discovered)
228
234
  overrides = _deep_merge_overrides(overrides, extract_meta_overrides(cfg))
@@ -249,6 +255,9 @@ def main():
249
255
  print(f"Error: {e}", file=sys.stderr)
250
256
  sys.exit(1)
251
257
  overrides = _deep_merge_overrides(overrides, cli_meta)
258
+ overrides = filter_reserved_overrides(
259
+ overrides, source_label="--meta/--meta-file/.any2md.toml"
260
+ )
252
261
 
253
262
  if (
254
263
  args.high_fidelity
@@ -42,17 +42,31 @@ def load_toml(path: Path) -> dict[str, Any]:
42
42
  return {}
43
43
 
44
44
 
45
+ _PROJECT_BOUNDARY_MARKERS = (
46
+ ".git",
47
+ "pyproject.toml",
48
+ "setup.py",
49
+ "setup.cfg",
50
+ "package.json",
51
+ ".any2md.toml.boundary",
52
+ )
53
+
54
+
45
55
  def discover_config(start: Path | None = None) -> Path | None:
46
56
  """Walk up from ``start`` (default: cwd) looking for ``.any2md.toml``.
47
57
 
48
- Returns the first match found or ``None`` if no config exists in
49
- any ancestor directory.
58
+ Stops at the first ancestor containing a project boundary marker
59
+ (``.git``, ``pyproject.toml``, ``setup.py``, ``setup.cfg``,
60
+ ``package.json``, or ``.any2md.toml.boundary``) so config in
61
+ unrelated parent directories cannot leak into a project's run.
50
62
  """
51
63
  cur = (start or Path.cwd()).resolve()
52
64
  while True:
53
65
  candidate = cur / ".any2md.toml"
54
66
  if candidate.is_file():
55
67
  return candidate
68
+ if any((cur / m).exists() for m in _PROJECT_BOUNDARY_MARKERS):
69
+ return None
56
70
  if cur.parent == cur:
57
71
  return None
58
72
  cur = cur.parent
@@ -10,9 +10,12 @@ produces the markdown body.
10
10
  from __future__ import annotations
11
11
 
12
12
  import logging
13
+ import re
13
14
  import sys
14
- import xml.etree.ElementTree as ET
15
15
  import zipfile
16
+
17
+ from defusedxml.ElementTree import ParseError as _XmlParseError
18
+ from defusedxml.ElementTree import parse as _xml_parse
16
19
  from datetime import date
17
20
  from pathlib import Path
18
21
 
@@ -25,11 +28,44 @@ from any2md.converters import add_warnings, is_quiet
25
28
  from any2md.frontmatter import SourceMeta, compose
26
29
  from any2md.heuristics import filter_organization
27
30
  from any2md.pipeline import PipelineOptions
28
- from any2md.utils import sanitize_filename
31
+ from any2md.utils import atomic_write_text, sanitize_filename
29
32
 
30
33
 
31
34
  _DOCLING_MSWORD_LOGGER = "docling.backend.msword_backend"
32
35
 
36
+ _LOG_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")
37
+
38
+
39
+ def _sanitize_log_text(s: str) -> str:
40
+ """Strip ASCII C0/C1 control chars from a log message before re-emission.
41
+
42
+ Defends terminals from ANSI-escape spoofing planted in malicious DOCX
43
+ content that surfaces as a Docling msword_backend warning. ``\\t``
44
+ and ``\\n`` are preserved (not in the strip class).
45
+ """
46
+ return _LOG_CONTROL_CHARS_RE.sub("", s)
47
+
48
+
49
+ _MAX_DOCX_METADATA_SIZE = 1 * 1024 * 1024 # 1 MB; core.xml/app.xml are typically <10 KB
50
+
51
+
52
+ def _safe_zip_open(
53
+ z: zipfile.ZipFile, name: str, max_size: int = _MAX_DOCX_METADATA_SIZE
54
+ ):
55
+ """Open a fixed-name DOCX member after a declared-size sanity check.
56
+
57
+ Defends against zip-bomb amplification: a malicious DOCX can declare
58
+ an uncompressed_size of 10 GB for ``core.xml`` and bomb the XML
59
+ parser. Raises ``ValueError`` (caught by ``convert_docx``'s existing
60
+ handler -> file fails cleanly).
61
+ """
62
+ info = z.getinfo(name)
63
+ if info.file_size > max_size:
64
+ raise ValueError(
65
+ f"{name} declared size {info.file_size} exceeds {max_size}-byte limit"
66
+ )
67
+ return z.open(name)
68
+
33
69
 
34
70
  class _DoclingMswordWarningCapture:
35
71
  """Buffer WARNING+ records from Docling's DOCX backend logger.
@@ -89,8 +125,8 @@ def _read_docx_metadata(docx_path: Path) -> dict[str, object]:
89
125
  try:
90
126
  with zipfile.ZipFile(docx_path) as z:
91
127
  try:
92
- with z.open("docProps/core.xml") as f:
93
- root = ET.parse(f).getroot()
128
+ with _safe_zip_open(z, "docProps/core.xml") as f:
129
+ root = _xml_parse(f).getroot()
94
130
  title = root.findtext("dc:title", namespaces=_NS_CORE)
95
131
  if title:
96
132
  out["title_hint"] = title.strip()
@@ -105,8 +141,8 @@ def _read_docx_metadata(docx_path: Path) -> dict[str, object]:
105
141
  except KeyError:
106
142
  pass
107
143
  try:
108
- with z.open("docProps/app.xml") as f:
109
- root = ET.parse(f).getroot()
144
+ with _safe_zip_open(z, "docProps/app.xml") as f:
145
+ root = _xml_parse(f).getroot()
110
146
  company = root.findtext("ext:Company", namespaces=_NS_APP)
111
147
  application = root.findtext("ext:Application", namespaces=_NS_APP)
112
148
  # v1.0.2: Company takes priority for `organization` (real
@@ -126,7 +162,7 @@ def _read_docx_metadata(docx_path: Path) -> dict[str, object]:
126
162
  out["produced_by"] = app_result.produced_by
127
163
  except KeyError:
128
164
  pass
129
- except (zipfile.BadZipFile, ET.ParseError):
165
+ except (zipfile.BadZipFile, _XmlParseError):
130
166
  pass
131
167
  return out
132
168
 
@@ -233,7 +269,12 @@ def convert_docx(
233
269
  md_text, _ = _extract_via_mammoth(docx_path, options)
234
270
  extracted_via = "docling→mammoth (warning fallback)"
235
271
  lane = "text"
236
- add_warnings([f"docling.msword_backend: {m}" for m in docling_warnings])
272
+ add_warnings(
273
+ [
274
+ f"docling.msword_backend: {_sanitize_log_text(m)}"
275
+ for m in docling_warnings
276
+ ]
277
+ )
237
278
 
238
279
  md_text, warnings = pipeline.run(md_text, lane, options)
239
280
  add_warnings(warnings)
@@ -260,7 +301,7 @@ def convert_docx(
260
301
  full = compose(md_text, meta, options, overrides=options.frontmatter_overrides)
261
302
 
262
303
  output_dir.mkdir(parents=True, exist_ok=True)
263
- out_path.write_text(full, encoding="utf-8", newline="\n")
304
+ atomic_write_text(out_path, full)
264
305
  wc = meta.word_count or 0
265
306
  suffix = f", {len(warnings)} warning(s)" if warnings else ""
266
307
  if not is_quiet():
@@ -2,12 +2,9 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- import ipaddress
6
- import socket
7
5
  import sys
8
- import urllib.parse
9
- import urllib.request
10
6
  from datetime import date
7
+ from email.utils import parsedate_to_datetime
11
8
  from pathlib import Path
12
9
 
13
10
  import markdownify
@@ -15,87 +12,51 @@ import trafilatura
15
12
  from bs4 import BeautifulSoup
16
13
 
17
14
  from any2md import pipeline
15
+ from any2md._http import safe_fetch
18
16
  from any2md.converters import add_warnings, is_quiet
19
17
  from any2md.frontmatter import SourceMeta, compose
20
18
  from any2md.pipeline import PipelineOptions
21
19
  from any2md.utils import (
20
+ atomic_write_text,
22
21
  read_text_with_fallback,
23
22
  sanitize_filename,
24
23
  url_to_filename,
25
24
  )
26
25
 
27
- _MAX_FILE_SIZE = 100 * 1024 * 1024
28
-
29
26
 
30
- def _validate_url_host(url: str) -> str | None:
31
- """Validate that a URL does not point to a private/reserved IP.
32
-
33
- Returns an error message if the host is disallowed, or None if safe.
34
- """
35
- parsed = urllib.parse.urlparse(url)
36
- hostname = parsed.hostname
37
- if not hostname:
38
- return f"No hostname in URL: {url}"
39
- try:
40
- infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
41
- except socket.gaierror:
42
- return f"Cannot resolve hostname: {hostname}"
43
- for _family, _type, _proto, _canon, sockaddr in infos:
44
- ip_str = sockaddr[0]
45
- try:
46
- addr = ipaddress.ip_address(ip_str)
47
- except ValueError:
48
- continue
49
- if (
50
- addr.is_private
51
- or addr.is_reserved
52
- or addr.is_loopback
53
- or addr.is_link_local
54
- ):
55
- return f"URL resolves to disallowed address: {ip_str}"
56
- return None
27
+ _MAX_FILE_SIZE = 100 * 1024 * 1024
57
28
 
58
29
 
59
30
  def fetch_url(url: str) -> tuple[str | None, str | None]:
60
- """Fetch HTML content from a URL.
61
-
62
- Only http and https schemes are accepted. SSRF protection blocks
63
- requests to private/reserved/loopback addresses.
31
+ """Fetch HTML from a URL using the SSRF-safe fetcher.
64
32
 
65
- Returns (html_string, None) on success or (None, error_message) on failure.
33
+ Returns (html_string, None) on success or (None, error_message)
34
+ on failure. Per-hop host revalidation defends against DNS rebind
35
+ via redirects.
66
36
  """
67
- parsed = urllib.parse.urlparse(url)
68
- if parsed.scheme not in ("http", "https"):
69
- return None, f"Unsupported URL scheme: {parsed.scheme!r}"
70
- err = _validate_url_host(url)
37
+ body, _headers, err = safe_fetch(url)
71
38
  if err:
72
39
  return None, err
73
- try:
74
- html = trafilatura.fetch_url(url)
75
- if html is None:
76
- return None, f"Failed to fetch URL: {url}"
77
- return html, None
78
- except Exception as e: # noqa: BLE001
79
- return None, f"Error fetching URL: {e}"
40
+ if body is None:
41
+ return None, f"empty body for {url}"
42
+ return body.decode("utf-8", errors="replace"), None
80
43
 
81
44
 
82
45
  def _http_last_modified(url: str) -> str | None:
83
- """Single HEAD request for Last-Modified. Best-effort."""
84
- try:
85
- req = urllib.request.Request(
86
- url,
87
- method="HEAD",
88
- headers={"User-Agent": "any2md/1.0"},
89
- )
90
- with urllib.request.urlopen(req, timeout=10) as resp:
91
- lm = resp.headers.get("Last-Modified")
92
- if lm:
93
- from email.utils import parsedate_to_datetime
46
+ """Single HEAD request for Last-Modified. Best-effort.
94
47
 
95
- return parsedate_to_datetime(lm).date().isoformat()
96
- except Exception: # noqa: BLE001
97
- pass
98
- return None
48
+ SSRF-safe: same scheme + IP validation as ``fetch_url``.
49
+ """
50
+ _body, headers, err = safe_fetch(url, method="HEAD")
51
+ if err or not headers:
52
+ return None
53
+ lm = headers.get("Last-Modified")
54
+ if not lm:
55
+ return None
56
+ try:
57
+ return parsedate_to_datetime(lm).date().isoformat()
58
+ except (TypeError, ValueError):
59
+ return None
99
60
 
100
61
 
101
62
  def _bs4_preclean(html: str) -> str:
@@ -227,7 +188,7 @@ def convert_html(
227
188
  full = compose(md_text, meta, options, overrides=options.frontmatter_overrides)
228
189
 
229
190
  output_dir.mkdir(parents=True, exist_ok=True)
230
- out_path.write_text(full, encoding="utf-8", newline="\n")
191
+ atomic_write_text(out_path, full)
231
192
  wc = meta.word_count or 0
232
193
  suffix = f", {len(warnings)} warning(s)" if warnings else ""
233
194
  if not is_quiet():
@@ -22,7 +22,7 @@ from any2md.converters import add_warnings, is_quiet
22
22
  from any2md.frontmatter import SourceMeta, compose
23
23
  from any2md.heuristics import filter_organization
24
24
  from any2md.pipeline import PipelineOptions
25
- from any2md.utils import sanitize_filename
25
+ from any2md.utils import atomic_write_text, safe_dir_name, sanitize_filename
26
26
 
27
27
 
28
28
  def _parse_pdf_authors(raw: str | None) -> list[str]:
@@ -140,7 +140,7 @@ def _extract_via_docling(
140
140
  if options.save_images:
141
141
  pictures = getattr(result.document, "pictures", None) or []
142
142
  if pictures:
143
- images_dir = output_dir / "images" / pdf_path.stem
143
+ images_dir = output_dir / "images" / safe_dir_name(pdf_path.stem)
144
144
  images_dir.mkdir(parents=True, exist_ok=True)
145
145
  for i, picture in enumerate(pictures):
146
146
  try:
@@ -148,6 +148,12 @@ def _extract_via_docling(
148
148
  if pil_image is None:
149
149
  continue
150
150
  img_path = images_dir / f"img{i + 1}.png"
151
+ if img_path.exists() and img_path.is_symlink():
152
+ print(
153
+ f" WARN: skipping image {i + 1}: target is a symlink.",
154
+ file=sys.stderr,
155
+ )
156
+ continue
151
157
  pil_image.save(str(img_path))
152
158
  except Exception as e: # noqa: BLE001
153
159
  print(
@@ -265,7 +271,7 @@ def convert_pdf(
265
271
  full = compose(md_text, meta, options, overrides=options.frontmatter_overrides)
266
272
 
267
273
  output_dir.mkdir(parents=True, exist_ok=True)
268
- out_path.write_text(full, encoding="utf-8", newline="\n")
274
+ atomic_write_text(out_path, full)
269
275
  suffix = f", {len(warnings)} warning(s)" if warnings else ""
270
276
  if not is_quiet():
271
277
  print(f" OK: {out_name} ({page_count} pages, via {extracted_via}{suffix})")
@@ -11,7 +11,7 @@ from any2md import pipeline
11
11
  from any2md.converters import add_warnings, is_quiet
12
12
  from any2md.frontmatter import SourceMeta, compose
13
13
  from any2md.pipeline import PipelineOptions
14
- from any2md.utils import read_text_with_fallback, sanitize_filename
14
+ from any2md.utils import atomic_write_text, read_text_with_fallback, sanitize_filename
15
15
 
16
16
  # Existing structurize() heuristic stays in this file from v0.7 — keep it.
17
17
  _SEPARATOR_RE = re.compile(r"^([=\-*_~])\1{2,}\s*$")
@@ -199,7 +199,7 @@ def convert_txt(
199
199
  full = compose(md_text, meta, options, overrides=options.frontmatter_overrides)
200
200
 
201
201
  output_dir.mkdir(parents=True, exist_ok=True)
202
- out_path.write_text(full, encoding="utf-8", newline="\n")
202
+ atomic_write_text(out_path, full)
203
203
  word_count = meta.word_count or 0
204
204
  suffix = f", {len(warnings)} warning(s)" if warnings else ""
205
205
  if not is_quiet():
@@ -10,7 +10,9 @@ from __future__ import annotations
10
10
  import hashlib
11
11
  import math
12
12
  import re
13
+ import sys
13
14
  import unicodedata
15
+ from collections.abc import Mapping
14
16
  from dataclasses import dataclass
15
17
  from datetime import date as _date_cls
16
18
  from typing import Any, Literal
@@ -19,6 +21,44 @@ from any2md import heuristics
19
21
  from any2md.pipeline import Lane, PipelineOptions
20
22
 
21
23
 
24
+ _RESERVED_OVERRIDE_KEYS = frozenset(
25
+ {
26
+ "content_hash",
27
+ "token_estimate",
28
+ "recommended_chunk_level",
29
+ "extracted_via",
30
+ "source_file",
31
+ "lane",
32
+ }
33
+ )
34
+
35
+
36
+ def filter_reserved_overrides(
37
+ overrides: Mapping[str, Any] | None,
38
+ *,
39
+ source_label: str = "overrides",
40
+ ) -> dict[str, Any] | None:
41
+ """Drop top-level reserved keys; emit stderr WARN per drop.
42
+
43
+ Reserved keys are derived from the source document and must not be
44
+ overridable via ``--meta``, ``--meta-file``, or ``.any2md.toml``.
45
+ Nested dicts under non-reserved keys are returned untouched.
46
+ """
47
+ if overrides is None:
48
+ return None
49
+ out: dict[str, Any] = {}
50
+ for k, v in overrides.items():
51
+ if k in _RESERVED_OVERRIDE_KEYS:
52
+ print(
53
+ f" WARN: ignoring {source_label} key {k!r} "
54
+ f"— reserved (derived from source).",
55
+ file=sys.stderr,
56
+ )
57
+ continue
58
+ out[k] = v
59
+ return out
60
+
61
+
22
62
  def compute_content_hash(body: str) -> str:
23
63
  """SHA-256 of NFC-normalized, LF-line-ended body. SSRM §5.1.
24
64
 
@@ -329,6 +369,7 @@ def compose(
329
369
  4. Emit YAML frontmatter in spec §3.2-3.4 order and concatenate the body.
330
370
  """
331
371
  body = _normalize_body(body)
372
+ overrides = filter_reserved_overrides(overrides, source_label="compose()")
332
373
  fields = _build_fields(body, meta, options)
333
374
  if overrides:
334
375
  fields = _deep_merge(fields, overrides)