any2md 1.0.5__tar.gz → 1.0.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {any2md-1.0.5 → any2md-1.0.6}/CHANGELOG.md +83 -0
- {any2md-1.0.5 → any2md-1.0.6}/PKG-INFO +11 -8
- {any2md-1.0.5 → any2md-1.0.6}/any2md/__init__.py +1 -1
- any2md-1.0.6/any2md/_http.py +106 -0
- {any2md-1.0.5 → any2md-1.0.6}/any2md/cli.py +10 -1
- {any2md-1.0.5 → any2md-1.0.6}/any2md/config.py +16 -2
- {any2md-1.0.5 → any2md-1.0.6}/any2md/converters/docx.py +50 -9
- {any2md-1.0.5 → any2md-1.0.6}/any2md/converters/html.py +26 -65
- {any2md-1.0.5 → any2md-1.0.6}/any2md/converters/pdf.py +9 -3
- {any2md-1.0.5 → any2md-1.0.6}/any2md/converters/txt.py +2 -2
- {any2md-1.0.5 → any2md-1.0.6}/any2md/frontmatter.py +41 -0
- {any2md-1.0.5 → any2md-1.0.6}/any2md/heuristics.py +14 -22
- {any2md-1.0.5 → any2md-1.0.6}/any2md/utils.py +44 -0
- any2md-1.0.6/docs/superpowers/plans/2026-04-27-v1.0.6-security-hardening.md +1867 -0
- any2md-1.0.6/docs/superpowers/specs/2026-04-27-v1.0.6-security-hardening-design.md +477 -0
- {any2md-1.0.5 → any2md-1.0.6}/pyproject.toml +10 -7
- any2md-1.0.6/requirements.txt +10 -0
- any2md-1.0.6/tests/integration/test_html_trafilatura.py +105 -0
- any2md-1.0.6/tests/unit/test_atomic_write.py +52 -0
- any2md-1.0.6/tests/unit/test_config_walkup.py +53 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_heuristics.py +21 -27
- any2md-1.0.6/tests/unit/test_http_safe.py +202 -0
- any2md-1.0.6/tests/unit/test_log_sanitizer.py +27 -0
- any2md-1.0.6/tests/unit/test_meta_reserved_keys.py +37 -0
- any2md-1.0.6/tests/unit/test_safe_dir_name.py +25 -0
- any2md-1.0.6/tests/unit/test_zipbomb_guard.py +47 -0
- any2md-1.0.5/requirements.txt +0 -7
- any2md-1.0.5/tests/integration/test_html_trafilatura.py +0 -29
- {any2md-1.0.5 → any2md-1.0.6}/.gitignore +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/CODE_OF_CONDUCT.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/CONTRIBUTING.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/LICENSE +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/README.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/SECURITY.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/any2md/__main__.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/any2md/_docling.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/any2md/converters/__init__.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/any2md/pipeline/__init__.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/any2md/pipeline/cleanup.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/any2md/pipeline/structured.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/any2md/pipeline/text.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/any2md/validators.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/assets/logo-original.jpeg +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/assets/logo.png +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/assets/social-preview.jpg +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/architecture.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/cli-reference.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/output-format.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase1-foundation.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase2-docling-backend.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase3-figures-ocr-text-lane.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1-phase4-polish-config-docs.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-any2md-v1.0.2-implementation.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/plans/2026-04-26-issue-17-toc-followups.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-any2md-v1-design.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-any2md-v1.0.2-design.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-issue-17-toc-followups-design.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/superpowers/specs/2026-04-26-v1.0.3-empty-title-orphan-punct-design.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/troubleshooting.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/docs/upgrading-from-0.7.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/scripts/audit-outputs.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/scripts/validate-phase1.sh +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/__init__.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/cli/__init__.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_backend.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_existing_flags.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_exit_codes.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_high_fidelity.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_meta.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_no_arxiv_lookup.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_ocr_save_images.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_quiet_verbose.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/cli/test_cli_strict.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/conftest.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/__init__.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/docs/ligatures_and_softhyphens.txt +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/docs/multi_column.pdf +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/docs/table_heavy.docx +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/docs/web_page.html +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/make_fixtures.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/ligatures_and_softhyphens.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/multi_column.docling.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/multi_column.fallback.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/table_heavy.docling.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/table_heavy.fallback.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/fixtures/snapshots/web_page.md +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/integration/__init__.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_docx_backend_override.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_docx_docling.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_docx_fallback_on_warn.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_docx_mammoth_fallback.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_pdf_backend_override.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_pdf_docling.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_pdf_pymupdf_fallback.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_save_images.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_snapshots.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/integration/test_txt.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/__init__.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/__init__.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_footnote_markers.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_html_entities.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_html_entities_double_encoded.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_ligatures.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_nfc.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_quotes_dashes.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_soft_hyphens.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_validate.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_cleanup_whitespace.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_runner.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_strip_orphan_punctuation.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_structured_body_cleanup_stages.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_structured_compact_tables.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_structured_heading_hierarchy.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_structured_lift_figures.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_structured_normalize_cites.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_dedupe_paragraphs.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_dedupe_toc.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_dedupe_toc_table.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_dehyphenate.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_repair_line_wraps.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_restore_lists_code.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_cover_artifacts.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_headers_footers.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_repeated_byline.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/pipeline/test_text_strip_web_fragments.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/scripts/__init__.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/scripts/test_audit_outputs.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_auto_id.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_config.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_content_hash.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_docling_helper.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_frontmatter_compose.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_frontmatter_helpers.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_meta_overrides.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_pdf_complexity.py +0 -0
- {any2md-1.0.5 → any2md-1.0.6}/tests/unit/test_validators.py +0 -0
|
@@ -4,6 +4,89 @@ All notable changes to this project will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
6
6
|
|
|
7
|
+
## [1.0.6] — 2026-04-27
|
|
8
|
+
|
|
9
|
+
Security hardening release. Closes 8 actionable findings from the
|
|
10
|
+
2026-04-27 multi-tool audit (semgrep + trivy + gitleaks + bandit +
|
|
11
|
+
pip-audit + manual review by the security-engineer agent + Opus 4.7).
|
|
12
|
+
No public API changes; behavior unchanged for legitimate inputs.
|
|
13
|
+
|
|
14
|
+
### Security
|
|
15
|
+
|
|
16
|
+
- **F1 (HIGH)** SSRF: replaced `trafilatura.fetch_url` and the bare
|
|
17
|
+
`urllib.request.urlopen` paths with a new `any2md/_http.py` module
|
|
18
|
+
that does manual redirect walking with per-hop revalidation. Defends
|
|
19
|
+
against DNS-rebinding and redirect-based bypass of the IP-class
|
|
20
|
+
check. New `validate_url` rejects non-http(s) schemes and
|
|
21
|
+
private/loopback/link-local/reserved targets. 20 MB body cap, 3-hop
|
|
22
|
+
redirect cap, redirect-loop detection. Applied to `convert_url`,
|
|
23
|
+
`_http_last_modified`, and `arxiv_lookup`.
|
|
24
|
+
- **F5 (MED)** `--meta` integrity: `content_hash`, `extracted_via`,
|
|
25
|
+
`source_file`, `lane`, `token_estimate`, `recommended_chunk_level`
|
|
26
|
+
are now **reserved** and silently filtered from `--meta` /
|
|
27
|
+
`--meta-file` / `.any2md.toml` overrides with a stderr `WARN` per
|
|
28
|
+
drop. Defense-in-depth filter also runs inside `frontmatter.compose`.
|
|
29
|
+
- **F3 (MED)** TOML auto-discovery now stops at the first ancestor
|
|
30
|
+
containing `.git` / `pyproject.toml` / `setup.py` / `setup.cfg` /
|
|
31
|
+
`package.json` / `.any2md.toml.boundary`. New `--no-config` flag
|
|
32
|
+
fully disables discovery.
|
|
33
|
+
- **F4 (MED)** DOCX zip-bomb: `_safe_zip_open` enforces a 1 MB cap on
|
|
34
|
+
the declared uncompressed size of `core.xml` and `app.xml` before
|
|
35
|
+
parsing. Oversized members raise `ValueError` → clean per-file FAIL.
|
|
36
|
+
- **F-CVE (MED)** Dependency floors bumped: `lxml>=6.1.0,<7`
|
|
37
|
+
(CVE-2026-41066), `pillow>=12.2.0,<13` (CVE-2026-25990,
|
|
38
|
+
CVE-2026-40192), `urllib3>=2.6.3,<3` (CVE-2026-21441). All ranges
|
|
39
|
+
acquire upper bounds (compat envelope). `requirements.txt` becomes
|
|
40
|
+
a development/CI lockfile with exact pins.
|
|
41
|
+
- **F6 (LOW)** Atomic writes: new `any2md.utils.atomic_write_text`
|
|
42
|
+
writes via `tempfile.mkstemp` + `os.replace`. Refuses to overwrite
|
|
43
|
+
a pre-existing symlink at the output path. Wired through
|
|
44
|
+
`pdf/docx/html/txt` converters; PDF image extraction adds a symlink
|
|
45
|
+
check before `pil_image.save`.
|
|
46
|
+
- **F2 (LOW)** Control-char sanitizer strips C0/C1 controls (preserves
|
|
47
|
+
`\t`, `\n`) from captured Docling `msword_backend` warnings before
|
|
48
|
+
forwarding them to `collected_warnings()` and stderr. Defends
|
|
49
|
+
terminals from ANSI-escape spoofing.
|
|
50
|
+
- **F11 (LOW)** PDF image-dir name now goes through `safe_dir_name`
|
|
51
|
+
(alphanumeric/`_`/`-` only). Closes a `Path.stem == ".."` corner
|
|
52
|
+
case that co-mingled images with markdown outputs.
|
|
53
|
+
- **XXE defense-in-depth**: migrated `xml.etree.ElementTree` →
|
|
54
|
+
`defusedxml.ElementTree` in `docx.py` and `heuristics.py`. Stdlib
|
|
55
|
+
was already safe in Python 3.7+; this silences bandit B405/B314
|
|
56
|
+
permanently and protects against future stdlib regressions.
|
|
57
|
+
|
|
58
|
+
### Added
|
|
59
|
+
|
|
60
|
+
- New module `any2md/_http.py` (`validate_url`, `safe_fetch`).
|
|
61
|
+
- New CLI flag `--no-config`.
|
|
62
|
+
- New helpers `any2md.utils.atomic_write_text` and `safe_dir_name`.
|
|
63
|
+
- New runtime dependency: `defusedxml>=0.7.1,<1`.
|
|
64
|
+
|
|
65
|
+
### Tests
|
|
66
|
+
|
|
67
|
+
- 7 new unit-test files (40 new tests): `test_http_safe.py`,
|
|
68
|
+
`test_meta_reserved_keys.py`, `test_config_walkup.py`,
|
|
69
|
+
`test_zipbomb_guard.py`, `test_atomic_write.py`,
|
|
70
|
+
`test_safe_dir_name.py`, `test_log_sanitizer.py`.
|
|
71
|
+
- Existing 290 tests continue to pass.
|
|
72
|
+
- Bandit audit went from **9 findings** (3 medium, 6 low) to **1 low**
|
|
73
|
+
(legitimate `try/except/pass` in arxiv `_warn`).
|
|
74
|
+
- 5-DOCX regression run reproduces v1.0.5 word counts exactly
|
|
75
|
+
(978/6726/15566/7811/1477).
|
|
76
|
+
|
|
77
|
+
### Audit summary
|
|
78
|
+
|
|
79
|
+
| ID | Title | Severity |
|
|
80
|
+
|----|-------|----------|
|
|
81
|
+
| F1 | SSRF: DNS rebind + redirect bypass | High |
|
|
82
|
+
| F5 | `--meta` clobbers reserved frontmatter | Medium |
|
|
83
|
+
| F3 | TOML discovery walks above project root | Medium |
|
|
84
|
+
| F4 | DOCX zip-bomb amplification | Medium |
|
|
85
|
+
| F-CVE | Transitive deps with known CVEs | Medium |
|
|
86
|
+
| F6 | Symlink-following on output writes | Low |
|
|
87
|
+
| F2 | Control-char passthrough in Docling logs | Low |
|
|
88
|
+
| F11 | PDF stem `..` corner case | Low |
|
|
89
|
+
|
|
7
90
|
## [1.0.5] — 2026-04-27
|
|
8
91
|
|
|
9
92
|
Patch release. Recovers DOCX content that Docling's msword backend
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: any2md
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.6
|
|
4
4
|
Summary: Convert PDF, DOCX, HTML, and TXT files — or web pages by URL — to clean, LLM-optimized Markdown with YAML frontmatter.
|
|
5
5
|
Project-URL: Homepage, https://github.com/rocklambros/any2md
|
|
6
6
|
Project-URL: Issues, https://github.com/rocklambros/any2md/issues
|
|
@@ -12,13 +12,16 @@ Classifier: Operating System :: OS Independent
|
|
|
12
12
|
Classifier: Programming Language :: Python :: 3
|
|
13
13
|
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
14
14
|
Requires-Python: >=3.10
|
|
15
|
-
Requires-Dist: beautifulsoup4
|
|
16
|
-
Requires-Dist:
|
|
17
|
-
Requires-Dist:
|
|
18
|
-
Requires-Dist:
|
|
19
|
-
Requires-Dist:
|
|
20
|
-
Requires-Dist:
|
|
21
|
-
Requires-Dist:
|
|
15
|
+
Requires-Dist: beautifulsoup4<5,>=4.12.0
|
|
16
|
+
Requires-Dist: defusedxml<1,>=0.7.1
|
|
17
|
+
Requires-Dist: lxml<7,>=6.1.0
|
|
18
|
+
Requires-Dist: mammoth<2,>=1.6.0
|
|
19
|
+
Requires-Dist: markdownify<2,>=0.13.0
|
|
20
|
+
Requires-Dist: pillow<13,>=12.2.0
|
|
21
|
+
Requires-Dist: pymupdf4llm<1,>=0.0.17
|
|
22
|
+
Requires-Dist: pymupdf<2,>=1.24.0
|
|
23
|
+
Requires-Dist: trafilatura<3,>=1.12.0
|
|
24
|
+
Requires-Dist: urllib3<3,>=2.6.3
|
|
22
25
|
Provides-Extra: dev
|
|
23
26
|
Requires-Dist: pytest-snapshot>=0.9.0; extra == 'dev'
|
|
24
27
|
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""SSRF-safe HTTP fetching utilities for any2md.
|
|
2
|
+
|
|
3
|
+
Centralizes URL scheme + IP validation and provides a fetcher that
|
|
4
|
+
walks redirects manually and revalidates the host on each hop. This
|
|
5
|
+
defends against:
|
|
6
|
+
- DNS rebinding (host -> public IP for validator, private IP for fetch)
|
|
7
|
+
- Redirect-based SSRF (public landing page -> 302 to 169.254.169.254)
|
|
8
|
+
- Non-http(s) schemes (file://, gopher://, etc.)
|
|
9
|
+
- Decompression / huge-response DoS (response-size cap)
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import ipaddress
|
|
15
|
+
import socket
|
|
16
|
+
import urllib.error
|
|
17
|
+
import urllib.parse
|
|
18
|
+
import urllib.request
|
|
19
|
+
|
|
20
|
+
_MAX_REDIRECT_HOPS = 3
|
|
21
|
+
_FETCH_TIMEOUT = 15 # seconds
|
|
22
|
+
_MAX_RESPONSE_BYTES = 20 * 1024 * 1024 # 20 MB cap on body read
|
|
23
|
+
_USER_AGENT = "any2md/1.0.6"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def validate_url(url: str) -> str | None:
|
|
27
|
+
"""Validate URL scheme + host. Returns an error message or None.
|
|
28
|
+
|
|
29
|
+
One-shot DNS lookup; rebind protection requires re-checking on
|
|
30
|
+
every redirect hop (handled by ``safe_fetch``).
|
|
31
|
+
"""
|
|
32
|
+
parsed = urllib.parse.urlparse(url)
|
|
33
|
+
if parsed.scheme not in ("http", "https"):
|
|
34
|
+
return f"unsupported scheme: {parsed.scheme!r}"
|
|
35
|
+
if not parsed.hostname:
|
|
36
|
+
return f"no hostname in URL: {url}"
|
|
37
|
+
try:
|
|
38
|
+
infos = socket.getaddrinfo(parsed.hostname, None, type=socket.SOCK_STREAM)
|
|
39
|
+
except socket.gaierror:
|
|
40
|
+
return f"cannot resolve host: {parsed.hostname}"
|
|
41
|
+
for *_, sockaddr in infos:
|
|
42
|
+
try:
|
|
43
|
+
addr = ipaddress.ip_address(sockaddr[0])
|
|
44
|
+
except ValueError:
|
|
45
|
+
continue
|
|
46
|
+
if (
|
|
47
|
+
addr.is_private
|
|
48
|
+
or addr.is_reserved
|
|
49
|
+
or addr.is_loopback
|
|
50
|
+
or addr.is_link_local
|
|
51
|
+
):
|
|
52
|
+
return f"URL resolves to disallowed address: {addr}"
|
|
53
|
+
return None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class _NoFollowRedirect(urllib.request.HTTPRedirectHandler):
|
|
57
|
+
"""Disable urllib's automatic redirect following."""
|
|
58
|
+
|
|
59
|
+
def redirect_request(self, *args, **kwargs): # noqa: ARG002
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def safe_fetch(
|
|
64
|
+
url: str,
|
|
65
|
+
*,
|
|
66
|
+
method: str = "GET",
|
|
67
|
+
max_hops: int = _MAX_REDIRECT_HOPS,
|
|
68
|
+
) -> tuple[bytes | None, dict | None, str | None]:
|
|
69
|
+
"""Fetch ``url`` with manual redirect walking + per-hop revalidation.
|
|
70
|
+
|
|
71
|
+
Returns ``(body, headers, None)`` on 2xx, or ``(None, None, error)``
|
|
72
|
+
on rejection / failure. ``headers`` is the response headers as a
|
|
73
|
+
plain ``dict``.
|
|
74
|
+
"""
|
|
75
|
+
visited: list[str] = []
|
|
76
|
+
current = url
|
|
77
|
+
opener = urllib.request.build_opener(_NoFollowRedirect())
|
|
78
|
+
for hop in range(max_hops + 1):
|
|
79
|
+
if current in visited:
|
|
80
|
+
return None, None, "redirect loop"
|
|
81
|
+
visited.append(current)
|
|
82
|
+
err = validate_url(current)
|
|
83
|
+
if err:
|
|
84
|
+
return None, None, err
|
|
85
|
+
req = urllib.request.Request(
|
|
86
|
+
current, method=method, headers={"User-Agent": _USER_AGENT}
|
|
87
|
+
)
|
|
88
|
+
try:
|
|
89
|
+
resp = opener.open(req, timeout=_FETCH_TIMEOUT)
|
|
90
|
+
except urllib.error.HTTPError as e:
|
|
91
|
+
if e.code in (301, 302, 303, 307, 308):
|
|
92
|
+
if hop >= max_hops:
|
|
93
|
+
return None, None, f"too many redirects (>{max_hops})"
|
|
94
|
+
location = e.headers.get("Location") if e.headers else None
|
|
95
|
+
if not location:
|
|
96
|
+
return None, None, f"HTTP {e.code} without Location"
|
|
97
|
+
current = urllib.parse.urljoin(current, location)
|
|
98
|
+
continue
|
|
99
|
+
return None, None, f"HTTP {e.code}"
|
|
100
|
+
except (urllib.error.URLError, OSError, TimeoutError) as e:
|
|
101
|
+
return None, None, f"fetch error: {e}"
|
|
102
|
+
body = resp.read(_MAX_RESPONSE_BYTES + 1)
|
|
103
|
+
if len(body) > _MAX_RESPONSE_BYTES:
|
|
104
|
+
return None, None, f"response exceeds {_MAX_RESPONSE_BYTES} bytes"
|
|
105
|
+
return body, dict(resp.headers), None
|
|
106
|
+
return None, None, f"too many redirects (>{max_hops})"
|
|
@@ -20,6 +20,7 @@ from any2md.converters import (
|
|
|
20
20
|
set_output_mode,
|
|
21
21
|
)
|
|
22
22
|
from any2md.converters.html import convert_url
|
|
23
|
+
from any2md.frontmatter import filter_reserved_overrides
|
|
23
24
|
from any2md.pipeline import PipelineOptions
|
|
24
25
|
from any2md.utils import sanitize_filename, url_to_filename
|
|
25
26
|
|
|
@@ -169,6 +170,11 @@ def main():
|
|
|
169
170
|
action="store_true",
|
|
170
171
|
help="Promote pipeline validation warnings to errors (exit 3).",
|
|
171
172
|
)
|
|
173
|
+
parser.add_argument(
|
|
174
|
+
"--no-config",
|
|
175
|
+
action="store_true",
|
|
176
|
+
help="Skip .any2md.toml auto-discovery (overrides default walk-up).",
|
|
177
|
+
)
|
|
172
178
|
parser.add_argument(
|
|
173
179
|
"--docx-fallback-on-warn",
|
|
174
180
|
action=argparse.BooleanOptionalAction,
|
|
@@ -222,7 +228,7 @@ def main():
|
|
|
222
228
|
overrides: dict[str, Any] = {}
|
|
223
229
|
auto_id_prefix, auto_id_type_code = "LOCAL", "DOC"
|
|
224
230
|
|
|
225
|
-
discovered = discover_config()
|
|
231
|
+
discovered = None if args.no_config else discover_config()
|
|
226
232
|
if discovered is not None:
|
|
227
233
|
cfg = load_toml(discovered)
|
|
228
234
|
overrides = _deep_merge_overrides(overrides, extract_meta_overrides(cfg))
|
|
@@ -249,6 +255,9 @@ def main():
|
|
|
249
255
|
print(f"Error: {e}", file=sys.stderr)
|
|
250
256
|
sys.exit(1)
|
|
251
257
|
overrides = _deep_merge_overrides(overrides, cli_meta)
|
|
258
|
+
overrides = filter_reserved_overrides(
|
|
259
|
+
overrides, source_label="--meta/--meta-file/.any2md.toml"
|
|
260
|
+
)
|
|
252
261
|
|
|
253
262
|
if (
|
|
254
263
|
args.high_fidelity
|
|
@@ -42,17 +42,31 @@ def load_toml(path: Path) -> dict[str, Any]:
|
|
|
42
42
|
return {}
|
|
43
43
|
|
|
44
44
|
|
|
45
|
+
_PROJECT_BOUNDARY_MARKERS = (
|
|
46
|
+
".git",
|
|
47
|
+
"pyproject.toml",
|
|
48
|
+
"setup.py",
|
|
49
|
+
"setup.cfg",
|
|
50
|
+
"package.json",
|
|
51
|
+
".any2md.toml.boundary",
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
45
55
|
def discover_config(start: Path | None = None) -> Path | None:
|
|
46
56
|
"""Walk up from ``start`` (default: cwd) looking for ``.any2md.toml``.
|
|
47
57
|
|
|
48
|
-
|
|
49
|
-
|
|
58
|
+
Stops at the first ancestor containing a project boundary marker
|
|
59
|
+
(``.git``, ``pyproject.toml``, ``setup.py``, ``setup.cfg``,
|
|
60
|
+
``package.json``, or ``.any2md.toml.boundary``) so config in
|
|
61
|
+
unrelated parent directories cannot leak into a project's run.
|
|
50
62
|
"""
|
|
51
63
|
cur = (start or Path.cwd()).resolve()
|
|
52
64
|
while True:
|
|
53
65
|
candidate = cur / ".any2md.toml"
|
|
54
66
|
if candidate.is_file():
|
|
55
67
|
return candidate
|
|
68
|
+
if any((cur / m).exists() for m in _PROJECT_BOUNDARY_MARKERS):
|
|
69
|
+
return None
|
|
56
70
|
if cur.parent == cur:
|
|
57
71
|
return None
|
|
58
72
|
cur = cur.parent
|
|
@@ -10,9 +10,12 @@ produces the markdown body.
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
import logging
|
|
13
|
+
import re
|
|
13
14
|
import sys
|
|
14
|
-
import xml.etree.ElementTree as ET
|
|
15
15
|
import zipfile
|
|
16
|
+
|
|
17
|
+
from defusedxml.ElementTree import ParseError as _XmlParseError
|
|
18
|
+
from defusedxml.ElementTree import parse as _xml_parse
|
|
16
19
|
from datetime import date
|
|
17
20
|
from pathlib import Path
|
|
18
21
|
|
|
@@ -25,11 +28,44 @@ from any2md.converters import add_warnings, is_quiet
|
|
|
25
28
|
from any2md.frontmatter import SourceMeta, compose
|
|
26
29
|
from any2md.heuristics import filter_organization
|
|
27
30
|
from any2md.pipeline import PipelineOptions
|
|
28
|
-
from any2md.utils import sanitize_filename
|
|
31
|
+
from any2md.utils import atomic_write_text, sanitize_filename
|
|
29
32
|
|
|
30
33
|
|
|
31
34
|
_DOCLING_MSWORD_LOGGER = "docling.backend.msword_backend"
|
|
32
35
|
|
|
36
|
+
_LOG_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _sanitize_log_text(s: str) -> str:
|
|
40
|
+
"""Strip ASCII C0/C1 control chars from a log message before re-emission.
|
|
41
|
+
|
|
42
|
+
Defends terminals from ANSI-escape spoofing planted in malicious DOCX
|
|
43
|
+
content that surfaces as a Docling msword_backend warning. ``\\t``
|
|
44
|
+
and ``\\n`` are preserved (not in the strip class).
|
|
45
|
+
"""
|
|
46
|
+
return _LOG_CONTROL_CHARS_RE.sub("", s)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
_MAX_DOCX_METADATA_SIZE = 1 * 1024 * 1024 # 1 MB; core.xml/app.xml are typically <10 KB
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _safe_zip_open(
|
|
53
|
+
z: zipfile.ZipFile, name: str, max_size: int = _MAX_DOCX_METADATA_SIZE
|
|
54
|
+
):
|
|
55
|
+
"""Open a fixed-name DOCX member after a declared-size sanity check.
|
|
56
|
+
|
|
57
|
+
Defends against zip-bomb amplification: a malicious DOCX can declare
|
|
58
|
+
an uncompressed_size of 10 GB for ``core.xml`` and bomb the XML
|
|
59
|
+
parser. Raises ``ValueError`` (caught by ``convert_docx``'s existing
|
|
60
|
+
handler -> file fails cleanly).
|
|
61
|
+
"""
|
|
62
|
+
info = z.getinfo(name)
|
|
63
|
+
if info.file_size > max_size:
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f"{name} declared size {info.file_size} exceeds {max_size}-byte limit"
|
|
66
|
+
)
|
|
67
|
+
return z.open(name)
|
|
68
|
+
|
|
33
69
|
|
|
34
70
|
class _DoclingMswordWarningCapture:
|
|
35
71
|
"""Buffer WARNING+ records from Docling's DOCX backend logger.
|
|
@@ -89,8 +125,8 @@ def _read_docx_metadata(docx_path: Path) -> dict[str, object]:
|
|
|
89
125
|
try:
|
|
90
126
|
with zipfile.ZipFile(docx_path) as z:
|
|
91
127
|
try:
|
|
92
|
-
with z
|
|
93
|
-
root =
|
|
128
|
+
with _safe_zip_open(z, "docProps/core.xml") as f:
|
|
129
|
+
root = _xml_parse(f).getroot()
|
|
94
130
|
title = root.findtext("dc:title", namespaces=_NS_CORE)
|
|
95
131
|
if title:
|
|
96
132
|
out["title_hint"] = title.strip()
|
|
@@ -105,8 +141,8 @@ def _read_docx_metadata(docx_path: Path) -> dict[str, object]:
|
|
|
105
141
|
except KeyError:
|
|
106
142
|
pass
|
|
107
143
|
try:
|
|
108
|
-
with z
|
|
109
|
-
root =
|
|
144
|
+
with _safe_zip_open(z, "docProps/app.xml") as f:
|
|
145
|
+
root = _xml_parse(f).getroot()
|
|
110
146
|
company = root.findtext("ext:Company", namespaces=_NS_APP)
|
|
111
147
|
application = root.findtext("ext:Application", namespaces=_NS_APP)
|
|
112
148
|
# v1.0.2: Company takes priority for `organization` (real
|
|
@@ -126,7 +162,7 @@ def _read_docx_metadata(docx_path: Path) -> dict[str, object]:
|
|
|
126
162
|
out["produced_by"] = app_result.produced_by
|
|
127
163
|
except KeyError:
|
|
128
164
|
pass
|
|
129
|
-
except (zipfile.BadZipFile,
|
|
165
|
+
except (zipfile.BadZipFile, _XmlParseError):
|
|
130
166
|
pass
|
|
131
167
|
return out
|
|
132
168
|
|
|
@@ -233,7 +269,12 @@ def convert_docx(
|
|
|
233
269
|
md_text, _ = _extract_via_mammoth(docx_path, options)
|
|
234
270
|
extracted_via = "docling→mammoth (warning fallback)"
|
|
235
271
|
lane = "text"
|
|
236
|
-
add_warnings(
|
|
272
|
+
add_warnings(
|
|
273
|
+
[
|
|
274
|
+
f"docling.msword_backend: {_sanitize_log_text(m)}"
|
|
275
|
+
for m in docling_warnings
|
|
276
|
+
]
|
|
277
|
+
)
|
|
237
278
|
|
|
238
279
|
md_text, warnings = pipeline.run(md_text, lane, options)
|
|
239
280
|
add_warnings(warnings)
|
|
@@ -260,7 +301,7 @@ def convert_docx(
|
|
|
260
301
|
full = compose(md_text, meta, options, overrides=options.frontmatter_overrides)
|
|
261
302
|
|
|
262
303
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
263
|
-
out_path
|
|
304
|
+
atomic_write_text(out_path, full)
|
|
264
305
|
wc = meta.word_count or 0
|
|
265
306
|
suffix = f", {len(warnings)} warning(s)" if warnings else ""
|
|
266
307
|
if not is_quiet():
|
|
@@ -2,12 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
import ipaddress
|
|
6
|
-
import socket
|
|
7
5
|
import sys
|
|
8
|
-
import urllib.parse
|
|
9
|
-
import urllib.request
|
|
10
6
|
from datetime import date
|
|
7
|
+
from email.utils import parsedate_to_datetime
|
|
11
8
|
from pathlib import Path
|
|
12
9
|
|
|
13
10
|
import markdownify
|
|
@@ -15,87 +12,51 @@ import trafilatura
|
|
|
15
12
|
from bs4 import BeautifulSoup
|
|
16
13
|
|
|
17
14
|
from any2md import pipeline
|
|
15
|
+
from any2md._http import safe_fetch
|
|
18
16
|
from any2md.converters import add_warnings, is_quiet
|
|
19
17
|
from any2md.frontmatter import SourceMeta, compose
|
|
20
18
|
from any2md.pipeline import PipelineOptions
|
|
21
19
|
from any2md.utils import (
|
|
20
|
+
atomic_write_text,
|
|
22
21
|
read_text_with_fallback,
|
|
23
22
|
sanitize_filename,
|
|
24
23
|
url_to_filename,
|
|
25
24
|
)
|
|
26
25
|
|
|
27
|
-
_MAX_FILE_SIZE = 100 * 1024 * 1024
|
|
28
|
-
|
|
29
26
|
|
|
30
|
-
|
|
31
|
-
"""Validate that a URL does not point to a private/reserved IP.
|
|
32
|
-
|
|
33
|
-
Returns an error message if the host is disallowed, or None if safe.
|
|
34
|
-
"""
|
|
35
|
-
parsed = urllib.parse.urlparse(url)
|
|
36
|
-
hostname = parsed.hostname
|
|
37
|
-
if not hostname:
|
|
38
|
-
return f"No hostname in URL: {url}"
|
|
39
|
-
try:
|
|
40
|
-
infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
|
|
41
|
-
except socket.gaierror:
|
|
42
|
-
return f"Cannot resolve hostname: {hostname}"
|
|
43
|
-
for _family, _type, _proto, _canon, sockaddr in infos:
|
|
44
|
-
ip_str = sockaddr[0]
|
|
45
|
-
try:
|
|
46
|
-
addr = ipaddress.ip_address(ip_str)
|
|
47
|
-
except ValueError:
|
|
48
|
-
continue
|
|
49
|
-
if (
|
|
50
|
-
addr.is_private
|
|
51
|
-
or addr.is_reserved
|
|
52
|
-
or addr.is_loopback
|
|
53
|
-
or addr.is_link_local
|
|
54
|
-
):
|
|
55
|
-
return f"URL resolves to disallowed address: {ip_str}"
|
|
56
|
-
return None
|
|
27
|
+
_MAX_FILE_SIZE = 100 * 1024 * 1024
|
|
57
28
|
|
|
58
29
|
|
|
59
30
|
def fetch_url(url: str) -> tuple[str | None, str | None]:
|
|
60
|
-
"""Fetch HTML
|
|
61
|
-
|
|
62
|
-
Only http and https schemes are accepted. SSRF protection blocks
|
|
63
|
-
requests to private/reserved/loopback addresses.
|
|
31
|
+
"""Fetch HTML from a URL using the SSRF-safe fetcher.
|
|
64
32
|
|
|
65
|
-
Returns (html_string, None) on success or (None, error_message)
|
|
33
|
+
Returns (html_string, None) on success or (None, error_message)
|
|
34
|
+
on failure. Per-hop host revalidation defends against DNS rebind
|
|
35
|
+
via redirects.
|
|
66
36
|
"""
|
|
67
|
-
|
|
68
|
-
if parsed.scheme not in ("http", "https"):
|
|
69
|
-
return None, f"Unsupported URL scheme: {parsed.scheme!r}"
|
|
70
|
-
err = _validate_url_host(url)
|
|
37
|
+
body, _headers, err = safe_fetch(url)
|
|
71
38
|
if err:
|
|
72
39
|
return None, err
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
return None, f"Failed to fetch URL: {url}"
|
|
77
|
-
return html, None
|
|
78
|
-
except Exception as e: # noqa: BLE001
|
|
79
|
-
return None, f"Error fetching URL: {e}"
|
|
40
|
+
if body is None:
|
|
41
|
+
return None, f"empty body for {url}"
|
|
42
|
+
return body.decode("utf-8", errors="replace"), None
|
|
80
43
|
|
|
81
44
|
|
|
82
45
|
def _http_last_modified(url: str) -> str | None:
|
|
83
|
-
"""Single HEAD request for Last-Modified. Best-effort.
|
|
84
|
-
try:
|
|
85
|
-
req = urllib.request.Request(
|
|
86
|
-
url,
|
|
87
|
-
method="HEAD",
|
|
88
|
-
headers={"User-Agent": "any2md/1.0"},
|
|
89
|
-
)
|
|
90
|
-
with urllib.request.urlopen(req, timeout=10) as resp:
|
|
91
|
-
lm = resp.headers.get("Last-Modified")
|
|
92
|
-
if lm:
|
|
93
|
-
from email.utils import parsedate_to_datetime
|
|
46
|
+
"""Single HEAD request for Last-Modified. Best-effort.
|
|
94
47
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
48
|
+
SSRF-safe: same scheme + IP validation as ``fetch_url``.
|
|
49
|
+
"""
|
|
50
|
+
_body, headers, err = safe_fetch(url, method="HEAD")
|
|
51
|
+
if err or not headers:
|
|
52
|
+
return None
|
|
53
|
+
lm = headers.get("Last-Modified")
|
|
54
|
+
if not lm:
|
|
55
|
+
return None
|
|
56
|
+
try:
|
|
57
|
+
return parsedate_to_datetime(lm).date().isoformat()
|
|
58
|
+
except (TypeError, ValueError):
|
|
59
|
+
return None
|
|
99
60
|
|
|
100
61
|
|
|
101
62
|
def _bs4_preclean(html: str) -> str:
|
|
@@ -227,7 +188,7 @@ def convert_html(
|
|
|
227
188
|
full = compose(md_text, meta, options, overrides=options.frontmatter_overrides)
|
|
228
189
|
|
|
229
190
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
230
|
-
out_path
|
|
191
|
+
atomic_write_text(out_path, full)
|
|
231
192
|
wc = meta.word_count or 0
|
|
232
193
|
suffix = f", {len(warnings)} warning(s)" if warnings else ""
|
|
233
194
|
if not is_quiet():
|
|
@@ -22,7 +22,7 @@ from any2md.converters import add_warnings, is_quiet
|
|
|
22
22
|
from any2md.frontmatter import SourceMeta, compose
|
|
23
23
|
from any2md.heuristics import filter_organization
|
|
24
24
|
from any2md.pipeline import PipelineOptions
|
|
25
|
-
from any2md.utils import sanitize_filename
|
|
25
|
+
from any2md.utils import atomic_write_text, safe_dir_name, sanitize_filename
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
def _parse_pdf_authors(raw: str | None) -> list[str]:
|
|
@@ -140,7 +140,7 @@ def _extract_via_docling(
|
|
|
140
140
|
if options.save_images:
|
|
141
141
|
pictures = getattr(result.document, "pictures", None) or []
|
|
142
142
|
if pictures:
|
|
143
|
-
images_dir = output_dir / "images" / pdf_path.stem
|
|
143
|
+
images_dir = output_dir / "images" / safe_dir_name(pdf_path.stem)
|
|
144
144
|
images_dir.mkdir(parents=True, exist_ok=True)
|
|
145
145
|
for i, picture in enumerate(pictures):
|
|
146
146
|
try:
|
|
@@ -148,6 +148,12 @@ def _extract_via_docling(
|
|
|
148
148
|
if pil_image is None:
|
|
149
149
|
continue
|
|
150
150
|
img_path = images_dir / f"img{i + 1}.png"
|
|
151
|
+
if img_path.exists() and img_path.is_symlink():
|
|
152
|
+
print(
|
|
153
|
+
f" WARN: skipping image {i + 1}: target is a symlink.",
|
|
154
|
+
file=sys.stderr,
|
|
155
|
+
)
|
|
156
|
+
continue
|
|
151
157
|
pil_image.save(str(img_path))
|
|
152
158
|
except Exception as e: # noqa: BLE001
|
|
153
159
|
print(
|
|
@@ -265,7 +271,7 @@ def convert_pdf(
|
|
|
265
271
|
full = compose(md_text, meta, options, overrides=options.frontmatter_overrides)
|
|
266
272
|
|
|
267
273
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
268
|
-
out_path
|
|
274
|
+
atomic_write_text(out_path, full)
|
|
269
275
|
suffix = f", {len(warnings)} warning(s)" if warnings else ""
|
|
270
276
|
if not is_quiet():
|
|
271
277
|
print(f" OK: {out_name} ({page_count} pages, via {extracted_via}{suffix})")
|
|
@@ -11,7 +11,7 @@ from any2md import pipeline
|
|
|
11
11
|
from any2md.converters import add_warnings, is_quiet
|
|
12
12
|
from any2md.frontmatter import SourceMeta, compose
|
|
13
13
|
from any2md.pipeline import PipelineOptions
|
|
14
|
-
from any2md.utils import read_text_with_fallback, sanitize_filename
|
|
14
|
+
from any2md.utils import atomic_write_text, read_text_with_fallback, sanitize_filename
|
|
15
15
|
|
|
16
16
|
# Existing structurize() heuristic stays in this file from v0.7 — keep it.
|
|
17
17
|
_SEPARATOR_RE = re.compile(r"^([=\-*_~])\1{2,}\s*$")
|
|
@@ -199,7 +199,7 @@ def convert_txt(
|
|
|
199
199
|
full = compose(md_text, meta, options, overrides=options.frontmatter_overrides)
|
|
200
200
|
|
|
201
201
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
202
|
-
out_path
|
|
202
|
+
atomic_write_text(out_path, full)
|
|
203
203
|
word_count = meta.word_count or 0
|
|
204
204
|
suffix = f", {len(warnings)} warning(s)" if warnings else ""
|
|
205
205
|
if not is_quiet():
|
|
@@ -10,7 +10,9 @@ from __future__ import annotations
|
|
|
10
10
|
import hashlib
|
|
11
11
|
import math
|
|
12
12
|
import re
|
|
13
|
+
import sys
|
|
13
14
|
import unicodedata
|
|
15
|
+
from collections.abc import Mapping
|
|
14
16
|
from dataclasses import dataclass
|
|
15
17
|
from datetime import date as _date_cls
|
|
16
18
|
from typing import Any, Literal
|
|
@@ -19,6 +21,44 @@ from any2md import heuristics
|
|
|
19
21
|
from any2md.pipeline import Lane, PipelineOptions
|
|
20
22
|
|
|
21
23
|
|
|
24
|
+
_RESERVED_OVERRIDE_KEYS = frozenset(
|
|
25
|
+
{
|
|
26
|
+
"content_hash",
|
|
27
|
+
"token_estimate",
|
|
28
|
+
"recommended_chunk_level",
|
|
29
|
+
"extracted_via",
|
|
30
|
+
"source_file",
|
|
31
|
+
"lane",
|
|
32
|
+
}
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def filter_reserved_overrides(
|
|
37
|
+
overrides: Mapping[str, Any] | None,
|
|
38
|
+
*,
|
|
39
|
+
source_label: str = "overrides",
|
|
40
|
+
) -> dict[str, Any] | None:
|
|
41
|
+
"""Drop top-level reserved keys; emit stderr WARN per drop.
|
|
42
|
+
|
|
43
|
+
Reserved keys are derived from the source document and must not be
|
|
44
|
+
overridable via ``--meta``, ``--meta-file``, or ``.any2md.toml``.
|
|
45
|
+
Nested dicts under non-reserved keys are returned untouched.
|
|
46
|
+
"""
|
|
47
|
+
if overrides is None:
|
|
48
|
+
return None
|
|
49
|
+
out: dict[str, Any] = {}
|
|
50
|
+
for k, v in overrides.items():
|
|
51
|
+
if k in _RESERVED_OVERRIDE_KEYS:
|
|
52
|
+
print(
|
|
53
|
+
f" WARN: ignoring {source_label} key {k!r} "
|
|
54
|
+
f"— reserved (derived from source).",
|
|
55
|
+
file=sys.stderr,
|
|
56
|
+
)
|
|
57
|
+
continue
|
|
58
|
+
out[k] = v
|
|
59
|
+
return out
|
|
60
|
+
|
|
61
|
+
|
|
22
62
|
def compute_content_hash(body: str) -> str:
|
|
23
63
|
"""SHA-256 of NFC-normalized, LF-line-ended body. SSRM §5.1.
|
|
24
64
|
|
|
@@ -329,6 +369,7 @@ def compose(
|
|
|
329
369
|
4. Emit YAML frontmatter in spec §3.2-3.4 order and concatenate the body.
|
|
330
370
|
"""
|
|
331
371
|
body = _normalize_body(body)
|
|
372
|
+
overrides = filter_reserved_overrides(overrides, source_label="compose()")
|
|
332
373
|
fields = _build_fields(body, meta, options)
|
|
333
374
|
if overrides:
|
|
334
375
|
fields = _deep_merge(fields, overrides)
|