any2md 1.1.1__tar.gz → 1.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {any2md-1.1.1 → any2md-1.2.1}/CHANGELOG.md +188 -1
- {any2md-1.1.1 → any2md-1.2.1}/CONTRIBUTING.md +8 -9
- {any2md-1.1.1 → any2md-1.2.1}/PKG-INFO +7 -8
- {any2md-1.1.1 → any2md-1.2.1}/SECURITY.md +2 -3
- {any2md-1.1.1 → any2md-1.2.1}/any2md/__init__.py +1 -1
- {any2md-1.1.1 → any2md-1.2.1}/any2md/_http.py +52 -1
- {any2md-1.1.1 → any2md-1.2.1}/any2md/cli.py +72 -22
- {any2md-1.1.1 → any2md-1.2.1}/any2md/config.py +2 -15
- {any2md-1.1.1 → any2md-1.2.1}/any2md/converters/docx.py +62 -16
- {any2md-1.1.1 → any2md-1.2.1}/any2md/converters/html.py +125 -7
- {any2md-1.1.1 → any2md-1.2.1}/any2md/converters/pdf.py +14 -12
- {any2md-1.1.1 → any2md-1.2.1}/any2md/converters/txt.py +9 -4
- {any2md-1.1.1 → any2md-1.2.1}/any2md/frontmatter.py +48 -15
- {any2md-1.1.1 → any2md-1.2.1}/any2md/heuristics.py +44 -10
- {any2md-1.1.1 → any2md-1.2.1}/any2md/pipeline/cleanup.py +53 -3
- any2md-1.2.1/any2md/pipeline/structured.py +639 -0
- {any2md-1.1.1 → any2md-1.2.1}/any2md/pipeline/text.py +146 -37
- {any2md-1.1.1 → any2md-1.2.1}/any2md/utils.py +0 -10
- {any2md-1.1.1 → any2md-1.2.1}/docs/architecture.md +10 -6
- {any2md-1.1.1 → any2md-1.2.1}/docs/cli-reference.md +6 -1
- {any2md-1.1.1 → any2md-1.2.1}/docs/output-format.md +46 -8
- {any2md-1.1.1 → any2md-1.2.1}/pyproject.toml +4 -5
- {any2md-1.1.1 → any2md-1.2.1}/requirements.txt +6 -6
- any2md-1.2.1/tests/cli/test_cli_existing_flags.py +65 -0
- any2md-1.2.1/tests/cli/test_cli_meta.py +96 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/cli/test_cli_ocr_save_images.py +11 -4
- any2md-1.2.1/tests/cli/test_cli_symlinks.py +197 -0
- any2md-1.2.1/tests/fixtures/docs/web_bold_title_first_line.html +15 -0
- any2md-1.2.1/tests/fixtures/docs/web_h1_in_hero.html +38 -0
- any2md-1.2.1/tests/fixtures/docs/web_h3_sections.html +21 -0
- any2md-1.2.1/tests/fixtures/docs/web_no_title.html +12 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/snapshots/table_heavy.fallback.md +0 -2
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_docx_mammoth_fallback.py +16 -0
- any2md-1.2.1/tests/integration/test_html_h1.py +148 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_html_trafilatura.py +13 -0
- any2md-1.2.1/tests/unit/pipeline/test_cleanup_strip_links.py +69 -0
- any2md-1.2.1/tests/unit/pipeline/test_fenced_lines.py +305 -0
- any2md-1.2.1/tests/unit/pipeline/test_structured_heading_hierarchy.py +195 -0
- any2md-1.2.1/tests/unit/pipeline/test_text_dehyphenate.py +117 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_text_repair_line_wraps.py +15 -0
- any2md-1.2.1/tests/unit/test_converter_emission_errors.py +42 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_docling_cache.py +2 -2
- any2md-1.2.1/tests/unit/test_docling_local_paths.py +86 -0
- any2md-1.2.1/tests/unit/test_docx_external_images.py +212 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_frontmatter_compose.py +28 -0
- any2md-1.2.1/tests/unit/test_html_heading_levels.py +128 -0
- any2md-1.2.1/tests/unit/test_html_title_h1.py +134 -0
- any2md-1.2.1/tests/unit/test_http_ipv6_embedded.py +86 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_lockfile_freshness.py +1 -1
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_meta_overrides.py +59 -0
- any2md-1.2.1/tests/unit/test_redos_equivalence.py +138 -0
- any2md-1.2.1/tests/unit/test_redos_hardening.py +147 -0
- any2md-1.2.1/tests/unit/test_redos_review_followups.py +143 -0
- any2md-1.2.1/tests/unit/test_scan_directory_case.py +28 -0
- any2md-1.2.1/tests/unit/test_terminal_escape_output.py +217 -0
- any2md-1.1.1/any2md/pipeline/structured.py +0 -172
- any2md-1.1.1/tests/cli/test_cli_existing_flags.py +0 -52
- any2md-1.1.1/tests/cli/test_cli_meta.py +0 -36
- any2md-1.1.1/tests/unit/pipeline/test_structured_heading_hierarchy.py +0 -34
- any2md-1.1.1/tests/unit/pipeline/test_text_dehyphenate.py +0 -29
- {any2md-1.1.1 → any2md-1.2.1}/.gitignore +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/CODE_OF_CONDUCT.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/LICENSE +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/README.md +1 -1
- {any2md-1.1.1 → any2md-1.2.1}/any2md/__main__.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/any2md/_docling.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/any2md/_docling_cache.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/any2md/_logging.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/any2md/converters/__init__.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/any2md/pipeline/__init__.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/any2md/validators.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/assets/logo-original.jpeg +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/assets/logo.png +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/assets/social-preview.jpg +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-any2md-v1-phase1-foundation.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-any2md-v1-phase2-docling-backend.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-any2md-v1-phase3-figures-ocr-text-lane.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-any2md-v1-phase4-polish-config-docs.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-any2md-v1.0.2-implementation.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/plans/2026-04-26-issue-17-toc-followups.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/plans/2026-04-27-v1.0.6-security-hardening.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/plans/2026-05-04-v1.1.0-persistent-converter.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/plans/2026-05-06-v1.1.1-security-hardening.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/specs/2026-04-26-any2md-v1-design.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/specs/2026-04-26-any2md-v1.0.2-design.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/specs/2026-04-26-issue-17-toc-followups-design.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/specs/2026-04-26-v1.0.3-empty-title-orphan-punct-design.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/specs/2026-04-27-v1.0.6-security-hardening-design.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/specs/2026-05-04-v1.1.0-persistent-converter-design.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/superpowers/specs/2026-05-06-v1.1.1-security-design.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/troubleshooting.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/docs/upgrading-from-0.7.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/scripts/audit-outputs.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/scripts/bench_docling_cache.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/scripts/validate-phase1.sh +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/__init__.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/cli/__init__.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/cli/test_cli_backend.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/cli/test_cli_exit_codes.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/cli/test_cli_high_fidelity.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/cli/test_cli_no_arxiv_lookup.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/cli/test_cli_quiet_verbose.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/cli/test_cli_strict.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/conftest.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/__init__.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/docs/ligatures_and_softhyphens.txt +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/docs/multi_column.pdf +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/docs/table_heavy.docx +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/docs/web_page.html +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/make_fixtures.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/snapshots/ligatures_and_softhyphens.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/snapshots/multi_column.docling.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/snapshots/multi_column.fallback.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/snapshots/table_heavy.docling.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/fixtures/snapshots/web_page.md +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/__init__.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_docling_persistence.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_docx_backend_override.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_docx_docling.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_docx_fallback_on_warn.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_pdf_backend_override.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_pdf_docling.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_pdf_pymupdf_fallback.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_save_images.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_snapshots.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/integration/test_txt.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/__init__.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/__init__.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_footnote_markers.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_html_entities.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_html_entities_double_encoded.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_ligatures.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_nfc.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_quotes_dashes.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_soft_hyphens.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_validate.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_cleanup_whitespace.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_runner.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_strip_orphan_punctuation.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_structured_body_cleanup_stages.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_structured_compact_tables.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_structured_lift_figures.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_structured_normalize_cites.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_text_dedupe_paragraphs.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_text_dedupe_toc.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_text_dedupe_toc_table.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_text_restore_lists_code.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_text_strip_cover_artifacts.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_text_strip_headers_footers.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_text_strip_repeated_byline.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/pipeline/test_text_strip_web_fragments.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/scripts/__init__.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/scripts/test_audit_outputs.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_atomic_write.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_atomic_write_symlink_race.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_auto_id.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_config.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_config_toml_warn.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_config_walkup.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_content_hash.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_docling_helper.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_frontmatter_helpers.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_frontmatter_key_injection.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_heuristics.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_heuristics_wikipedia_host.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_http_ip_allowlist.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_http_safe.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_http_ssrf_pin.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_http_user_agent.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_log_sanitizer.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_logging_module.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_meta_reserved_keys.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_pdf_complexity.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_pdf_image_symlink.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_safe_dir_name.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_security_error_strings.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_url_creds_scrubbed.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_validators.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_workflow_permissions.py +0 -0
- {any2md-1.1.1 → any2md-1.2.1}/tests/unit/test_zipbomb_guard.py +0 -0
|
@@ -4,7 +4,194 @@ All notable changes to this project will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
6
6
|
|
|
7
|
-
## [
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [1.2.1] — 2026-11-10
|
|
10
|
+
|
|
11
|
+
Security fixes from a full-repository scan (ReDoS, SSRF, symlink
|
|
12
|
+
following, terminal escape injection) and fixes for nested `--meta`,
|
|
13
|
+
text-lane dehyphenation, `--strip-links`, and structured-lane headings.
|
|
14
|
+
|
|
15
|
+
### Security
|
|
16
|
+
|
|
17
|
+
- **Regex denial of service on crafted documents (CWE-1333, CWE-407).**
|
|
18
|
+
Several patterns that run on converted text backtracked in quadratic,
|
|
19
|
+
cubic, or exponential time on hostile input, so a document of a few KB
|
|
20
|
+
to a few MB could hang a conversion for hours and stall the rest of a
|
|
21
|
+
batch. Fixed sites: the academic byline (exponential), the first-H1
|
|
22
|
+
title, TOC-line and TOC-hint detection, body-heading collection, the
|
|
23
|
+
`Contact:` byline check, trailing-whitespace trimming in C5, inline-link
|
|
24
|
+
stripping in C9 and in abstract cleanup, the S1 image and `<figure>`
|
|
25
|
+
handlers, the S4 heading matcher, and a blank-line rescan in the TXT
|
|
26
|
+
structurizer. Each now runs in linear time, through procedural parsing
|
|
27
|
+
or unambiguous, bounded patterns, plus length caps where real input is
|
|
28
|
+
short (TOC lines and author lists, which are cut at a comma rather than
|
|
29
|
+
dropped). Escaped brackets in link text are kept, and backslash parity
|
|
30
|
+
decides whether a bracket opens a link. Behavior on normal
|
|
31
|
+
input is unchanged: a fuzz test compares every rewrite against the
|
|
32
|
+
regex it replaced. Known differences are limited to degenerate input.
|
|
33
|
+
A bare `#` line no longer turns the next line into the H1, empty or
|
|
34
|
+
dots-only TOC titles no longer count, and 2-letter byline names
|
|
35
|
+
written with a space before the comma are no longer matched.
|
|
36
|
+
|
|
37
|
+
- **SSRF guard accepted IPv6 forms that embed an internal IPv4 address
|
|
38
|
+
(CWE-918).** NAT64 (`64:ff9b::/96`), 6to4 (`2002::/16`), SIIT, and on
|
|
39
|
+
some Python builds IPv4-mapped addresses passed the URL guard even when
|
|
40
|
+
the embedded IPv4 was private, loopback, or a cloud metadata address.
|
|
41
|
+
The guard now extracts the embedded address and applies the IPv4 policy,
|
|
42
|
+
with the same verdict on Python 3.11 through 3.13. Teredo, NAT64
|
|
43
|
+
local-use, and deprecated IPv4-compatible addresses are rejected.
|
|
44
|
+
- **DOCX image links could make mammoth read local files or fetch URLs
|
|
45
|
+
(CWE-918, CVE-2025-11849).** The minimum mammoth version is now 1.11.0,
|
|
46
|
+
the mammoth lane passes `external_file_access=False`, and image bytes
|
|
47
|
+
are not read at all unless `--save-images` is set.
|
|
48
|
+
- **Docling treated local paths as URLs (CWE-918).** A file named like
|
|
49
|
+
`http:host#.pdf` was resolved as a URL. Both Docling call sites now pass
|
|
50
|
+
a `Path`, which Docling reads from disk.
|
|
51
|
+
- **Directory scans followed symlinks (CWE-59, CWE-61).** A link such as
|
|
52
|
+
`report.txt -> ~/.ssh/id_rsa` in a scanned tree copied its target into
|
|
53
|
+
the output. Scans now skip symlinked files, never enter symlinked
|
|
54
|
+
directories, and skip FIFOs and device nodes, each with a `SKIP` line.
|
|
55
|
+
A symlink named on the command line still converts. Extension matching
|
|
56
|
+
keeps the platform's case rules. Known limit: a file swapped for a link
|
|
57
|
+
between the scan and the read is not caught.
|
|
58
|
+
- **Terminal escape injection through filenames and error text (CWE-117,
|
|
59
|
+
CWE-150).** Filenames, output paths, and Docling exception messages in
|
|
60
|
+
`OK`, `SKIP`, `FAIL`, `WARN`, and `FALLBACK` lines reached the terminal
|
|
61
|
+
raw, so a crafted name could emit ANSI or OSC sequences, bidi
|
|
62
|
+
overrides, or forge extra status lines. Every such line now goes
|
|
63
|
+
through `safe_oneline`.
|
|
64
|
+
|
|
65
|
+
### Fixed
|
|
66
|
+
|
|
67
|
+
- **Text-lane dehyphenation never ran.** T1 `repair_line_wraps` ran before
|
|
68
|
+
T2 `dehyphenate` and joined `co-\noperation` into `co- operation`, so T2
|
|
69
|
+
never saw a line-end hyphen. T2 now runs first and decides from evidence
|
|
70
|
+
in the same document. It merges the word when the joined form appears
|
|
71
|
+
elsewhere (`cooperation`), keeps the hyphen when the hyphenated form
|
|
72
|
+
appears elsewhere (`co-pilot`), and leaves suspended hyphens
|
|
73
|
+
(`first- and second-order`) and breaks with no evidence as they were.
|
|
74
|
+
It also tolerates trailing spaces after the hyphen, handles non-ASCII
|
|
75
|
+
letters and capitalized words, and no longer rewrites fenced code.
|
|
76
|
+
- **T1 joined setext underlines and rules onto the line above.** A `---`,
|
|
77
|
+
`===`, `***`, or `___` line under text came out as `Heading ---`. These
|
|
78
|
+
lines are now structural, so T1 leaves them alone.
|
|
79
|
+
- **`--strip-links` and `--profile maximum` left links in place.** The
|
|
80
|
+
flag reached `PipelineOptions`, but no stage read it, so link URLs
|
|
81
|
+
always survived. A new shared cleanup stage, C9 `strip_links`, runs
|
|
82
|
+
after entity decoding in both lanes. It replaces `[text](url)` with
|
|
83
|
+
`text` and keeps images, fenced code, and inline code. The unused
|
|
84
|
+
`any2md.utils.strip_links()` helper is removed.
|
|
85
|
+
- **Tables lost their header row on the DOCX mammoth fallback and the
|
|
86
|
+
HTML bs4 fallback.** mammoth never emits `<thead>`, and many web tables
|
|
87
|
+
omit it. markdownify 1.x then wrote an empty header row (`| | |`) and
|
|
88
|
+
demoted the real header to a data row. Both call sites now pass
|
|
89
|
+
`table_infer_header=True`, so the first row stays the header. The
|
|
90
|
+
stale `table_heavy.fallback.md` snapshot is regenerated (the table
|
|
91
|
+
matches the original snapshot again, and two stray blank lines are gone).
|
|
92
|
+
|
|
93
|
+
- **Nested `--meta` lists crashed the batch.** `--meta a.b=x,y` (and any
|
|
94
|
+
list or table below the top level) raised `TypeError` in the
|
|
95
|
+
frontmatter emitter, and the txt, docx, and pdf converters did not catch
|
|
96
|
+
it, so one flag ended the whole run. Nested values now render like
|
|
97
|
+
top-level ones, an empty table renders as `{}`, and an unsupported value
|
|
98
|
+
fails only its own file with a `FAIL:` line.
|
|
99
|
+
- **Structured-lane S4 turned sibling headings into parent and children.**
|
|
100
|
+
`# T`, `### A`, `### B` came out as `## A`, `### B`. S4 now ranks
|
|
101
|
+
headings with the same stack the HTML lane uses, so siblings stay
|
|
102
|
+
siblings and nesting is kept. Both lanes share one block scanner
|
|
103
|
+
that follows CommonMark block rules (fence delimiter character and
|
|
104
|
+
length, indentation and tab stops, list items, paragraph interruption,
|
|
105
|
+
thematic breaks, and raw HTML blocks), so headings inside fenced code
|
|
106
|
+
or raw HTML blocks such as `<pre>` are never rewritten, and real
|
|
107
|
+
headings after them are never skipped.
|
|
108
|
+
|
|
109
|
+
### Changed
|
|
110
|
+
|
|
111
|
+
- **CI runs the full test suite** (`pytest -q`: unit, CLI, and
|
|
112
|
+
integration) instead of `tests/unit` only. `test_flags_compose` now
|
|
113
|
+
asserts the documented pre-flight exit 1 when `--save-images` /
|
|
114
|
+
`--ocr-figures` are given without Docling, instead of exit 0.
|
|
115
|
+
CONTRIBUTING drops a stale test count and a reference to a
|
|
116
|
+
network test file that does not exist.
|
|
117
|
+
|
|
118
|
+
## [1.2.0] — 2026-11-09
|
|
119
|
+
|
|
120
|
+
Drops Python 3.10 to take the Pillow 13 security fixes, and fixes
|
|
121
|
+
missing H1 headings in HTML/URL output.
|
|
122
|
+
|
|
123
|
+
### Removed
|
|
124
|
+
|
|
125
|
+
- **Python 3.10 support.** `requires-python` is now `>=3.11`. Pillow
|
|
126
|
+
13.0.0 is the only release that fixes the advisories below, and it
|
|
127
|
+
requires Python 3.11. Python 3.10 reached end-of-life in October 2026.
|
|
128
|
+
pip on 3.10 keeps resolving any2md 1.1.1, which pins the vulnerable
|
|
129
|
+
pillow 12.x; upgrade the interpreter to get these fixes. The `tomli`
|
|
130
|
+
fallback dependency is gone with it (stdlib `tomllib` covers 3.11+),
|
|
131
|
+
and the CI matrix now tests 3.11, 3.12, and 3.13.
|
|
132
|
+
|
|
133
|
+
### Security
|
|
134
|
+
|
|
135
|
+
- **Pillow 13.0.0.** The `pillow>=12.2.0,<13` range held installs on a
|
|
136
|
+
release with 39 published advisories (e.g. CVE-2026-112395), all fixed
|
|
137
|
+
in 13.0.0. The range is now `pillow>=13.0.0,<14`. Docling's packages
|
|
138
|
+
accept `pillow<14`, so the `high-fidelity` extra still resolves.
|
|
139
|
+
- **Locked dev-container dependencies.** `.devcontainer/requirements.lock`
|
|
140
|
+
and `requirements.txt` move certifi, lxml-html-clean, soupsieve, and
|
|
141
|
+
urllib3 to their fixed releases (lxml follows to 6.1.3). Both
|
|
142
|
+
lockfiles now audit clean with `pip-audit`.
|
|
143
|
+
|
|
144
|
+
### Fixed
|
|
145
|
+
|
|
146
|
+
- **HTML/URL bodies had no H1.** trafilatura keeps a heading only
|
|
147
|
+
when it sits inside the content node it selects. A page `<h1>` in a
|
|
148
|
+
hero banner or `<header>` block outside that node was dropped, so
|
|
149
|
+
nearly every blog page warned `validator: H1 count is 0 (expected 1)`
|
|
150
|
+
(106 of 106 labs.zenity.io posts, 182 of 214 zenity.io/blog posts).
|
|
151
|
+
The HTML converter now restores the H1 before the pipeline runs:
|
|
152
|
+
- The body already has an H1: unchanged.
|
|
153
|
+
- The first body line repeats the title as plain or bold text: that
|
|
154
|
+
line is promoted to the H1, so the title appears once.
|
|
155
|
+
- Otherwise `# <title>` is inserted as the first line.
|
|
156
|
+
- No title is available: unchanged (the validator still warns).
|
|
157
|
+
|
|
158
|
+
The H1 text is the page's own `<h1>` when the page has exactly one
|
|
159
|
+
non-empty `<h1>`, else trafilatura's metadata title. `content_hash`,
|
|
160
|
+
`token_estimate`, `word_count`, `recommended_chunk_level`, and
|
|
161
|
+
`abstract_for_rag` are computed from the body with the H1 in place.
|
|
162
|
+
PDF, DOCX, and TXT output is unchanged.
|
|
163
|
+
- **HTML/URL heading levels skipped below the H1.** Many CMS templates
|
|
164
|
+
mark body sections as H3 or H4 directly under the page H1, so the
|
|
165
|
+
restored H1 surfaced `validator: heading level skip h1 -> h3` (34 of
|
|
166
|
+
the 320 zenity posts above). Headings are now re-ranked by section
|
|
167
|
+
depth below the H1: siblings stay siblings, nesting is kept, and only
|
|
168
|
+
the `#` prefix changes (`H3, H3, H4, H3` becomes `H2, H2, H3, H2`).
|
|
169
|
+
Bodies without skips are byte-identical. All 320 posts now convert
|
|
170
|
+
with 0 validator warnings.
|
|
171
|
+
|
|
172
|
+
### Added
|
|
173
|
+
|
|
174
|
+
- **`seo_title`** frontmatter extension field (HTML/URL only). Holds
|
|
175
|
+
trafilatura's metadata title (`og:title` / `<title>`) when it differs
|
|
176
|
+
from `title`. Omitted when the two match.
|
|
177
|
+
|
|
178
|
+
### Changed
|
|
179
|
+
|
|
180
|
+
- **CI pins ruff to 0.15.22.** Ruff 0.16 and later enable a wider default
|
|
181
|
+
rule set this repo has not adopted, which failed the required lint check
|
|
182
|
+
on an unchanged `main`. The `dev` extra pins the same version.
|
|
183
|
+
- **Release workflow pins `pypa/gh-action-pypi-publish` v1.14.3.** The
|
|
184
|
+
previous pin bundled twine 6.1, which rejects the Metadata-Version 2.5
|
|
185
|
+
that current hatchling writes, so uploads failed before reaching the
|
|
186
|
+
index. The validate step now pins `build==1.6.1` and `twine==7.0.0`.
|
|
187
|
+
- **HTML/URL `title` follows the visible page heading.** `title` is
|
|
188
|
+
still derived from the body H1, and the body H1 is now the page's
|
|
189
|
+
`<h1>`. On pages whose `og:title` differs from their `<h1>`, `title`
|
|
190
|
+
changes from the SEO title to the visible heading. The previous value
|
|
191
|
+
moves to `seo_title`. Pin a title with `--meta title=...` if you
|
|
192
|
+
depend on the old value.
|
|
193
|
+
|
|
194
|
+
## [1.1.1] — 2026-05-07
|
|
8
195
|
|
|
9
196
|
Security patch release. Closes 10 findings from a comprehensive scan
|
|
10
197
|
(bandit, snyk-code, gitleaks, pip-audit, trivy, manual review) plus
|
|
@@ -29,11 +29,12 @@ the test suite gates Docling-dependent tests with
|
|
|
29
29
|
`@pytest.mark.skipif(not has_docling(), …)` so they're transparently skipped
|
|
30
30
|
without it. Coverage of fallback paths is unaffected.
|
|
31
31
|
|
|
32
|
-
Python 3.
|
|
32
|
+
Python 3.11 or later is required.
|
|
33
33
|
|
|
34
34
|
## Running tests
|
|
35
35
|
|
|
36
|
-
The default invocation runs the whole suite
|
|
36
|
+
The default invocation runs the whole suite. CI runs the same command on
|
|
37
|
+
Python 3.11, 3.12, and 3.13:
|
|
37
38
|
|
|
38
39
|
```bash
|
|
39
40
|
pytest -v
|
|
@@ -59,13 +60,11 @@ UPDATE_SNAPSHOTS=1 pytest tests/integration/test_snapshots.py
|
|
|
59
60
|
Then inspect the diff (`git diff tests/fixtures/snapshots/`) before
|
|
60
61
|
committing — the snapshots are review material, not just bytes.
|
|
61
62
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
pytest tests/integration/test_url_wikipedia.py -v
|
|
68
|
-
```
|
|
63
|
+
The suite makes no network calls. URL and SSRF behavior is tested with
|
|
64
|
+
stubbed DNS and a fake opener (`tests/unit/test_http_safe.py`,
|
|
65
|
+
`tests/integration/test_html_trafilatura.py`), and HTML conversion with
|
|
66
|
+
saved fixtures under `tests/fixtures/docs/`. When you change
|
|
67
|
+
`converters/html.py`, also spot-check a few live pages by hand.
|
|
69
68
|
|
|
70
69
|
## Adding a new converter
|
|
71
70
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: any2md
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.1
|
|
4
4
|
Summary: Convert PDF, DOCX, HTML, and TXT files — or web pages by URL — to clean, LLM-optimized Markdown with YAML frontmatter.
|
|
5
5
|
Project-URL: Homepage, https://github.com/rocklambros/any2md
|
|
6
6
|
Project-URL: Issues, https://github.com/rocklambros/any2md/issues
|
|
@@ -11,16 +11,15 @@ Classifier: License :: OSI Approved :: MIT License
|
|
|
11
11
|
Classifier: Operating System :: OS Independent
|
|
12
12
|
Classifier: Programming Language :: Python :: 3
|
|
13
13
|
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
14
|
-
Requires-Python: >=3.
|
|
14
|
+
Requires-Python: >=3.11
|
|
15
15
|
Requires-Dist: beautifulsoup4<5,>=4.12.0
|
|
16
16
|
Requires-Dist: defusedxml<1,>=0.7.1
|
|
17
17
|
Requires-Dist: lxml<7,>=6.1.0
|
|
18
|
-
Requires-Dist: mammoth<2,>=1.
|
|
18
|
+
Requires-Dist: mammoth<2,>=1.11.0
|
|
19
19
|
Requires-Dist: markdownify<2,>=0.13.0
|
|
20
|
-
Requires-Dist: pillow<13
|
|
20
|
+
Requires-Dist: pillow<14,>=13.0.0
|
|
21
21
|
Requires-Dist: pymupdf4llm<1,>=0.0.17
|
|
22
22
|
Requires-Dist: pymupdf<2,>=1.24.0
|
|
23
|
-
Requires-Dist: tomli>=2.0.0; python_version < '3.11'
|
|
24
23
|
Requires-Dist: trafilatura<3,>=1.12.0
|
|
25
24
|
Requires-Dist: urllib3<3,>=2.6.3
|
|
26
25
|
Provides-Extra: dev
|
|
@@ -28,7 +27,7 @@ Requires-Dist: pytest-snapshot>=0.9.0; extra == 'dev'
|
|
|
28
27
|
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
29
28
|
Requires-Dist: pyyaml>=6.0; extra == 'dev'
|
|
30
29
|
Requires-Dist: reportlab>=4.0.0; extra == 'dev'
|
|
31
|
-
Requires-Dist: ruff
|
|
30
|
+
Requires-Dist: ruff==0.15.22; extra == 'dev'
|
|
32
31
|
Provides-Extra: high-fidelity
|
|
33
32
|
Requires-Dist: docling>=2.0.0; extra == 'high-fidelity'
|
|
34
33
|
Description-Content-Type: text/markdown
|
|
@@ -517,8 +516,8 @@ any2md uses a two-lane post-processing pipeline: one lane for layout-trustworthy
|
|
|
517
516
|
│
|
|
518
517
|
┌────────────┴────────────┐
|
|
519
518
|
│ pipeline/text.py │ text lane
|
|
520
|
-
│ T1 line-wrap repair │
|
|
521
519
|
│ T2 dehyphenate │
|
|
520
|
+
│ T1 line-wrap repair │
|
|
522
521
|
│ T9 strip repeated │ ◀ NEW (E1)
|
|
523
522
|
│ byline │
|
|
524
523
|
│ T3 dedupe paragraphs │
|
|
@@ -4,9 +4,8 @@
|
|
|
4
4
|
|
|
5
5
|
| Version | Status |
|
|
6
6
|
|---|---|
|
|
7
|
-
| 1.
|
|
8
|
-
|
|
|
9
|
-
| < 0.7 | ❌ Not supported. Please upgrade. |
|
|
7
|
+
| 1.2.x | ✅ Supported (current stable). Patch releases for security and quality fixes are published as needed. |
|
|
8
|
+
| < 1.2 | ❌ Not supported. Please upgrade to the latest 1.2.x release. |
|
|
10
9
|
|
|
11
10
|
## Reporting a vulnerability
|
|
12
11
|
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
Defenses against:
|
|
4
4
|
- non-http(s) schemes
|
|
5
5
|
- URLs resolving to non-public addresses (private, multicast, CGNAT, etc.)
|
|
6
|
+
- IPv6 forms that embed an internal IPv4 target (IPv4-mapped, NAT64,
|
|
7
|
+
6to4, Teredo, IPv4-compatible)
|
|
6
8
|
- DNS rebinding via redirect (per-hop revalidation)
|
|
7
9
|
- DNS rebinding INTRA-hop (IP pinned through to socket.create_connection,
|
|
8
10
|
so the IP we validated is the IP we connect to)
|
|
@@ -28,8 +30,57 @@ _MAX_RESPONSE_BYTES = 20 * 1024 * 1024 # 20 MB cap on body read
|
|
|
28
30
|
_USER_AGENT = f"any2md/{__version__}"
|
|
29
31
|
|
|
30
32
|
|
|
33
|
+
# Shared address space (RFC 6598). Cloud providers host metadata services
|
|
34
|
+
# here, for example Alibaba's 100.100.100.200. ``is_global`` already
|
|
35
|
+
# excludes it, but the explicit check keeps the rule visible and stable.
|
|
36
|
+
_CGNAT_V4 = ipaddress.IPv4Network("100.64.0.0/10")
|
|
37
|
+
|
|
38
|
+
# IPv6 prefixes whose low 32 bits carry an IPv4 destination. A socket or
|
|
39
|
+
# translator delivers the packet to that IPv4 address, so the IPv4 policy
|
|
40
|
+
# decides. Python's ``is_global`` does not unwrap NAT64 or IPv4-translated
|
|
41
|
+
# forms on any version, and unwraps IPv4-mapped forms only on builds with
|
|
42
|
+
# the CVE-2024-4032 fix, so the guard extracts the address itself.
|
|
43
|
+
_V4_IN_LOW_32 = (
|
|
44
|
+
ipaddress.IPv6Network("::ffff:0:0/96"), # IPv4-mapped
|
|
45
|
+
ipaddress.IPv6Network("::ffff:0:0:0/96"), # SIIT IPv4-translated
|
|
46
|
+
ipaddress.IPv6Network("64:ff9b::/96"), # NAT64 well-known prefix
|
|
47
|
+
)
|
|
48
|
+
_SIXTOFOUR = ipaddress.IPv6Network("2002::/16")
|
|
49
|
+
|
|
50
|
+
# Rejected outright. Teredo (2001::/32) tunnels to an obfuscated IPv4
|
|
51
|
+
# client through a relay. NAT64 local-use (64:ff9b:1::/48) maps into a
|
|
52
|
+
# site's own IPv4 space by definition. IPv4-compatible (::/96) is
|
|
53
|
+
# deprecated by RFC 4291 and has no legitimate use in a DNS answer.
|
|
54
|
+
_V6_DENY = (
|
|
55
|
+
ipaddress.IPv6Network("2001::/32"),
|
|
56
|
+
ipaddress.IPv6Network("64:ff9b:1::/48"),
|
|
57
|
+
ipaddress.IPv6Network("::/96"),
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _v4_allowed(addr: ipaddress.IPv4Address) -> bool:
|
|
62
|
+
return (
|
|
63
|
+
bool(addr.is_global)
|
|
64
|
+
and addr not in _CGNAT_V4
|
|
65
|
+
and not addr.is_multicast
|
|
66
|
+
and not addr.is_unspecified
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
31
70
|
def _addr_allowed(addr) -> bool:
|
|
32
|
-
"""Allowlist policy: globally routable, non-multicast, non-unspecified.
|
|
71
|
+
"""Allowlist policy: globally routable, non-multicast, non-unspecified.
|
|
72
|
+
|
|
73
|
+
IPv6 addresses that embed an IPv4 destination are judged by the IPv4
|
|
74
|
+
policy alone, so the verdict does not vary with the Python version.
|
|
75
|
+
"""
|
|
76
|
+
if addr.version == 4:
|
|
77
|
+
return _v4_allowed(addr)
|
|
78
|
+
if any(addr in net for net in _V6_DENY):
|
|
79
|
+
return False
|
|
80
|
+
if any(addr in net for net in _V4_IN_LOW_32):
|
|
81
|
+
return _v4_allowed(ipaddress.IPv4Address(int(addr) & 0xFFFFFFFF))
|
|
82
|
+
if addr in _SIXTOFOUR:
|
|
83
|
+
return _v4_allowed(ipaddress.IPv4Address((int(addr) >> 80) & 0xFFFFFFFF))
|
|
33
84
|
return bool(addr.is_global) and not addr.is_multicast and not addr.is_unspecified
|
|
34
85
|
|
|
35
86
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""CLI entry point for any2md."""
|
|
2
2
|
|
|
3
3
|
import argparse
|
|
4
|
+
import os
|
|
4
5
|
import sys
|
|
5
6
|
import time
|
|
6
7
|
from pathlib import Path
|
|
@@ -74,6 +75,63 @@ def _deep_merge_overrides(
|
|
|
74
75
|
return out
|
|
75
76
|
|
|
76
77
|
|
|
78
|
+
def _scan_directory(root: Path, recursive: bool) -> list[Path]:
|
|
79
|
+
"""Return supported regular files under ``root``, sorted.
|
|
80
|
+
|
|
81
|
+
Never follows symlinks below ``root``. An untrusted document tree can
|
|
82
|
+
hold a link such as ``report.txt -> ~/.ssh/id_rsa``, and following it
|
|
83
|
+
would copy the target into the output (F14 CWE-59, F22 CWE-61).
|
|
84
|
+
``Path.glob`` and ``Path.rglob`` return file links, and their handling
|
|
85
|
+
of directory links has changed across Python releases, so the walk
|
|
86
|
+
uses ``os.scandir`` with ``follow_symlinks=False`` to get one answer
|
|
87
|
+
on every supported version. Skipping directory links also rules out
|
|
88
|
+
symlink loops. ``root`` itself is followed when it is a link,
|
|
89
|
+
because the user named it.
|
|
90
|
+
|
|
91
|
+
Suffixes are compared after ``os.path.normcase``, which matches the
|
|
92
|
+
old glob semantics: case-insensitive on Windows, case-sensitive on
|
|
93
|
+
POSIX.
|
|
94
|
+
"""
|
|
95
|
+
suffixes = tuple(os.path.normcase(ext) for ext in SUPPORTED_EXTENSIONS)
|
|
96
|
+
found: list[Path] = []
|
|
97
|
+
# An explicit stack instead of recursion keeps deep trees clear of
|
|
98
|
+
# the interpreter recursion limit.
|
|
99
|
+
pending = [root]
|
|
100
|
+
while pending:
|
|
101
|
+
current = pending.pop()
|
|
102
|
+
try:
|
|
103
|
+
with os.scandir(current) as entries:
|
|
104
|
+
children = list(entries)
|
|
105
|
+
except OSError as e:
|
|
106
|
+
_logging.warn(f"{current} -- {e}", prefix="SKIP (unreadable)")
|
|
107
|
+
continue
|
|
108
|
+
for entry in children:
|
|
109
|
+
path = Path(entry.path)
|
|
110
|
+
try:
|
|
111
|
+
if entry.is_symlink():
|
|
112
|
+
# Report only links the old scan would have picked up,
|
|
113
|
+
# so unrelated links in a tree do not flood stderr.
|
|
114
|
+
wanted = os.path.normcase(entry.name).endswith(suffixes)
|
|
115
|
+
if wanted or (recursive and entry.is_dir()):
|
|
116
|
+
_logging.warn(str(path), prefix="SKIP (symlink)")
|
|
117
|
+
continue
|
|
118
|
+
if entry.is_dir(follow_symlinks=False):
|
|
119
|
+
if recursive:
|
|
120
|
+
pending.append(path)
|
|
121
|
+
continue
|
|
122
|
+
if not os.path.normcase(entry.name).endswith(suffixes):
|
|
123
|
+
continue
|
|
124
|
+
# FIFOs and device nodes can hang the read or never end.
|
|
125
|
+
if not entry.is_file(follow_symlinks=False):
|
|
126
|
+
_logging.warn(str(path), prefix="SKIP (not a regular file)")
|
|
127
|
+
continue
|
|
128
|
+
except OSError as e:
|
|
129
|
+
_logging.warn(f"{path} -- {e}", prefix="SKIP (unreadable)")
|
|
130
|
+
continue
|
|
131
|
+
found.append(path)
|
|
132
|
+
return sorted(found)
|
|
133
|
+
|
|
134
|
+
|
|
77
135
|
def main():
|
|
78
136
|
script_dir = Path.cwd()
|
|
79
137
|
default_output_dir = script_dir / "Text"
|
|
@@ -325,14 +383,7 @@ def main():
|
|
|
325
383
|
_logging.fail(f"{f}", prefix="NOT FOUND")
|
|
326
384
|
continue
|
|
327
385
|
if p.is_dir():
|
|
328
|
-
|
|
329
|
-
file_paths.extend(
|
|
330
|
-
sorted(
|
|
331
|
-
fp
|
|
332
|
-
for ext in SUPPORTED_EXTENSIONS
|
|
333
|
-
for fp in glob_method(f"*{ext}")
|
|
334
|
-
)
|
|
335
|
-
)
|
|
386
|
+
file_paths.extend(_scan_directory(p, args.recursive))
|
|
336
387
|
continue
|
|
337
388
|
if p.suffix.lower() not in SUPPORTED_EXTENSIONS:
|
|
338
389
|
_logging.fail(f"{f}", prefix="UNSUPPORTED FORMAT")
|
|
@@ -344,22 +395,18 @@ def main():
|
|
|
344
395
|
f"Error: not a directory: {_logging.safe_oneline(str(args.input_dir))}\n"
|
|
345
396
|
)
|
|
346
397
|
sys.exit(1)
|
|
347
|
-
|
|
348
|
-
file_paths = sorted(
|
|
349
|
-
p for ext in SUPPORTED_EXTENSIONS for p in glob_method(f"*{ext}")
|
|
350
|
-
)
|
|
398
|
+
file_paths = _scan_directory(args.input_dir, args.recursive)
|
|
351
399
|
else:
|
|
352
|
-
|
|
353
|
-
file_paths = sorted(
|
|
354
|
-
p for ext in SUPPORTED_EXTENSIONS for p in glob_method(f"*{ext}")
|
|
355
|
-
)
|
|
400
|
+
file_paths = _scan_directory(script_dir, args.recursive)
|
|
356
401
|
|
|
357
402
|
if not file_paths and not urls:
|
|
358
403
|
print("No supported files to process.")
|
|
359
404
|
sys.exit(0)
|
|
360
405
|
|
|
361
406
|
total = len(file_paths) + len(urls)
|
|
362
|
-
print(
|
|
407
|
+
print(
|
|
408
|
+
f"Processing {total} file(s) → {_logging.safe_oneline(str(args.output_dir))}/\n"
|
|
409
|
+
)
|
|
363
410
|
start = time.time()
|
|
364
411
|
ok = 0
|
|
365
412
|
fail = 0
|
|
@@ -370,7 +417,7 @@ def main():
|
|
|
370
417
|
out_name = url_to_filename(url)
|
|
371
418
|
out_path = args.output_dir / out_name
|
|
372
419
|
if out_path.exists() and not args.force:
|
|
373
|
-
print(f" SKIP (exists): {out_name}")
|
|
420
|
+
print(f" SKIP (exists): {_logging.safe_oneline(out_name)}")
|
|
374
421
|
skip += 1
|
|
375
422
|
continue
|
|
376
423
|
|
|
@@ -387,10 +434,13 @@ def main():
|
|
|
387
434
|
|
|
388
435
|
# Process local files
|
|
389
436
|
for file_path in file_paths:
|
|
437
|
+
# sanitize_filename strips only C0 and DEL, so C1 controls and bidi
|
|
438
|
+
# overrides survive into out_name. Every status line that prints it
|
|
439
|
+
# goes through safe_oneline as well.
|
|
390
440
|
out_name = sanitize_filename(file_path.name)
|
|
391
441
|
out_path = args.output_dir / out_name
|
|
392
442
|
if out_path.exists() and not args.force:
|
|
393
|
-
print(f" SKIP (exists): {out_name}")
|
|
443
|
+
print(f" SKIP (exists): {_logging.safe_oneline(out_name)}")
|
|
394
444
|
skip += 1
|
|
395
445
|
continue
|
|
396
446
|
|
|
@@ -403,9 +453,9 @@ def main():
|
|
|
403
453
|
continue
|
|
404
454
|
|
|
405
455
|
if file_size > args.max_file_size:
|
|
406
|
-
|
|
407
|
-
f"
|
|
408
|
-
|
|
456
|
+
_logging.warn(
|
|
457
|
+
f"{file_path.name} ({file_size} bytes, max {args.max_file_size})",
|
|
458
|
+
prefix="SKIP (too large)",
|
|
409
459
|
)
|
|
410
460
|
skip += 1
|
|
411
461
|
continue
|
|
@@ -5,28 +5,17 @@ is treated as frontmatter overrides (deep-merged below ``--meta-file``
|
|
|
5
5
|
and CLI ``--meta`` arguments). The ``[document_id]`` table provides
|
|
6
6
|
``publisher_prefix`` / ``type_code`` overrides for ``--auto-id``.
|
|
7
7
|
|
|
8
|
-
TOML parsing uses stdlib ``tomllib``
|
|
9
|
-
interpreters we fall back to the optional ``tomli`` package; if neither
|
|
10
|
-
is importable, ``load_toml`` returns ``{}`` and config is silently
|
|
11
|
-
disabled (no hard error — config is opt-in convenience).
|
|
8
|
+
TOML parsing uses stdlib ``tomllib`` (any2md requires Python 3.11+).
|
|
12
9
|
"""
|
|
13
10
|
|
|
14
11
|
from __future__ import annotations
|
|
15
12
|
|
|
16
|
-
import
|
|
13
|
+
import tomllib
|
|
17
14
|
from pathlib import Path
|
|
18
15
|
from typing import Any
|
|
19
16
|
|
|
20
17
|
from any2md import _logging
|
|
21
18
|
|
|
22
|
-
if sys.version_info >= (3, 11):
|
|
23
|
-
import tomllib
|
|
24
|
-
else: # pragma: no cover - exercised only on Python 3.10
|
|
25
|
-
try:
|
|
26
|
-
import tomli as tomllib # type: ignore[import-not-found,no-redef]
|
|
27
|
-
except ImportError:
|
|
28
|
-
tomllib = None # type: ignore[assignment]
|
|
29
|
-
|
|
30
19
|
|
|
31
20
|
def load_toml(path: Path) -> dict[str, Any]:
|
|
32
21
|
"""Load a TOML file. Returns ``{}`` on any failure.
|
|
@@ -35,8 +24,6 @@ def load_toml(path: Path) -> dict[str, Any]:
|
|
|
35
24
|
not block conversion. Callers that need strict validation should
|
|
36
25
|
parse the file themselves.
|
|
37
26
|
"""
|
|
38
|
-
if tomllib is None: # pragma: no cover - 3.10 without tomli
|
|
39
|
-
return {}
|
|
40
27
|
try:
|
|
41
28
|
with open(path, "rb") as f:
|
|
42
29
|
return tomllib.load(f)
|