html-to-markdown-cli 3.23.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,323 @@
1
+ # Claude Code local state
2
+ .claude/
3
+
4
+ # Go FFI download cache (populated by `go generate ./...` post-install)
5
+ packages/go/.lib/
6
+ packages/go/**/.lib/
7
+ test_apps/go/.lib/
8
+ test_apps/go/**/.lib/
9
+
10
+ # Python
11
+ __pycache__/
12
+ *.py[cod]
13
+ *$py.class
14
+ *.pyd
15
+ .Python
16
+ develop-eggs/
17
+ downloads/
18
+ eggs/
19
+ .eggs/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ *.egg-info/
24
+ .installed.cfg
25
+ *.egg
26
+ .pytest_cache/
27
+ .ruff_cache/
28
+ htmlcov/
29
+ .coverage
30
+ .coverage.*
31
+ coverage.lcov
32
+
33
+ # Rust
34
+ target/
35
+ rust-coverage.lcov
36
+ examples/**/.cargo/config.toml
37
+
38
+ # Node.js / TypeScript
39
+ **/node_modules/
40
+ dist/
41
+ dist-node/
42
+ dist-web/
43
+ *.tsbuildinfo
44
+ .pnpm-debug.log
45
+
46
+ # C / C++ build outputs
47
+ *.so
48
+ *.o
49
+ *.dSYM/
50
+ *.h.bak
51
+ *.node
52
+
53
+ # Ruby
54
+ .yardoc/
55
+ !packages/ruby/lib/
56
+ !packages/ruby/lib/**/*.rb
57
+ packages/ruby/lib/bin/
58
+ packages/ruby/lib/**/*.bundle
59
+ packages/ruby/ext/**/*.bundle
60
+ packages/ruby/ext/**/mkmf.log
61
+ packages/ruby/ext/**/Makefile
62
+ packages/ruby/tmp/
63
+ packages/ruby/.bundle/
64
+ packages/ruby/vendor/html-to-markdown-rs/
65
+ packages/ruby/vendor/Cargo.toml
66
+ *.gem
67
+ .gems/
68
+
69
+ # PHP
70
+ packages/php-ext/workspace/
71
+ packages/php/.php-cs-fixer.cache
72
+
73
+ # R
74
+ !packages/r/R/
75
+ !packages/r/R/**/*.R
76
+ packages/r/src/*.o
77
+ packages/r/src/*.so
78
+ packages/r/src/*.dll
79
+ packages/r/src/*.dylib
80
+ packages/r/*.tar.gz
81
+ packages/r/src/rust/html-to-markdown/
82
+
83
+ # Elixir
84
+ !packages/elixir/lib/
85
+ !packages/elixir/lib/**/*.ex
86
+ erl_crash.dump
87
+ e2e/elixir/test_convert_debug.exs
88
+ e2e/elixir/test_convert_with_metadata.exs
89
+ e2e/elixir/test_visitor_debug.exs
90
+ e2e/elixir/deps/
91
+ e2e/elixir/_build/
92
+ e2e/elixir/.elixir_ls/
93
+ e2e/elixir/.fetch
94
+ e2e/elixir/erl_crash.dump
95
+ test_apps/elixir/deps/
96
+ test_apps/elixir/_build/
97
+
98
+ # C# / .NET
99
+ packages/csharp/**/bin/
100
+ test_apps/csharp/**/bin/
101
+ e2e/csharp/**/bin/
102
+ obj/
103
+ *.dll
104
+ *.exe
105
+ *.pdb
106
+ *.nupkg
107
+ packages/csharp/HtmlToMarkdown/runtimes/
108
+
109
+ # Java
110
+ *.class
111
+ packages/java/src/main/resources/natives/
112
+
113
+ # Kotlin Android JNI test libs (built locally / in CI, never committed)
114
+ e2e/kotlin_android/src/test/jniLibs/
115
+
116
+ # Dart
117
+ e2e/dart/libhtml_to_markdown_rs_dart.dylib
118
+ e2e/dart/libhtml_to_markdown_rs_dart.so
119
+ e2e/dart/html_to_markdown_rs_dart.dll
120
+ packages/dart/lib/src/native/
121
+
122
+ # Java test_app native libraries (built/downloaded per platform at test time)
123
+ test_apps/java/src/main/resources/natives/
124
+ packages/dart/rust/target/
125
+
126
+ # C FFI test binaries
127
+ crates/html-to-markdown-ffi/tests/c/test_*
128
+ !crates/html-to-markdown-ffi/tests/c/test_*.c
129
+ crates/html-to-markdown-ffi/tests/c/*.o
130
+ crates/html-to-markdown-ffi/tests/c/*.dSYM/
131
+
132
+ # C e2e test binaries
133
+ e2e/c/run_tests
134
+ e2e/c/test_*
135
+ !e2e/c/test_*.c
136
+ !e2e/c/test_runner.h
137
+ e2e/c/*.o
138
+ e2e/c/*.dSYM/
139
+ test_apps/c/run_tests
140
+ test_apps/c/test_*
141
+ !test_apps/c/test_*.c
142
+ !test_apps/c/test_runner.h
143
+ test_apps/c/*.o
144
+ test_apps/c/*.dSYM/
145
+ # Download cache (populated by download_ffi.sh at smoke time).
146
+ test_apps/c/ffi/
147
+
148
+ # Vendor / package managers
149
+ **/vendor/
150
+ **/vendor/bundle/
151
+ pkg/
152
+
153
+ # IDEs & editors
154
+ .vscode/
155
+ .idea/
156
+ *.swp
157
+ *.swo
158
+ *~
159
+ .cursorrules
160
+ .windsurfrules
161
+
162
+ # Environment & virtualenvs
163
+ .env
164
+ .env.local
165
+ **/.venv/
166
+
167
+ # MkDocs
168
+ site/
169
+
170
+ # OS
171
+ .DS_Store
172
+ Thumbs.db
173
+
174
+ # Benchmarks
175
+ .benchmarks/
176
+ benchmark-harness-results-*/
177
+ tools/runtime-bench/results/
178
+ tools/benchmark-harness/results/
179
+ tools/benchmark-harness/artifacts/
180
+ tools/benchmark-harness/artifacts-*/
181
+ tools/benchmark-harness/results-consolidated/
182
+ tools/benchmark-harness/results-local-*/
183
+ artifacts/
184
+
185
+ # Cache & temp
186
+ *.cache
187
+ .cache/
188
+ .tmp/
189
+ .wrangler/
190
+ .rumdl_cache/
191
+ [Tt][Oo][Dd][Oo]*
192
+
193
+ # Generated artifacts
194
+ .remote-cache/
195
+ .alef/
196
+ .gemini/
197
+ GEMINI.md
198
+ *.test
199
+
200
+ # Go benchmarks
201
+ !packages/go/v2/bin/
202
+ !packages/go/v2/bin/benchmark.go
203
+
204
+ .venv/
205
+ node_modules/
206
+ vendor/bundle/
207
+ vendor/
208
+
209
+
210
+ # Build caches (added by docs-quality sweep)
211
+ **/.zig-cache/
212
+ **/.zig-global-cache/
213
+ **/.dart_tool/
214
+ **/.build/
215
+ test_apps/zig/zig-pkg/
216
+ e2e/kotlin_android/build/
217
+ test_apps/kotlin_android/build/
218
+ packages/kotlin-android/build/
219
+ **/.gradle/
220
+
221
+ # Swift / SwiftPM build cache
222
+ .build/
223
+ .swiftpm/
224
+ Package.resolved
225
+ *.xcodeproj/
226
+ *.xcworkspace/
227
+ DerivedData/
228
+
229
+ **/logs/
230
+ *.log
231
+ test_apps/**/test_output.log
232
+
233
+ # basemind code-map index cache (per-developer)
234
+ .basemind/
235
+
236
+ # polylint result cache (per-developer)
237
+ .polylint/
238
+
239
+
240
+ # Alef-generated skills are source-controlled; other agent-local state stays ignored.
241
+ !.codex/
242
+ .codex/*
243
+ !.codex/skills/
244
+ .codex/skills/*
245
+ !.codex/skills/api/
246
+ !.codex/skills/api/SKILL.md
247
+ !.codex/skills/cli/
248
+ !.codex/skills/cli/SKILL.md
249
+ !.codex/skills/mcp/
250
+ !.codex/skills/mcp/SKILL.md
251
+ !.agents/
252
+ .agents/*
253
+ !.agents/skills/
254
+ .agents/skills/*
255
+ !.agents/skills/api/
256
+ !.agents/skills/api/SKILL.md
257
+ !.agents/skills/cli/
258
+ !.agents/skills/cli/SKILL.md
259
+ !.agents/skills/mcp/
260
+ !.agents/skills/mcp/SKILL.md
261
+ !.agents/plugins/
262
+ !.agents/plugins/marketplace.json
263
+ !.claude/
264
+ .claude/*
265
+ !.claude/skills/
266
+ .claude/skills/*
267
+ !.claude/skills/api/
268
+ !.claude/skills/api/SKILL.md
269
+ !.claude/skills/cli/
270
+ !.claude/skills/cli/SKILL.md
271
+ !.claude/skills/mcp/
272
+ !.claude/skills/mcp/SKILL.md
273
+ !.github/skills/
274
+ .github/skills/*
275
+ !.github/skills/api/
276
+ !.github/skills/api/SKILL.md
277
+ !.github/skills/cli/
278
+ !.github/skills/cli/SKILL.md
279
+ !.github/skills/mcp/
280
+ !.github/skills/mcp/SKILL.md
281
+
282
+
283
+ # Astro Starlight docs-site build output
284
+ docs-site/dist/
285
+ docs-site/.astro/
286
+ docs-site/node_modules/
287
+
288
+ e2e/java/src/main/resources/natives/
289
+ poly.local.toml
290
+
291
+ # Alef snippet validator scratch workspaces and caches ~keep
292
+ **/.alef-snippet-*/
293
+ **/.alef-cache/
294
+ **/.alef/
295
+
296
+ bin/
297
+
298
+ # BEGIN ai-rulez (DO NOT EDIT - managed by ai-rulez)
299
+ .ai-rulez/.config.local.*
300
+ .opencode/agents/
301
+ .opencode/commands/
302
+ /.mcp.json
303
+ /AGENTS.md
304
+ /CLAUDE.md
305
+ /opencode.json
306
+ # END ai-rulez
307
+
308
+ # Agent harness output outside the ai-rulez managed block: the machine-local overlay,
309
+ # plugin-root manifests, and presets this repository does not enable.
310
+ .ai-rulez/config.local.*
311
+ .ai-rulez/local/
312
+ **/.ai-rulez/.generated-manifest.local.json
313
+ plugin/.ai-rulez/.generated-manifest.json
314
+ .agents/settings.json
315
+ .claude/settings.json
316
+ .claude/plugins.json
317
+ .codex/commands/
318
+ .codex/plugins.json
319
+ .cursor/
320
+ .github/commands/
321
+
322
+ # Output of the rustc stdin lint probes xberg-io/actions/setup-rust runs in the checkout
323
+ /rust_out
@@ -0,0 +1,29 @@
1
+ Metadata-Version: 2.5
2
+ Name: html-to-markdown-cli
3
+ Version: 3.23.1
4
+ Summary: CLI proxy for html-to-markdown — downloads and runs the native html-to-markdown binary from GitHub releases.
5
+ Project-URL: Homepage, https://github.com/xberg-io/html-to-markdown
6
+ Project-URL: Issues, https://github.com/xberg-io/html-to-markdown/issues
7
+ Project-URL: Repository, https://github.com/xberg-io/html-to-markdown.git
8
+ Author-email: Na'aman Hirschfeld <naaman@xberg.io>, Tobias Perlstein <tobias@xberg.io>
9
+ Maintainer-email: Na'aman Hirschfeld <naaman@xberg.io>, Tobias Perlstein <tobias@xberg.io>
10
+ License: MIT
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3 :: Only
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Programming Language :: Python :: 3.14
20
+ Requires-Python: >=3.11
21
+ Description-Content-Type: text/markdown
22
+
23
+ # html-to-markdown-cli
24
+
25
+ CLI proxy for [`html-to-markdown`](https://github.com/xberg-io/html-to-markdown). Installing
26
+ this package provides a `html-to-markdown` command that downloads the matching native
27
+ binary from GitHub releases and runs it.
28
+
29
+ Pin a specific release tag with the `HTML_TO_MARKDOWN_CLI_VERSION` environment variable.
@@ -0,0 +1,7 @@
1
+ # html-to-markdown-cli
2
+
3
+ CLI proxy for [`html-to-markdown`](https://github.com/xberg-io/html-to-markdown). Installing
4
+ this package provides a `html-to-markdown` command that downloads the matching native
5
+ binary from GitHub releases and runs it.
6
+
7
+ Pin a specific release tag with the `HTML_TO_MARKDOWN_CLI_VERSION` environment variable.
@@ -0,0 +1,136 @@
1
+ """Custom hatchling build hook that bundles the native html-to-markdown binary into a wheel.
2
+
3
+ When building a platform-specific wheel in CI, the target triple is supplied via
4
+ the ``HTML_TO_MARKDOWN_CLI_TARGET`` env var (the build host is always linux/amd64, so
5
+ the triple cannot be inferred from ``platform.*``). The matching
6
+ ``cli-<target>.tar.gz`` / ``.zip`` release archive is located (repo root or ``dist/``),
7
+ the binary is extracted into ``html_to_markdown_cli/bin/<target>/``, force-included in
8
+ the wheel, and the wheel is tagged for that platform so PyPI serves the right artifact.
9
+
10
+ If no target is set (e.g. the sdist build), the hook is a no-op and the package falls
11
+ back to the runtime downloader in ``html_to_markdown_cli/downloader.py`` (see ``cli.py``).
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import os
17
+ import shutil
18
+ import tarfile
19
+ import zipfile
20
+ from pathlib import Path
21
+
22
+ from hatchling.builders.hooks.plugin.interface import BuildHookInterface
23
+
24
+ _TAG_MAP = {
25
+ "x86_64-pc-windows-msvc": "win_amd64",
26
+ "x86_64-unknown-linux-gnu": "manylinux_2_28_x86_64",
27
+ "aarch64-unknown-linux-gnu": "manylinux_2_28_aarch64",
28
+ "x86_64-unknown-linux-musl": "musllinux_1_2_x86_64",
29
+ "aarch64-unknown-linux-musl": "musllinux_1_2_aarch64",
30
+ "aarch64-apple-darwin": "macosx_15_0_arm64",
31
+ "x86_64-apple-darwin": "macosx_15_0_x86_64",
32
+ }
33
+
34
+
35
+ def _safe_destination(root: Path, member_name: str) -> Path:
36
+ """Resolve an archive member path and require it to stay under root."""
37
+ root = root.resolve()
38
+ target = (root / member_name.replace("\\", "/")).resolve(strict=False)
39
+ if target != root and not target.is_relative_to(root):
40
+ raise RuntimeError(f"archive member escapes extraction directory: {member_name}")
41
+ return target
42
+
43
+
44
+ def _extract_zip_bounded(archive: Path, extract_dir: Path) -> None:
45
+ """Extract zip entries after bounding each destination path."""
46
+ with zipfile.ZipFile(archive) as zf:
47
+ for member in zf.infolist():
48
+ target = _safe_destination(extract_dir, member.filename)
49
+ if member.is_dir():
50
+ target.mkdir(parents=True, exist_ok=True)
51
+ continue
52
+
53
+ target.parent.mkdir(parents=True, exist_ok=True)
54
+ with zf.open(member) as source, target.open("wb") as destination:
55
+ shutil.copyfileobj(source, destination)
56
+
57
+
58
+ def _extract_tar_bounded(archive: Path, extract_dir: Path) -> None:
59
+ """Extract regular tar entries after bounding each destination path."""
60
+ with tarfile.open(archive, "r:gz") as tf:
61
+ for member in tf.getmembers():
62
+ target = _safe_destination(extract_dir, member.name)
63
+ if member.isdir():
64
+ target.mkdir(parents=True, exist_ok=True)
65
+ continue
66
+ if not member.isfile():
67
+ raise RuntimeError(f"unsupported archive member type: {member.name}")
68
+
69
+ target.parent.mkdir(parents=True, exist_ok=True)
70
+ source = tf.extractfile(member)
71
+ if source is None:
72
+ raise RuntimeError(f"could not read archive member: {member.name}")
73
+ with source, target.open("wb") as destination:
74
+ shutil.copyfileobj(source, destination)
75
+
76
+
77
+ class CustomBuildHook(BuildHookInterface):
78
+ """Inject the matching native binary into a platform-tagged wheel."""
79
+
80
+ PLUGIN_NAME = "custom"
81
+
82
+ def initialize(self, version: str, build_data: dict) -> None: # noqa: ARG002
83
+ """Bundle a staged native binary when building a targeted wheel."""
84
+ target = os.environ.get("HTML_TO_MARKDOWN_CLI_TARGET", "").strip()
85
+ if not target:
86
+ return
87
+
88
+ archive = self._find_archive(target)
89
+ if archive is None:
90
+ raise RuntimeError(
91
+ f"HTML_TO_MARKDOWN_CLI_TARGET={target} but no cli-{target}.(tar.gz|zip) "
92
+ f"found in repo root or dist/; refusing to build an empty platform wheel."
93
+ )
94
+
95
+ wheel_tag = _TAG_MAP.get(target)
96
+ if wheel_tag is None:
97
+ raise RuntimeError(f"no wheel platform tag mapped for target {target}")
98
+
99
+ binary = self._extract_binary(archive, target)
100
+ if binary.stat().st_size == 0:
101
+ raise RuntimeError(f"binary {binary.name} in {archive.name} is empty")
102
+
103
+ force_include = build_data.setdefault("force_include", {})
104
+ force_include[str(binary)] = f"html_to_markdown_cli/bin/{target}/{binary.name}"
105
+
106
+ build_data["pure_python"] = False
107
+ build_data["infer_tag"] = False
108
+ build_data["tag"] = f"py3-none-{wheel_tag}"
109
+
110
+ def _find_archive(self, target: str) -> Path | None:
111
+ root = Path(self.root)
112
+ repo_root = root.parent.parent
113
+ for base in (repo_root, repo_root / "dist", root, root / "dist"):
114
+ for ext in ("tar.gz", "zip"):
115
+ candidate = base / f"cli-{target}.{ext}"
116
+ if candidate.is_file():
117
+ return candidate
118
+ return None
119
+
120
+ def _extract_binary(self, archive: Path, target: str) -> Path:
121
+ binary_name = "html-to-markdown.exe" if target.endswith("windows-msvc") else "html-to-markdown"
122
+ extract_dir = Path(self.root) / ".build-extract" / target
123
+ if extract_dir.exists():
124
+ shutil.rmtree(extract_dir, ignore_errors=True)
125
+ extract_dir.mkdir(parents=True, exist_ok=True)
126
+
127
+ if str(archive).lower().endswith(".zip"):
128
+ _extract_zip_bounded(archive, extract_dir)
129
+ else:
130
+ _extract_tar_bounded(archive, extract_dir)
131
+
132
+ for candidate in extract_dir.rglob(binary_name):
133
+ if candidate.is_file():
134
+ candidate.chmod(0o755)
135
+ return candidate
136
+ raise RuntimeError(f"binary {binary_name} not found inside {archive.name}")
@@ -0,0 +1,8 @@
1
+ """CLI proxy package."""
2
+
3
+ from importlib import metadata
4
+
5
+ try:
6
+ __version__ = metadata.version("html-to-markdown-cli")
7
+ except metadata.PackageNotFoundError:
8
+ __version__ = "0+unknown"
@@ -0,0 +1,45 @@
1
+ """CLI entry point for the html-to-markdown proxy.
2
+
3
+ A platform-specific wheel bundles the native binary under
4
+ ``html_to_markdown_cli/bin/<target>/``; the sdist (and unknown platforms) ship no binary
5
+ and fall back to the runtime downloader. Each wheel contains exactly one ``bin/<target>``
6
+ directory, so the binary is located by globbing rather than by recomputing the target
7
+ triple (which cannot distinguish glibc from musl at runtime).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import os
13
+ import platform
14
+ import subprocess
15
+ import sys
16
+ from pathlib import Path
17
+
18
+ from .downloader import run
19
+
20
+
21
+ def _find_bundled_binary() -> str | None:
22
+ """Return the path to the bundled native binary if this wheel shipped one."""
23
+ bin_root = Path(__file__).parent / "bin"
24
+ if not bin_root.is_dir():
25
+ return None
26
+
27
+ binary_name = "html-to-markdown.exe" if platform.system().lower() == "windows" else "html-to-markdown"
28
+ for candidate in bin_root.glob(f"*/{binary_name}"):
29
+ if candidate.is_file() and os.access(candidate, os.X_OK):
30
+ return str(candidate)
31
+ return None
32
+
33
+
34
+ def main() -> None:
35
+ """Resolve the native binary (bundled or downloaded) and exec it with forwarded argv."""
36
+ bundled = _find_bundled_binary()
37
+ if bundled:
38
+ completed = subprocess.run([bundled, *sys.argv[1:]], check=False) # noqa: S603
39
+ sys.exit(completed.returncode)
40
+
41
+ sys.exit(run(sys.argv[1:]))
42
+
43
+
44
+ if __name__ == "__main__":
45
+ main()
@@ -0,0 +1,292 @@
1
+ """Resolve, download, verify, and run the native html-to-markdown binary.
2
+
3
+ Fallback for installs that ship no bundled binary (the sdist). Queries the GitHub
4
+ release tagged with this package's own version, picks the asset whose name
5
+ contains the platform target triple, and refuses to install it unless it matches
6
+ ``cli-SHA256SUMS``. All diagnostics go to stderr.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import hashlib
12
+ import json
13
+ import os
14
+ import platform
15
+ import re
16
+ import shutil
17
+ import subprocess
18
+ import sys
19
+ import tarfile
20
+ import tempfile
21
+ import zipfile
22
+ from importlib import metadata
23
+ from pathlib import Path, PurePosixPath
24
+ from typing import IO, TYPE_CHECKING, Any
25
+ from urllib.error import URLError
26
+ from urllib.parse import quote, urlsplit
27
+ from urllib.request import HTTPRedirectHandler, Request, build_opener
28
+
29
+ if TYPE_CHECKING:
30
+ from http.client import HTTPMessage
31
+
32
+ REPO = "xberg-io/html-to-markdown"
33
+ BIN_NAME = "html-to-markdown"
34
+ PKG_NAME = "html-to-markdown-cli"
35
+ VERSION_ENV = "HTML_TO_MARKDOWN_CLI_VERSION"
36
+ _USER_AGENT = "html-to-markdown-cli-python-proxy"
37
+ SUMS_NAME = "cli-SHA256SUMS"
38
+ _PEP440_PRERELEASE = re.compile(r"^(?P<base>\d+\.\d+\.\d+)(?:(?P<kind>a|b|rc)(?P<num>\d+))?$")
39
+ _PRERELEASE_LABEL = {"a": "alpha", "b": "beta", "rc": "rc"}
40
+
41
+
42
+ def release_tag(version: str | None = None) -> str:
43
+ """Return the release tag matching this package's version.
44
+
45
+ The package version is the product version, kept in lockstep by
46
+ scripts/sync_consumer_versions.py, so the tag is derived from it rather
47
+ than resolved as "latest". PEP 440 prereleases (``3.24.0rc1``) map back to the
48
+ semver spelling the release tags use (``v3.24.0-rc.1``).
49
+ """
50
+ if version is None:
51
+ version = metadata.version(PKG_NAME)
52
+ match = _PEP440_PRERELEASE.match(version)
53
+ if not match:
54
+ raise RuntimeError(f"cannot derive a release tag from package version {version!r}")
55
+ if match["kind"] is None:
56
+ return f"v{match['base']}"
57
+ return f"v{match['base']}-{_PRERELEASE_LABEL[match['kind']]}.{match['num']}"
58
+
59
+
60
+ def _target_triple() -> str:
61
+ system = platform.system().lower()
62
+ machine = platform.machine().lower()
63
+
64
+ if system == "windows":
65
+ if machine in {"amd64", "x86_64"}:
66
+ return "x86_64-pc-windows-msvc"
67
+ raise RuntimeError(f"unsupported Windows arch: {machine}")
68
+ if system == "linux":
69
+ if machine in {"amd64", "x86_64"}:
70
+ return "x86_64-unknown-linux-gnu"
71
+ if machine in {"aarch64", "arm64"}:
72
+ return "aarch64-unknown-linux-gnu"
73
+ raise RuntimeError(f"unsupported Linux arch: {machine}")
74
+ if system == "darwin":
75
+ if machine in {"aarch64", "arm64"}:
76
+ return "aarch64-apple-darwin"
77
+ if machine in {"amd64", "x86_64"}:
78
+ return "x86_64-apple-darwin"
79
+ raise RuntimeError(f"unsupported macOS arch: {machine}")
80
+ raise RuntimeError(f"unsupported platform: {system} {machine}")
81
+
82
+
83
+ def _binary_name() -> str:
84
+ return f"{BIN_NAME}.exe" if platform.system().lower() == "windows" else BIN_NAME
85
+
86
+
87
+ class _HttpsOnlyRedirectHandler(HTTPRedirectHandler):
88
+ """Reject any redirect whose target is not https (downgrade/SSRF guard)."""
89
+
90
+ def redirect_request(
91
+ self,
92
+ req: Request,
93
+ fp: IO[bytes],
94
+ code: int,
95
+ msg: str,
96
+ headers: HTTPMessage,
97
+ newurl: str,
98
+ ) -> Request | None:
99
+ if urlsplit(newurl).scheme.lower() != "https":
100
+ raise URLError(f"refusing non-https redirect to: {newurl}")
101
+ return super().redirect_request(req, fp, code, msg, headers, newurl)
102
+
103
+
104
+ _opener = build_opener(_HttpsOnlyRedirectHandler())
105
+
106
+
107
+ def _http_get(url: str, accept: str | None = None) -> bytes:
108
+ if urlsplit(url).scheme.lower() != "https":
109
+ raise RuntimeError(f"refusing non-https URL: {url}")
110
+ headers = {"User-Agent": _USER_AGENT}
111
+ if accept:
112
+ headers["Accept"] = accept
113
+ request = Request(url, headers=headers) # noqa: S310 (https-only enforced above)
114
+ try:
115
+ with _opener.open(request, timeout=60) as response:
116
+ if response.status != 200:
117
+ raise RuntimeError(f"HTTP {response.status} for {url}")
118
+ data: bytes = response.read()
119
+ return data
120
+ except URLError as exc:
121
+ raise RuntimeError(f"failed to fetch {url}: {exc}") from exc
122
+
123
+
124
+ def _asset_score(name: str) -> int:
125
+ lowered = name.lower()
126
+ score = 0
127
+ if BIN_NAME.lower() in lowered:
128
+ score += 2
129
+ if "cli" in lowered:
130
+ score += 1
131
+ return score
132
+
133
+
134
+ def _resolve_release() -> tuple[str, dict[str, Any], dict[str, Any] | None]:
135
+ """Return (tag, archive_asset, checksums_asset_or_none) for this platform."""
136
+ triple = _target_triple()
137
+ pinned = os.getenv(VERSION_ENV) or release_tag()
138
+ api_url = f"https://api.github.com/repos/{REPO}/releases/tags/{quote(pinned, safe='')}"
139
+
140
+ release = json.loads(_http_get(api_url, accept="application/vnd.github+json"))
141
+ assets = release.get("assets") or []
142
+ tag = release.get("tag_name") or pinned
143
+
144
+ archives = [
145
+ a
146
+ for a in assets
147
+ if triple in (a.get("name") or "").lower()
148
+ and ((a.get("name") or "").lower().endswith(".tar.gz") or (a.get("name") or "").lower().endswith(".zip"))
149
+ ]
150
+ if not archives:
151
+ raise RuntimeError(f'no release asset matching target triple "{triple}" in {REPO} release {tag}')
152
+ archives.sort(key=lambda a: _asset_score(a.get("name") or ""), reverse=True)
153
+ archive = archives[0]
154
+
155
+ checksums = next((a for a in assets if a.get("name") == SUMS_NAME), None)
156
+ return tag, archive, checksums
157
+
158
+
159
+ def _expected_digest(text: str, asset_name: str) -> str | None:
160
+ for line in text.splitlines():
161
+ stripped = line.strip()
162
+ if not stripped:
163
+ continue
164
+ parts = stripped.split()
165
+ if len(parts) < 2:
166
+ continue
167
+ name = parts[-1].lstrip("*")
168
+ if name == asset_name:
169
+ return parts[0].lower()
170
+ return None
171
+
172
+
173
+ def _verify_checksum(archive_path: Path, asset_name: str, sums_text: str) -> None:
174
+ expected = _expected_digest(sums_text, asset_name)
175
+ if not expected:
176
+ raise RuntimeError(f"no checksum entry for {asset_name} in {SUMS_NAME} — refusing to install unverified binary")
177
+ digest = hashlib.sha256(archive_path.read_bytes()).hexdigest().lower()
178
+ if digest != expected:
179
+ raise RuntimeError(f"checksum mismatch for {asset_name} (expected {expected}, got {digest})")
180
+
181
+
182
+ def _verify_download(archive_path: Path, asset_name: str, checksums: dict[str, Any] | None) -> None:
183
+ if not checksums:
184
+ raise RuntimeError(f"release has no {SUMS_NAME} asset for {asset_name} — refusing to install unverified binary")
185
+ sums_text = _http_get(checksums["browser_download_url"]).decode("utf-8")
186
+ _verify_checksum(archive_path, asset_name, sums_text)
187
+ print(f"Checksum verified for {asset_name}.", file=sys.stderr) # noqa: T201 (diagnostic to stderr)
188
+
189
+
190
+ def _is_within(base: Path, target: Path) -> bool:
191
+ """Return True if `target`'s resolved path stays inside `base`."""
192
+ base_resolved = base.resolve()
193
+ try:
194
+ target.resolve().relative_to(base_resolved)
195
+ except ValueError:
196
+ return False
197
+ return True
198
+
199
+
200
+ def _reject_unsafe_member(name: str) -> None:
201
+ """Reject absolute paths and any `..` component (zip-slip / tar-slip)."""
202
+ normalized = name.replace("\\", "/")
203
+ pure = PurePosixPath(normalized)
204
+ if pure.is_absolute() or normalized.startswith("/"):
205
+ raise RuntimeError(f"refusing absolute path in archive: {name}")
206
+ if len(normalized) >= 2 and normalized[1] == ":":
207
+ raise RuntimeError(f"refusing absolute path in archive: {name}")
208
+ if any(part == ".." for part in pure.parts):
209
+ raise RuntimeError(f"refusing parent-directory escape in archive: {name}")
210
+
211
+
212
+ def _safe_extract(archive_path: Path, asset_name: str, dest: Path) -> None:
213
+ """Extract a tar/zip, validating every member stays within `dest`.
214
+
215
+ Does not rely on extractall() defaults or the 3.12-only ``filter='data'``:
216
+ each member name is rejected up front if it is absolute or contains ``..``,
217
+ and the resolved destination path is re-checked against ``dest``.
218
+ """
219
+ dest = dest.resolve()
220
+ dest.mkdir(parents=True, exist_ok=True)
221
+
222
+ if asset_name.lower().endswith(".zip"):
223
+ with zipfile.ZipFile(archive_path) as zf:
224
+ for member in zf.namelist():
225
+ _reject_unsafe_member(member)
226
+ if not _is_within(dest, dest / member):
227
+ raise RuntimeError(f"refusing archive entry escaping dest: {member}")
228
+ for member in zf.namelist():
229
+ zf.extract(member, dest)
230
+ else:
231
+ with tarfile.open(archive_path, "r:gz") as tf:
232
+ for tar_member in tf.getmembers():
233
+ _reject_unsafe_member(tar_member.name)
234
+ if not _is_within(dest, dest / tar_member.name):
235
+ raise RuntimeError(f"refusing archive entry escaping dest: {tar_member.name}")
236
+ if tar_member.islnk() or tar_member.issym():
237
+ link_target = dest / tar_member.name
238
+ if not _is_within(dest, link_target.parent / tar_member.linkname):
239
+ raise RuntimeError(f"refusing link escaping dest: {tar_member.name}")
240
+ tf.extractall(dest) # noqa: S202 (members validated above)
241
+
242
+
243
+ def _find_binary(root: Path, name: str) -> Path | None:
244
+ for candidate in root.rglob(name):
245
+ if candidate.is_file():
246
+ return candidate
247
+ return None
248
+
249
+
250
+ def _cache_dir(tag: str) -> Path:
251
+ cache = Path.home() / ".cache" / PKG_NAME / tag
252
+ cache.mkdir(parents=True, exist_ok=True, mode=0o700)
253
+ return cache
254
+
255
+
256
+ def ensure_binary() -> str:
257
+ """Ensure the native binary exists locally, downloading if necessary."""
258
+ override = os.getenv("HTML_TO_MARKDOWN_BINARY")
259
+ if override:
260
+ return override
261
+
262
+ tag, archive, checksums = _resolve_release()
263
+ cache = _cache_dir(tag)
264
+ binary_path = cache / _binary_name()
265
+ if binary_path.exists() and os.access(binary_path, os.X_OK):
266
+ return str(binary_path)
267
+
268
+ print(f"Downloading {BIN_NAME} {tag} asset {archive['name']}...", file=sys.stderr) # noqa: T201
269
+ with tempfile.TemporaryDirectory() as tmpdir:
270
+ tmp = Path(tmpdir)
271
+ archive_path = tmp / Path(archive["name"]).name
272
+ archive_path.write_bytes(_http_get(archive["browser_download_url"]))
273
+ _verify_download(archive_path, archive["name"], checksums)
274
+ extract_dir = tmp / "extract"
275
+ extract_dir.mkdir()
276
+ _safe_extract(archive_path, archive["name"], extract_dir)
277
+ found = _find_binary(extract_dir, _binary_name())
278
+ if not found:
279
+ raise RuntimeError(f"binary {_binary_name()} not found after extracting {archive['name']}")
280
+ shutil.move(str(found), str(binary_path))
281
+
282
+ if platform.system().lower() != "windows":
283
+ binary_path.chmod(0o755)
284
+ print(f"{BIN_NAME} installed.", file=sys.stderr) # noqa: T201 (diagnostic to stderr)
285
+ return str(binary_path)
286
+
287
+
288
+ def run(args: list[str]) -> int:
289
+ """Run the native binary with the given args, returning its exit code."""
290
+ binary_path = ensure_binary()
291
+ completed = subprocess.run([binary_path, *args], check=False) # noqa: S603
292
+ return completed.returncode
@@ -0,0 +1,38 @@
1
+ [build-system]
2
+ build-backend = "hatchling.build"
3
+ requires = ["hatchling"]
4
+
5
+ [project]
6
+ maintainers = [
7
+ { name = "Na'aman Hirschfeld", email = "naaman@xberg.io" },
8
+ { name = "Tobias Perlstein", email = "tobias@xberg.io" },
9
+ ]
10
+ name = "html-to-markdown-cli"
11
+ version = "3.23.1"
12
+ description = "CLI proxy for html-to-markdown — downloads and runs the native html-to-markdown binary from GitHub releases."
13
+ readme = "README.md"
14
+ license = { text = "MIT" }
15
+ authors = [
16
+ { name = "Na'aman Hirschfeld", email = "naaman@xberg.io" },
17
+ { name = "Tobias Perlstein", email = "tobias@xberg.io" },
18
+ ]
19
+ requires-python = ">=3.11"
20
+ classifiers = [
21
+ "Development Status :: 4 - Beta",
22
+ "Environment :: Console",
23
+ "Intended Audience :: Developers",
24
+ "License :: OSI Approved :: MIT License",
25
+ "Programming Language :: Python :: 3 :: Only",
26
+ "Programming Language :: Python :: 3.11",
27
+ "Programming Language :: Python :: 3.12",
28
+ "Programming Language :: Python :: 3.13",
29
+ "Programming Language :: Python :: 3.14",
30
+ ]
31
+ urls.Homepage = "https://github.com/xberg-io/html-to-markdown"
32
+ urls.Issues = "https://github.com/xberg-io/html-to-markdown/issues"
33
+ urls.Repository = "https://github.com/xberg-io/html-to-markdown.git"
34
+ scripts.html-to-markdown = "html_to_markdown_cli.cli:main"
35
+
36
+ [tool.hatch]
37
+ build.hooks.custom.path = "hatch_build.py"
38
+ build.targets.wheel.packages = ["html_to_markdown_cli"]
@@ -0,0 +1,13 @@
1
+ """Make the build hook and the package importable from the tests directory.
2
+
3
+ ``hatch_build.py`` sits at the package root because hatchling loads it by path
4
+ rather than as an installed module, so it is not importable from ``tests/``
5
+ without help.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import sys
11
+ from pathlib import Path
12
+
13
+ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
@@ -0,0 +1,90 @@
1
+ """Pinned-release and checksum behaviour of the runtime downloader."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ from pathlib import Path
8
+
9
+ import pytest
10
+ from html_to_markdown_cli import downloader
11
+
12
+
13
+ @pytest.mark.parametrize(
14
+ ("version", "tag"),
15
+ [
16
+ ("3.23.0", "v3.23.0"),
17
+ ("3.24.0rc1", "v3.24.0-rc.1"),
18
+ ("3.24.0a2", "v3.24.0-alpha.2"),
19
+ ("3.24.0b3", "v3.24.0-beta.3"),
20
+ ],
21
+ )
22
+ def test_release_tag_is_derived_from_the_package_version(version: str, tag: str) -> None:
23
+ assert downloader.release_tag(version) == tag
24
+
25
+
26
+ def test_release_tag_rejects_a_version_it_cannot_map() -> None:
27
+ with pytest.raises(RuntimeError, match="cannot derive a release tag"):
28
+ downloader.release_tag("3.24.0.dev1")
29
+
30
+
31
+ def _release(monkeypatch: pytest.MonkeyPatch, requested: list[str]) -> None:
32
+ assets = [
33
+ {"name": "cli-x86_64-unknown-linux-gnu.tar.gz", "browser_download_url": "https://example.invalid/a"},
34
+ {"name": "cli-SHA256SUMS", "browser_download_url": "https://example.invalid/sums"},
35
+ ]
36
+
37
+ def fake_get(url: str, accept: str | None = None) -> bytes: # noqa: ARG001
38
+ requested.append(url)
39
+ return b'{"tag_name": "v3.23.0", "assets": %s}' % json.dumps(assets).encode()
40
+
41
+ monkeypatch.setattr(downloader, "_http_get", fake_get)
42
+ monkeypatch.setattr(downloader, "_target_triple", lambda: "x86_64-unknown-linux-gnu")
43
+
44
+
45
+ def test_resolve_release_requests_the_pinned_tag_not_latest(monkeypatch: pytest.MonkeyPatch) -> None:
46
+ requested: list[str] = []
47
+ _release(monkeypatch, requested)
48
+ monkeypatch.delenv(downloader.VERSION_ENV, raising=False)
49
+ monkeypatch.setattr(downloader, "release_tag", lambda version=None: "v3.23.0") # noqa: ARG005
50
+
51
+ tag, archive, checksums = downloader._resolve_release()
52
+
53
+ assert requested == ["https://api.github.com/repos/xberg-io/html-to-markdown/releases/tags/v3.23.0"]
54
+ assert tag == "v3.23.0"
55
+ assert archive["name"] == "cli-x86_64-unknown-linux-gnu.tar.gz"
56
+ assert checksums is not None
57
+ assert checksums["name"] == "cli-SHA256SUMS"
58
+
59
+
60
+ def test_verify_checksum_accepts_the_matching_digest(tmp_path: Path) -> None:
61
+ archive = tmp_path / "cli-x86_64-unknown-linux-gnu.tar.gz"
62
+ archive.write_bytes(b"archive-bytes")
63
+ digest = hashlib.sha256(b"archive-bytes").hexdigest()
64
+ sums = f"{'0' * 64} cli-other.tar.gz\n{digest} {archive.name}\n"
65
+
66
+ downloader._verify_checksum(archive, archive.name, sums)
67
+
68
+
69
+ def test_verify_checksum_rejects_a_mismatch(tmp_path: Path) -> None:
70
+ archive = tmp_path / "cli-x86_64-unknown-linux-gnu.tar.gz"
71
+ archive.write_bytes(b"tampered")
72
+
73
+ with pytest.raises(RuntimeError, match="checksum mismatch"):
74
+ downloader._verify_checksum(archive, archive.name, f"{'0' * 64} {archive.name}\n")
75
+
76
+
77
+ def test_verify_checksum_rejects_a_missing_entry(tmp_path: Path) -> None:
78
+ archive = tmp_path / "cli-x86_64-unknown-linux-gnu.tar.gz"
79
+ archive.write_bytes(b"x")
80
+
81
+ with pytest.raises(RuntimeError, match="no checksum entry"):
82
+ downloader._verify_checksum(archive, archive.name, f"{'0' * 64} cli-other.tar.gz\n")
83
+
84
+
85
+ def test_download_without_a_sums_asset_is_refused(tmp_path: Path) -> None:
86
+ archive = tmp_path / "cli-x86_64-unknown-linux-gnu.tar.gz"
87
+ archive.write_bytes(b"x")
88
+
89
+ with pytest.raises(RuntimeError, match="refusing to install unverified binary"):
90
+ downloader._verify_download(archive, archive.name, None)
@@ -0,0 +1,61 @@
1
+ """The wheel build hook repackages the released CLI archive for one target."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import io
6
+ import tarfile
7
+ from pathlib import Path
8
+
9
+ import pytest
10
+ from hatch_build import CustomBuildHook
11
+
12
+ _TARGET = "x86_64-unknown-linux-gnu"
13
+
14
+
15
+ def _hook(root: Path) -> CustomBuildHook:
16
+ return CustomBuildHook(
17
+ str(root),
18
+ config={},
19
+ build_config={},
20
+ metadata=None,
21
+ directory=str(root / "dist"),
22
+ target_name="wheel",
23
+ )
24
+
25
+
26
+ def _stage(root: Path, members: dict[str, bytes]) -> None:
27
+ dist = root / "dist"
28
+ dist.mkdir(parents=True, exist_ok=True)
29
+ with tarfile.open(dist / f"cli-{_TARGET}.tar.gz", "w:gz") as archive:
30
+ for name, payload in members.items():
31
+ info = tarfile.TarInfo(name)
32
+ info.size = len(payload)
33
+ archive.addfile(info, io.BytesIO(payload))
34
+
35
+
36
+ def test_target_wheel_force_includes_the_binary_and_tags_the_platform(tmp_path: Path, monkeypatch) -> None:
37
+ _stage(tmp_path, {f"cli-{_TARGET}/html-to-markdown": b"\x7fELF", f"cli-{_TARGET}/LICENSE": b"MIT"})
38
+ monkeypatch.setenv("HTML_TO_MARKDOWN_CLI_TARGET", _TARGET)
39
+ build_data: dict = {}
40
+
41
+ _hook(tmp_path).initialize("3.23.0", build_data)
42
+
43
+ assert set(build_data["force_include"].values()) == {f"html_to_markdown_cli/bin/{_TARGET}/html-to-markdown"}
44
+ assert build_data["tag"] == "py3-none-manylinux_2_28_x86_64"
45
+ assert build_data["pure_python"] is False
46
+
47
+
48
+ def test_missing_archive_refuses_to_build_an_empty_platform_wheel(tmp_path: Path, monkeypatch) -> None:
49
+ monkeypatch.setenv("HTML_TO_MARKDOWN_CLI_TARGET", _TARGET)
50
+
51
+ with pytest.raises(RuntimeError, match="refusing to build an empty platform wheel"):
52
+ _hook(tmp_path).initialize("3.23.0", {})
53
+
54
+
55
+ def test_without_a_target_the_hook_is_a_noop(tmp_path: Path, monkeypatch) -> None:
56
+ monkeypatch.delenv("HTML_TO_MARKDOWN_CLI_TARGET", raising=False)
57
+ build_data: dict = {}
58
+
59
+ _hook(tmp_path).initialize("3.23.0", build_data)
60
+
61
+ assert build_data == {}