loremfile 0.1.0__py3-none-any.whl

loremfile-0.1.0.dist-info/METADATA

@@ -0,0 +1,95 @@
+Metadata-Version: 2.4
+Name: loremfile
+Version: 0.1.0
+Summary: Download CC0 sample files from loremfile.dev and verify every byte
+Author: the loremfile.dev contributors
+License-Expression: MIT
+Project-URL: Homepage, https://loremfile.dev
+Project-URL: Source, https://github.com/kumarprabhashanand/loremfile
+Project-URL: Issues, https://github.com/kumarprabhashanand/loremfile/issues
+Keywords: fixtures,test-data,sample-files,cc0,download
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Testing
+Classifier: Environment :: Console
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# loremfile
+
+Download [loremfile.dev](https://loremfile.dev) sample files and check every byte against
+the published manifest.
+
+```console
+$ pip install loremfile
+$ loremfile get pdf/minimal.pdf mp4/720p-5s.mp4
+  written  mp4/720p-5s.mp4  1,974,574 bytes
+  written  pdf/minimal.pdf  317 bytes
+files=2, dest=loremfile-fixtures
+```
+
+Everything it does can be done with `curl` and `sha256sum -c`. What it adds is doing it
+correctly by default: every file is verified, and a file that fails is not written.
+
+## Commands
+
+| Command | |
+|---|---|
+| `loremfile get PATH…` | Download those fixtures into `--dest` (default `loremfile-fixtures`), verifying each. |
+| `loremfile get --format svg --format pdf` | The same for every published file of a format. |
+| `loremfile list [--format F] [--tag T] [--max-bytes N]` | What is published, without downloading. |
+| `loremfile verify [PATH…]` | Check files already on disk against the manifest. |
+
+Every command takes `--json` for scripts, `--quiet`, and `--catalog-version X.Y.Z` to
+refuse to run if loremfile.dev has moved on. `get` also takes `--force` (overwrite) and
+`--dry-run` (what it would fetch, and how many bytes).
+
+Exit codes: `0` fine, `1` a file failed verification or is missing, `2` the request was
+wrong, `3` loremfile.dev could not be read.
+
+## What it will not do
+
+- **Write a file it could not verify.** There is no `--no-verify`.
+- **Talk to any host but loremfile.dev.** No mirror option, and redirects off the host are
+  not followed.
+- **Take credentials.** Nothing it does needs authentication, so it accepts none.
+- **Exceed the published rate limit**: one file at a time, a pause between them, and a
+  429 is waited out rather than retried tightly.
+- **Generate fixtures.** Those are built in a pinned toolchain image; a client that made
+  its own would produce different bytes and undermine the only promise the project makes.
+
+## Python versions
+
+3.11, 3.12, 3.13 and 3.14, tested in CI on Linux, macOS and Windows. The floor is the
+oldest CPython still receiving security fixes when a release is cut; a new one is
+supported within six weeks of its release, and an end-of-life one is dropped in the next
+minor release.
+
+Zero dependencies — standard library only.
+
+## In GitHub Actions, prefer the action
+
+```yaml
+- uses: kumarprabhashanand/loremfile/action@action-v1
+  with:
+    paths: pdf/minimal.pdf mp4/720p-5s.mp4
+```
+
+It is shell only and needs no install step. This client is for everywhere else: a laptop,
+a Makefile, a CI system that is not Actions, or Windows — which the action does not
+support because it needs `sha256sum`.
+
+## One name, two tools
+
+The console script installed here is `loremfile`, which is also the name of the
+repository's own development CLI (the one that generates and publishes the fixtures).
+They are never installed together — that tool lives only inside the project's toolchain
+image — but a contributor who installs both into one environment gets whichever was
+installed last. The published package contains the client only: no generators, no
+validators, no uploader, no infrastructure code.
+
+## Licence
+
+MIT for the client; the files it downloads are CC0.

loremfile-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+loremfile_client/__init__.py,sha256=_-0ETP3BJsoTsVDvAMygrvgU2zRaseHmuBlxveRUH9M,112
+loremfile_client/__main__.py,sha256=7NJZ7jIxrhNrkI834PwWD-5TbhXb9hMT6IEHr6xyOhw,64
+loremfile_client/api.py,sha256=8KAnboF63UaAmU8NCJtmZzSggDknfW2yfj6c_gsF1xg,10420
+loremfile_client/cli.py,sha256=eeaqRlN7PC1kWrNY_mxZBxY7gzkJRyvPkW4rhhCBcxU,6019
+loremfile-0.1.0.dist-info/METADATA,sha256=tvGbPvTx2v5sYo-GHSOu0mAiYLFXE-r3MBATBzDbMpY,3968
+loremfile-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+loremfile-0.1.0.dist-info/entry_points.txt,sha256=QwjMb_oQd21294DidZ5E9htPaSTGek3hABkjyfzGI4A,56
+loremfile-0.1.0.dist-info/top_level.txt,sha256=F1MyxSFV3mq7FtWf9ySjuPwMA0F1P7PEOmCNvKTKN2E,17
+loremfile-0.1.0.dist-info/RECORD,,

loremfile-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

loremfile-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+loremfile = loremfile_client.cli:main

loremfile-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+loremfile_client

loremfile_client/__init__.py

@@ -0,0 +1,3 @@
+"""A small client for loremfile.dev: download CC0 sample files and verify every byte."""
+
+__version__ = "0.1.0"

loremfile_client/__main__.py

@@ -0,0 +1,3 @@
+from loremfile_client.cli import main
+
+raise SystemExit(main())

loremfile_client/api.py

@@ -0,0 +1,246 @@
+"""Fetching fixtures from loremfile.dev, and checking every byte that arrives.
+
+Standard library only, on purpose (the CLI spec): a client whose job is to download a
+file and hash it should not ask anyone to resolve a dependency tree, and the smaller the
+supply chain the less there is to audit.
+
+The one thing to know about this module: `BASE_URL` is a constant. There is no flag, no
+configuration file and no environment variable that points the client at another host —
+only a loopback override this project's own tests inject, which is rejected for anything
+that is not 127.0.0.1, ::1 or localhost. A mirror flag would contradict the README's
+"do not rely on any other mirror", and an override that accepted a public host would be a
+way to make `loremfile get` fetch from somewhere else entirely.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import http.client
+import json
+import os
+import tempfile
+import time
+import urllib.error
+import urllib.request
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Any, cast
+
+#: The only host this client talks to.
+BASE_URL = "https://loremfile.dev/"
+
+#: Injected by this project's tests so they never touch production. Loopback only.
+OVERRIDE = "LOREMFILE_BASE_URL"
+LOOPBACK = ("http://127.0.0.1:", "http://localhost:", "http://[::1]:")
+
+#: The published rate limit is 30 requests a second per client; one file at a time with a
+#: pause between them is well under it, and a 429 is answered with patience.
+PAUSE_SECONDS = 0.5
+BACKOFF_SECONDS = (2, 4, 8)
+TIMEOUT_SECONDS = 300
+USER_AGENT = "loremfile-client"
+
+#: Read and hashed a megabyte at a time, so a 100 MB fixture costs a megabyte of memory.
+CHUNK_BYTES = 1024 * 1024
+
+
+class Refused(Exception):
+    """The request was wrong: an unknown path, a bad override, a version that has moved."""
+
+
+class Unreachable(Exception):
+    """loremfile.dev could not be read."""
+
+
+class Mismatch(Exception):
+    """Bytes arrived that are not the bytes the manifest describes."""
+
+
+def base_url() -> str:
+    override = os.environ.get(OVERRIDE, "").strip()
+    if not override:
+        return BASE_URL
+    if not override.startswith(LOOPBACK):
+        raise Refused(
+            f"{OVERRIDE} accepts a loopback address only, so that this project's tests can "
+            f"run without touching production; {override!r} is not one. There is no way to "
+            "point this client at another host, by design."
+        )
+    return override if override.endswith("/") else override + "/"
+
+
+def _open(path: str) -> http.client.HTTPResponse:
+    """An open response, retrying only a 429 — our own rate limit asking for patience.
+
+    The retry is here, around opening, because that is where a 429 arrives. A stream
+    that fails halfway is not retried: the caller would have to decide what to do with
+    the bytes it already has, and this client's answer to a half-file is to delete it.
+    """
+    url = base_url() + path.lstrip("/")
+    request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})  # noqa: S310
+    for pause in (*BACKOFF_SECONDS, None):
+        try:
+            response = urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS)  # noqa: S310
+        except urllib.error.HTTPError as exc:
+            if exc.code == 429 and pause is not None:  # noqa: PLR2004 - the HTTP status
+                time.sleep(pause)
+                continue
+            raise Unreachable(f"{url} answered {exc.code}") from exc
+        except urllib.error.URLError as exc:
+            raise Unreachable(f"{url} could not be read: {exc.reason}") from exc
+        if response.geturl().split("/")[2] != url.split("/")[2]:
+            response.close()
+            raise Refused(f"{url} redirected off loremfile.dev, which is never followed")
+        # urlopen is typed as returning Any for non-HTTP schemes; this client only ever
+        # opens http(s), which `base_url` is the single gate for. A cast rather than an
+        # assert: `python -O` strips asserts, and shipped code should not depend on a
+        # statement that may not be there.
+        return cast("http.client.HTTPResponse", response)
+    raise Unreachable(f"{url} is still answering 429")
+
+
+def fetch(path: str) -> bytes:
+    """The whole body at once. For the manifest, which is JSON and has to be parsed."""
+    with _open(path) as response:
+        return bytes(response.read())
+
+
+def stream(path: str) -> Iterator[bytes]:
+    """The body in chunks, for fixtures — which run to 100 MB.
+
+    Reading one of those into memory to hash it would be 100 MB of a CI container's
+    often 512 MB, for no benefit: sha256 is happy to be fed a chunk at a time.
+    """
+    with _open(path) as response:
+        while chunk := response.read(CHUNK_BYTES):
+            yield bytes(chunk)
+
+
+def manifest(*, catalog_version: str | None = None) -> dict[str, Any]:
+    try:
+        document = json.loads(fetch("manifest.json"))
+    except json.JSONDecodeError as exc:
+        raise Unreachable(f"the manifest is not JSON: {exc}") from exc
+    if not isinstance(document, dict):
+        raise Unreachable("the manifest is not a JSON object")
+    published = str(document.get("catalog_version", ""))
+    if catalog_version and published != catalog_version:
+        raise Refused(f"loremfile.dev publishes catalog {published}, not {catalog_version}")
+    return dict(document)
+
+
+def active(document: dict[str, Any]) -> list[dict[str, Any]]:
+    return [e for e in document.get("fixtures", []) if e.get("status", "active") == "active"]
+
+
+def select(
+    entries: list[dict[str, Any]],
+    *,
+    paths: list[str] | None = None,
+    formats: list[str] | None = None,
+) -> list[dict[str, Any]]:
+    """The entries a request names, or `Refused` naming what does not exist."""
+    wanted_paths, wanted_formats = set(paths or []), set(formats or [])
+    if not wanted_paths and not wanted_formats:
+        raise Refused("name at least one path or format")
+    by_path = {e["path"]: e for e in entries}
+    unknown = sorted(wanted_paths - set(by_path))
+    if unknown:
+        raise Refused(f"not published: {', '.join(unknown)}")
+    published_formats = {e["format"] for e in entries}
+    unknown_formats = sorted(wanted_formats - published_formats)
+    if unknown_formats:
+        raise Refused(f"no such format: {', '.join(unknown_formats)}")
+    chosen = {
+        e["path"]: e for e in entries if e["path"] in wanted_paths or e["format"] in wanted_formats
+    }
+    return [chosen[path] for path in sorted(chosen)]
+
+
+def target(dest: Path, path: str) -> Path:
+    """Where a fixture lands, refusing anything that would escape `dest`.
+
+    The paths come from our own manifest, so this is not the day's most likely failure —
+    but a client that writes where a downloaded document tells it to is the shape of
+    problem this project publishes a fixture about (`edge/zip-directory-traversal-name.zip`).
+    """
+    relative = PurePosixPath(path)
+    if relative.is_absolute() or ".." in relative.parts:
+        raise Refused(f"refusing {path!r}: not the shape of a fixture path")
+    resolved = (dest / Path(*relative.parts)).resolve()
+    if not str(resolved).startswith(str(dest.resolve())):
+        raise Refused(f"refusing {path!r}: it would be written outside {dest}")
+    return resolved
+
+
+def pace() -> None:
+    """The wait between two downloads. One place, so a test can make it instant and the
+    rate manners stay a property of this module rather than of whoever loops."""
+    time.sleep(PAUSE_SECONDS)
+
+
+@dataclass(frozen=True)
+class Result:
+    """What happened to one fixture."""
+
+    path: str
+    status: str  # written | skipped | ok | missing | changed
+    detail: str = ""
+
+    @property
+    def ok(self) -> bool:
+        return self.status in {"written", "skipped", "ok"}
+
+
+def download(entry: dict[str, Any], dest: Path, *, force: bool = False) -> Result:
+    """Fetch one fixture, hashing as it arrives, and put it at its path only if it matches.
+
+    The bytes go to a temporary file **in the destination directory**, not to memory and
+    not to the system temporary directory: the first would cost 100 MB of RAM for the
+    largest fixtures, and the second could be on another filesystem, where the final move
+    would be a copy rather than a rename. `Path.replace` within one directory is atomic,
+    so a reader of that directory sees either no file or the whole verified one.
+
+    The guarantee is unchanged: nothing unverified is ever at the published path. What is
+    new is that nothing unverified survives anywhere — the temporary file is removed on a
+    mismatch, on an interrupted download, and on any other failure.
+    """
+    destination = target(dest, entry["path"])
+    if destination.exists() and not force:
+        return Result(entry["path"], "skipped", "already here; --force overwrites")
+    destination.parent.mkdir(parents=True, exist_ok=True)
+
+    digest = hashlib.sha256()
+    written = 0
+    handle = tempfile.NamedTemporaryFile(  # noqa: SIM115 - closed by the `with` below
+        dir=destination.parent, prefix=f".{destination.name}.", suffix=".part", delete=False
+    )
+    partial = Path(handle.name)
+    try:
+        with handle:
+            for chunk in stream(entry["path"]):
+                digest.update(chunk)
+                written += handle.write(chunk)
+        if digest.hexdigest() != entry["sha256"]:
+            raise Mismatch(
+                f"{entry['path']}: downloaded sha256 {digest.hexdigest()}, "
+                f"the manifest says {entry['sha256']}"
+            )
+        partial.replace(destination)
+    finally:
+        # A no-op once the rename has happened, and the whole point otherwise.
+        partial.unlink(missing_ok=True)
+    return Result(entry["path"], "written", f"{written:,} bytes")
+
+
+def verify(entry: dict[str, Any], dest: Path) -> Result:
+    """Check a fixture already on disk against the published manifest."""
+    destination = target(dest, entry["path"])
+    if not destination.is_file():
+        return Result(entry["path"], "missing", f"not in {dest}")
+    digest = hashlib.sha256(destination.read_bytes()).hexdigest()
+    if digest != entry["sha256"]:
+        expected = str(entry["sha256"])[:12]
+        return Result(entry["path"], "changed", f"sha256 {digest[:12]}, expected {expected}")
+    return Result(entry["path"], "ok", f"{destination.stat().st_size:,} bytes")

loremfile_client/cli.py

@@ -0,0 +1,150 @@
+"""`loremfile get | list | verify` (the CLI spec).
+
+argparse rather than a framework, because the package has no dependencies and this is
+three subcommands. Exit codes are the contract: 0 fine, 1 a file failed verification or
+is missing, 2 the request was wrong, 3 loremfile.dev could not be read.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+from loremfile_client import __version__, api
+
+OK, FAILED, REFUSED, UNREACHABLE = 0, 1, 2, 3
+DEFAULT_DEST = "loremfile-fixtures"
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="loremfile",
+        description="Download CC0 sample files from loremfile.dev and check every byte.",
+        epilog="Files come from loremfile.dev and nowhere else; there is no mirror option.",
+    )
+    parser.add_argument("--version", action="version", version=f"loremfile {__version__}")
+    subcommands = parser.add_subparsers(dest="command", required=True)
+
+    def shared(sub: argparse.ArgumentParser) -> None:
+        sub.add_argument("--json", action="store_true", help="Print one JSON object.")
+        sub.add_argument("--quiet", action="store_true", help="Print nothing but failures.")
+        sub.add_argument(
+            "--catalog-version",
+            metavar="X.Y.Z",
+            help="Refuse to run unless loremfile.dev publishes this catalog version.",
+        )
+
+    get = subcommands.add_parser("get", help="Download fixtures and verify them.")
+    get.add_argument("paths", nargs="*", metavar="PATH", help="e.g. pdf/minimal.pdf")
+    get.add_argument(
+        "--format",
+        action="append",
+        default=[],
+        metavar="FMT",
+        help="Every published file of this format. Repeatable.",
+    )
+    get.add_argument("--dest", default=DEFAULT_DEST, type=Path, help=f"Default: {DEFAULT_DEST}")
+    get.add_argument("--force", action="store_true", help="Overwrite files already there.")
+    get.add_argument("--dry-run", action="store_true", help="List what would be fetched.")
+    shared(get)
+
+    listing = subcommands.add_parser("list", help="What is published, without downloading.")
+    listing.add_argument("--format", action="append", default=[], metavar="FMT")
+    listing.add_argument("--tag", action="append", default=[], metavar="TAG")
+    listing.add_argument("--max-bytes", type=int, metavar="N")
+    shared(listing)
+
+    check = subcommands.add_parser("verify", help="Check files on disk against the manifest.")
+    check.add_argument("paths", nargs="*", metavar="PATH")
+    check.add_argument("--format", action="append", default=[], metavar="FMT")
+    check.add_argument("--dest", default=DEFAULT_DEST, type=Path, help=f"Default: {DEFAULT_DEST}")
+    shared(check)
+    return parser
+
+
+def emit(args: argparse.Namespace, summary: dict[str, Any], results: list[api.Result]) -> None:
+    if args.json:
+        print(json.dumps({"summary": summary, "items": [vars(r) for r in results]}, indent=2))
+        return
+    for result in results:
+        if not result.ok or not args.quiet:
+            print(
+                f"  {result.status:8} {result.path}{'  ' + result.detail if result.detail else ''}"
+            )
+    if not args.quiet:
+        print(", ".join(f"{key}={value}" for key, value in summary.items()))
+
+
+def run_get(args: argparse.Namespace) -> int:
+    entries = api.select(
+        api.active(api.manifest(catalog_version=args.catalog_version)),
+        paths=args.paths,
+        formats=args.format,
+    )
+    if args.dry_run:
+        total = sum(e["bytes"] for e in entries)
+        emit(
+            args,
+            {"files": len(entries), "bytes": total, "dest": str(args.dest), "dry-run": True},
+            [api.Result(e["path"], "would fetch", f"{e['bytes']:,} bytes") for e in entries],
+        )
+        return OK
+    results = []
+    for index, entry in enumerate(entries):
+        results.append(api.download(entry, args.dest, force=args.force))
+        if index + 1 < len(entries):
+            api.pace()
+    emit(args, {"files": len(results), "dest": str(args.dest)}, results)
+    return OK
+
+
+def run_list(args: argparse.Namespace) -> int:
+    entries = api.active(api.manifest(catalog_version=args.catalog_version))
+    if args.format:
+        entries = [e for e in entries if e["format"] in set(args.format)]
+    if args.tag:
+        entries = [e for e in entries if set(args.tag) & set(e.get("tags", []))]
+    if args.max_bytes is not None:
+        entries = [e for e in entries if e["bytes"] <= args.max_bytes]
+    results = [
+        api.Result(e["path"], "listed", f"{e['bytes']:,} bytes  {e['mime']}") for e in entries
+    ]
+    emit(args, {"files": len(results)}, results)
+    return OK
+
+
+def run_verify(args: argparse.Namespace) -> int:
+    entries = api.active(api.manifest(catalog_version=args.catalog_version))
+    if args.paths or args.format:
+        entries = api.select(entries, paths=args.paths, formats=args.format)
+    else:
+        # Everything the destination actually holds, so `verify` alone checks a directory.
+        entries = [e for e in entries if (args.dest / Path(e["path"])).is_file()]
+    results = [api.verify(entry, args.dest) for entry in entries]
+    failing = [r for r in results if not r.ok]
+    emit(args, {"checked": len(results), "failing": len(failing)}, results)
+    return FAILED if failing else OK
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    runner = {"get": run_get, "list": run_list, "verify": run_verify}[args.command]
+    try:
+        return runner(args)
+    except api.Refused as exc:
+        print(f"loremfile: {exc}", file=sys.stderr)
+        return REFUSED
+    except api.Mismatch as exc:
+        print(f"loremfile: {exc}", file=sys.stderr)
+        return FAILED
+    except api.Unreachable as exc:
+        print(f"loremfile: {exc}", file=sys.stderr)
+        return UNREACHABLE
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())