awrtifact 0.1.0__py3-none-any.whl

awrtifact/__init__.py

@@ -0,0 +1,12 @@
+"""awrtifact — deliberately chunk artifacts into GitHub release assets.
+
+Aither World Artifact is the productized aitherkvcache mirror lane: ANY artifact
+(model weights, datasets, builds, backups) is split into `.partN` slices under
+GitHub's 2 GiB per-asset cap, stored as a versioned GitHub release, served by a
+generated Cloudflare Worker (CORS + HTTP Range + `.partN` stitching), and
+fetched back byte-verified.
+
+The core is stdlib-only; the spec-shaped commands need PyYAML (extra: `spec`).
+"""
+
+__version__ = "0.1.0"

awrtifact/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

awrtifact/backup.py

@@ -0,0 +1,92 @@
+"""`awrtifact backup-catalog` — dispatch the mirror workflow for gaps.
+
+The point of the precaution: a spec with `source_url` per artifact is the
+backup MANIFEST — every artifact listed has a named, size-checked origin and
+a target release. This command turns the spec into workflow dispatches for
+artifacts whose parts are NOT yet in the release (the resumable plan check),
+so the mirror can be brought up deliberately and capped (the workflow's own
+≤20-runner fan-out; the origin box's uplink is never involved).
+
+`--dry-run` prints the dispatches; without it, dispatches via `gh workflow
+run`. The workflow must exist in the store's repo — this checks, because a
+dispatch to a missing workflow fails as a SILENCE (gh prints a warning and
+exits 0 — measured class; `gh workflow run` on an unknown workflow does not
+fail the caller).
+"""
+
+from __future__ import annotations
+
+import sys
+
+from . import gh
+from . import spec as spec_mod
+
+DEFAULT_WORKFLOW = "mirror-hf-to-release.yml"
+
+
+def backup_catalog(
+    spec: dict,
+    workflow: str = DEFAULT_WORKFLOW,
+    dry_run: bool = False,
+) -> dict:
+    """Dispatch mirror runs for artifacts missing from their releases."""
+    repo = spec["store"]["repo"]
+    if not dry_run and not gh.workflow_exists(repo, workflow):
+        raise ValueError(
+            f"workflow {workflow} does not exist in {repo} — cannot dispatch; "
+            f"re-push the workflow first"
+        )
+    planned: list[dict] = []
+    local_only: list[str] = []
+    for art in spec.get("artifacts") or []:
+        name = art["name"]
+        if not art.get("source_url"):
+            # Local-only artifact: no cloud lane can mirror it; it is a
+            # `awrtifact mirror <local-path>` candidate. A None source_url
+            # reaching the dispatch would upload parts named "None.partN"
+            # (the measured 2026-08-27 incident) — skipped and reported.
+            local_only.append(name)
+            continue
+        release = spec_mod.artifact_release(spec, art)
+        existing = gh.release_assets(repo, release)
+        parts = spec_mod.chunked_map(spec).get(name)
+        if not parts:
+            # Whole-file artifact: present when the asset itself exists.
+            if name in existing:
+                continue
+            inputs = {
+                "hf_url": art["source_url"],
+                "name": name,
+                "total_bytes": str(art["total"]),
+                "release": release,
+            }
+            planned.append({"artifact": name, "inputs": inputs})
+            continue
+        missing = [p["name"] for p in parts["parts"] if p["name"] not in existing]
+        if not missing:
+            continue
+        inputs = {
+            "hf_url": art["source_url"],
+            "name": name,
+            "total_bytes": str(art["total"]),
+            "release": release,
+            "part_size": str(art.get("part_size") or 1900000000),
+        }
+        planned.append(
+            {"artifact": name, "inputs": inputs, "missing_parts": len(missing)}
+        )
+    if dry_run:
+        for item in planned:
+            print(f"dispatch {item['artifact']}: {item['inputs']}", file=sys.stderr)
+    else:
+        for item in planned:
+            gh.workflow_dispatch(repo, workflow, item["inputs"])
+            print(f"dispatched {item['artifact']} -> {workflow} in {repo}")
+    return {
+        "dispatched": len(planned),
+        "local_only": local_only,
+        "already_present": len(
+            [a for a in spec.get("artifacts") or [] if a["name"] not in
+             {i["artifact"] for i in planned} and a["name"] not in local_only]
+        ),
+    }

awrtifact/cli.py

@@ -0,0 +1,253 @@
+"""awrtifact CLI — the deliberate chunk-and-release loop.
+
+    awrtifact split FILE [--part-size N] [--out DIR]
+    awrtifact plan MANIFEST --repo OWNER/REPO --release TAG
+    awrtifact upload MANIFEST --repo OWNER/REPO --release TAG [--dir DIR] [--parallel N] [--create]
+    awrtifact verify MANIFEST [--dir DIR]
+    awrtifact fetch NAME --url BASE --out DIR [--expected N] [--verify-only]
+    awrtifact serve-spec SPEC [--emit-dir DIR] [--check]
+    awrtifact backup-catalog SPEC [--workflow W] [--dry-run]
+
+Exit codes: 0 ok · 1 operational failure (upload failed, fetch short, drift) ·
+2 usage or data error (bad manifest/spec, missing dependency).
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__, gh
+from . import backup as backup_mod
+from . import fetch as fetch_mod
+from . import manifest as manifest_mod
+from . import mirror as mirror_mod
+from . import plan as plan_mod
+from . import serve_spec as serve_spec_mod
+from . import split as split_mod
+from . import upload as upload_mod
+from . import verify as verify_mod
+
+
+def _die(message: str, code: int = 2) -> int:
+    print(f"awrtifact: {message}", file=sys.stderr)
+    return code
+
+
+def _cmd_split(args: argparse.Namespace) -> int:
+    try:
+        m = split_mod.split_file(Path(args.file), args.part_size,
+                                 Path(args.out) if args.out else None)
+    except (ValueError, OSError) as exc:
+        return _die(str(exc))
+    out = Path(args.out) if args.out else Path(args.file).parent
+    manifest_path = out / "manifest.json"
+    manifest_mod.write(m, manifest_path)
+    print(f"split {m['name']}: {m['total']} bytes -> "
+          f"{len(m['parts'])} parts, manifest {manifest_path}")
+    return 0
+
+
+def _cmd_plan(args: argparse.Namespace) -> int:
+    try:
+        m = manifest_mod.load(Path(args.manifest))
+        planned = plan_mod.plan_parts(m, args.repo, args.release)
+    except (ValueError, OSError) as exc:
+        return _die(str(exc))
+    gh.print_json(planned)
+    return 0
+
+
+def _cmd_upload(args: argparse.Namespace) -> int:
+    try:
+        m = manifest_mod.load(Path(args.manifest))
+        result = upload_mod.upload_manifest(
+            m, args.repo, args.release,
+            Path(args.dir) if args.dir else Path(args.manifest).parent,
+            parallel=args.parallel, create=args.create,
+        )
+    except (ValueError, OSError) as exc:
+        return _die(str(exc))
+    print(f"uploaded {len(result['uploaded'])} part(s); "
+          f"{len(result['skipped_present'])} already present")
+    if result["failed"]:
+        for f in result["failed"]:
+            print(f"  FAILED: {f}", file=sys.stderr)
+        return 1
+    return 0
+
+
+def _cmd_verify(args: argparse.Namespace) -> int:
+    try:
+        m = manifest_mod.load(Path(args.manifest))
+        report = verify_mod.verify_manifest(m, Path(args.dir) if args.dir
+                                            else Path(args.manifest).parent)
+    except (ValueError, OSError) as exc:
+        return _die(str(exc))
+    if args.json:
+        gh.print_json(report)
+    else:
+        for check in report["checks"]:
+            mark = "ok" if check["ok"] else "FAIL " + ", ".join(check["errors"])
+            print(f"  {check['part']}: {mark}")
+        whole = report["whole"]
+        mark = "ok" if whole["ok"] else "FAIL " + ", ".join(whole["errors"])
+        print(f"  whole-file: {mark}")
+    return 0 if report["ok"] else 1
+
+
+def _cmd_fetch(args: argparse.Namespace) -> int:
+    try:
+        result = fetch_mod.fetch(
+            args.name, args.url, Path(args.out),
+            expected=args.expected, lockfile=Path(args.lock) if args.lock else None,
+            verify_only=args.verify_only,
+        )
+    except (ValueError, OSError, fetch_mod.FetchError) as exc:
+        return _die(str(exc), 1)
+    print(f"{result['status']}: {result['path']} ({result['bytes']} bytes, "
+          f"sha256 {result['sha256'][:16]}…)")
+    return 0 if result["status"] in ("fetched", "verified", "up-to-date") else 1
+
+
+def _cmd_serve_spec(args: argparse.Namespace) -> int:
+    spec_path = Path(args.spec)
+    try:
+        dirs = serve_spec_mod.emit(spec_mod_load(spec_path), spec_path, check=args.check)
+    except ValueError as exc:
+        return _die(str(exc))
+    if args.check:
+        print(f"spec {spec_path}: generated workers are current")
+    else:
+        for d in dirs:
+            print(f"generated worker -> {d}")
+    return 0
+
+
+def spec_mod_load(path: Path) -> dict:
+    # Imported lazily so the core commands stay stdlib-only; the spec module
+    # raises a clear "pip install awrtifact[spec]" when PyYAML is absent.
+    from . import spec as spec_mod  # noqa: PLC0415 — optional dependency
+
+    return spec_mod.load(path)
+
+
+def _cmd_mirror(args: argparse.Namespace) -> int:
+    try:
+        if mirror_mod.URL_RE.match(args.source):
+            result = mirror_mod.mirror_url(
+                args.source, args.name, args.release, args.repo,
+                total=args.total, workflow=args.workflow,
+            )
+        else:
+            result = mirror_mod.mirror_file(
+                Path(args.source), args.release, args.repo,
+                name=args.name, parallel=args.parallel,
+            )
+    except (ValueError, OSError) as exc:
+        return _die(str(exc))
+    print(f"mirror {result['lane']}: {result}")
+    return 0
+
+
+def _cmd_backup(args: argparse.Namespace) -> int:
+    try:
+        spec = spec_mod_load(Path(args.spec))
+        result = backup_mod.backup_catalog(
+            spec, workflow=args.workflow, dry_run=args.dry_run
+        )
+    except (ValueError, OSError) as exc:
+        return _die(str(exc))
+    print(f"planned {result['dispatched']} dispatch(es); "
+          f"{result['already_present']} artifact(s) already in their releases")
+    if result.get("local_only"):
+        print("local-only (no cloud source — mirror from the local path):")
+        for name in result["local_only"]:
+            print(f"  {name}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="awrtifact",
+        description="Deliberately chunk artifacts into GitHub release assets "
+        "and fetch them back byte-verified.",
+    )
+    parser.add_argument("--version", action="version", version=__version__)
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("split", help="split a file into .partN slices + manifest")
+    p.add_argument("file")
+    p.add_argument("--part-size", type=int, default=1900000000)
+    p.add_argument("--out")
+    p.set_defaults(func=_cmd_split)
+
+    p = sub.add_parser("plan", help="which parts are missing from the release")
+    p.add_argument("manifest")
+    p.add_argument("--repo", required=True)
+    p.add_argument("--release", required=True)
+    p.set_defaults(func=_cmd_plan)
+
+    p = sub.add_parser("upload", help="upload missing parts (resumable)")
+    p.add_argument("manifest")
+    p.add_argument("--repo", required=True)
+    p.add_argument("--release", required=True)
+    p.add_argument("--dir")
+    p.add_argument("--parallel", type=int, default=1)
+    p.add_argument("--create", action="store_true")
+    p.set_defaults(func=_cmd_upload)
+
+    p = sub.add_parser("verify", help="verify local parts against the manifest")
+    p.add_argument("manifest")
+    p.add_argument("--dir")
+    p.add_argument("--json", action="store_true")
+    p.set_defaults(func=_cmd_verify)
+
+    p = sub.add_parser("fetch", help="resumable, verified download")
+    p.add_argument("name")
+    p.add_argument("--url", required=True)
+    p.add_argument("--out", required=True)
+    p.add_argument("--expected", type=int)
+    p.add_argument("--lock")
+    p.add_argument("--verify-only", action="store_true")
+    p.set_defaults(func=_cmd_fetch)
+
+    p = sub.add_parser("serve-spec", help="generate the worker from the spec")
+    p.add_argument("spec")
+    p.add_argument("--emit-dir")
+    p.add_argument("--check", action="store_true")
+    p.set_defaults(func=_cmd_serve_spec)
+
+    p = sub.add_parser(
+        "mirror",
+        help="feed it a URL or file — it mirrors to GitHub seamlessly",
+    )
+    p.add_argument("source", help="Range-serving URL or a local file path")
+    p.add_argument("--release", required=True, help="release tag to upload into")
+    p.add_argument("--repo", default="Aitherium/aitherkvcache")
+    p.add_argument("--name", help="served asset name (default: URL basename / filename)")
+    p.add_argument("--total", type=int, help="declared size; fail loud on mismatch")
+    p.add_argument("--workflow", default=mirror_mod.DEFAULT_WORKFLOW)
+    p.add_argument("--parallel", type=int, default=4)
+    p.set_defaults(func=_cmd_mirror)
+
+    p = sub.add_parser("backup-catalog", help="dispatch mirror runs for gaps")
+    p.add_argument("spec")
+    p.add_argument("--workflow", default=backup_mod.DEFAULT_WORKFLOW)
+    p.add_argument("--dry-run", action="store_true")
+    p.set_defaults(func=_cmd_backup)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        return args.func(args)
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())

awrtifact/fetch.py

@@ -0,0 +1,174 @@
+"""`awrtifact fetch` — resumable, size-verified, TOFU-hashed download.
+
+Ported from the tenant fetch lane (`.DEPLOYMENT/templates/tenant-repo/models/
+fetch_models.py`) because that script's lessons are the point:
+
+1. RESUME. These are multi-GB streams. A restarted download continues from
+   the byte count already on disk via `Range: bytes=<have>-`, and a server
+   that answers non-206 to a resume restarts from byte 0 explicitly.
+2. SIZE VERIFICATION. A truncated artifact is not a download error — the
+   loader reports a corrupt model, which reads like a bad quant rather than
+   a short file. Every download is checked against the expected size before
+   it is accepted.
+3. STITCHED ASSETS ARE TRANSPARENT. Files over GitHub's 2 GiB cap live
+   upstream as `.partN` slices; the worker reassembles them behind the
+   original filename. Ask for the original name and it works.
+4. TOFU SHA256. The first successful fetch records the digest into the
+   lockfile; every later fetch (or --verify-only) compares. That makes a
+   silently-changed mirror asset visible on the second machine. It is
+   trust-on-first-use, not a signed digest, and the output says so.
+
+MAX_STALLS is the real stop condition: attempts that make no progress.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+CHUNK = 8 * 1024 * 1024
+MAX_ATTEMPTS = 200
+MAX_STALLS = 4
+UA = "awrtifact-fetch/1"
+
+
+class FetchError(RuntimeError):
+    """A download failed honestly (not a silent truncation)."""
+
+
+def _resume_headers(have: int) -> dict[str, str]:
+    return {"Range": f"bytes={have}-", "User-Agent": UA} if have else {"User-Agent": UA}
+
+
+def _download_once(url: str, dest: Path, expected: int | None) -> int:
+    """One attempt; returns bytes present on disk after the attempt."""
+    have = dest.stat().st_size if dest.exists() else 0
+    if expected is not None and have > expected:
+        raise FetchError(
+            f"on-disk {dest.name} is {have} bytes, larger than expected "
+            f"{expected} — remove it or fetch elsewhere"
+        )
+    headers = _resume_headers(have)
+    req = urllib.request.Request(url, headers=headers)
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:  # noqa: S310 — https checked by caller
+            status = getattr(resp, "status", 200)
+            if have and status != 206:
+                # Server ignored the resume; start over rather than append.
+                have = 0
+            if have:
+                with open(dest, "ab") as f:
+                    while True:
+                        chunk = resp.read(CHUNK)
+                        if not chunk:
+                            break
+                        f.write(chunk)
+            else:
+                with open(dest, "wb") as f:
+                    while True:
+                        chunk = resp.read(CHUNK)
+                        if not chunk:
+                            break
+                        f.write(chunk)
+    except urllib.error.HTTPError as exc:
+        if exc.code in (416,):
+            # Range not satisfiable: the server has more than we think.
+            raise FetchError(f"server 416 — {dest.name} may be complete here "
+                             f"and short at the mirror") from exc
+        raise FetchError(f"HTTP {exc.code} fetching {dest.name}") from exc
+    except urllib.error.URLError as exc:
+        raise FetchError(f"network error fetching {dest.name}: {exc.reason}") from exc
+    return dest.stat().st_size
+
+
+def _load_lock(path: Path) -> dict:
+    if not path.is_file():
+        return {}
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    return raw if isinstance(raw, dict) else {}
+
+
+def _save_lock(path: Path, lock: dict) -> None:
+    path.write_text(json.dumps(lock, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        while True:
+            chunk = f.read(CHUNK)
+            if not chunk:
+                break
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def fetch(
+    name: str,
+    url: str,
+    dest_dir: Path,
+    expected: int | None,
+    lockfile: Path | None = None,
+    verify_only: bool = False,
+) -> dict:
+    """Fetch `name` from `url` into `dest_dir`, size- and hash-verified.
+
+    `expected` is the manifest's total (or None to trust Content-Length).
+    Returns {"path", "bytes", "sha256", "status"} where status is one of
+    fetched / verified / up-to-date / sha-mismatch.
+    """
+    dest_dir = Path(dest_dir)
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    dest = dest_dir / name
+    lock_path = Path(lockfile) if lockfile else dest_dir / "awrtifact.lock.json"
+    lock = _load_lock(lock_path)
+    known = lock.get(name)
+
+    if verify_only and known and dest.is_file():
+        got = _sha256(dest)
+        if got == known["sha256"]:
+            return {"path": str(dest), "bytes": dest.stat().st_size,
+                    "sha256": got, "status": "verified"}
+        return {"path": str(dest), "bytes": dest.stat().st_size,
+                "sha256": got, "status": "sha-mismatch"}
+
+    attempts = 0
+    stalls = 0
+    last = dest.stat().st_size if dest.exists() else 0
+    while attempts < MAX_ATTEMPTS and stalls < MAX_STALLS:
+        attempts += 1
+        now = _download_once(url, dest, expected)
+        if expected is not None and now > expected:
+            raise FetchError(
+                f"{name}: server delivered more than the expected {expected} "
+                f"bytes — the mirror is serving a different artifact"
+            )
+        if now == last:
+            stalls += 1
+        else:
+            stalls = 0
+        last = now
+        if expected is not None and now == expected:
+            break
+    if expected is not None and last != expected:
+        raise FetchError(
+            f"{name}: stopped at {last} of {expected} bytes — the bytes on "
+            f"disk are valid; re-run to resume from here"
+        )
+
+    got = _sha256(dest)
+    if known:
+        if got != known["sha256"]:
+            return {"path": str(dest), "bytes": last, "sha256": got,
+                    "status": "sha-mismatch"}
+        lock[name]["bytes"] = last
+    else:
+        lock[name] = {"sha256": got, "bytes": last}
+    _save_lock(lock_path, lock)
+    return {"path": str(dest), "bytes": last, "sha256": got, "status": "fetched"}

awrtifact/gh.py

@@ -0,0 +1,132 @@
+"""`gh` CLI wrapper — the smallest shell around GitHub's release API.
+
+Deliberately shells the ambient `gh` CLI (like the lanes it productizes —
+`seed-q1-mirror.ps1`, `build_webml_cdn.mjs`, `mirror-hf-to-release.yml` all
+use it): the operator's existing auth is the auth, no token handling here.
+Every call is size-bounded (release asset lists) and text-decoded explicitly.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from typing import Sequence
+
+
+class GhError(RuntimeError):
+    """A gh invocation failed — message carries gh's stderr."""
+
+
+def _run(args: Sequence[str]) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        ["gh", *args],
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        check=False,
+    )
+
+
+def release_assets(repo: str, release: str) -> set[str]:
+    """Asset names currently in the release. Empty set on a missing release."""
+    proc = _run(
+        [
+            "release",
+            "view",
+            release,
+            "--repo",
+            repo,
+            "--json",
+            "assets",
+            "--jq",
+            ".assets[].name",
+        ]
+    )
+    if proc.returncode != 0:
+        return set()
+    names = {line.strip() for line in proc.stdout.splitlines() if line.strip()}
+    return names
+
+
+def release_exists(repo: str, release: str) -> bool:
+    proc = _run(["release", "view", release, "--repo", repo])
+    return proc.returncode == 0
+
+
+def create_release(repo: str, release: str, title: str, notes: str) -> None:
+    proc = _run(
+        [
+            "release",
+            "create",
+            release,
+            "--repo",
+            repo,
+            "--title",
+            title,
+            "--notes",
+            notes,
+        ]
+    )
+    if proc.returncode != 0:
+        raise GhError(f"gh release create {release}: {proc.stderr.strip()}")
+
+
+def upload(repo: str, release: str, path: str) -> None:
+    """Upload one asset with --clobber (idempotent re-upload)."""
+    proc = _run(
+        ["release", "upload", release, path, "--repo", repo, "--clobber"]
+    )
+    if proc.returncode != 0:
+        raise GhError(f"gh release upload {path}: {proc.stderr.strip()}")
+
+
+def workflow_dispatch(
+    repo: str, workflow: str, inputs: dict[str, str]
+) -> None:
+    """Fire a workflow_dispatch with string inputs (mirror lane)."""
+    args = ["workflow", "run", workflow, "--repo", repo]
+    for key, value in sorted(inputs.items()):
+        args.extend(["-f", f"{key}={value}"])
+    proc = _run(args)
+    if proc.returncode != 0:
+        raise GhError(f"gh workflow run {workflow}: {proc.stderr.strip()}")
+
+
+def asset_sizes(repo: str, release: str) -> dict[str, int]:
+    """name -> size for the release's assets (verify/plan use this)."""
+    proc = _run(
+        [
+            "release",
+            "view",
+            release,
+            "--repo",
+            repo,
+            "--json",
+            "assets",
+            "--jq",
+            '.assets[] | "\\(.name)\t\\(.size)"',
+        ]
+    )
+    if proc.returncode != 0:
+        return {}
+    out: dict[str, int] = {}
+    for line in proc.stdout.splitlines():
+        if "\t" not in line:
+            continue
+        name, size = line.split("\t", 1)
+        try:
+            out[name] = int(size)
+        except ValueError:
+            continue
+    return out
+
+
+def workflow_exists(repo: str, workflow: str) -> bool:
+    """Is the named workflow present in the repo (backup-catalog gate)?"""
+    proc = _run(["workflow", "view", workflow, "--repo", repo])
+    return proc.returncode == 0
+
+
+def print_json(payload: dict) -> None:
+    print(json.dumps(payload, indent=2, sort_keys=True))