acadfetch 0.1.0__py3-none-any.whl

acadfetch/downloader.py

@@ -0,0 +1,289 @@
+"""The transfer engine.
+
+Resume is built on three ideas:
+
+* bytes always land in ``<name>.part`` and are renamed only after verification,
+  so a killer ``Ctrl-C`` never leaves a truncated file looking finished;
+* the validator (ETag / Last-Modified) lives in a sidecar so ``If-Range`` can
+  prove the remote object did not change between runs;
+* the retry loop re-uses the partial file instead of restarting, which makes a
+  mid-stream socket drop cost at most one chunk.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+from . import naming, verify
+from .client import Client
+from .consts import CHUNK
+from .errors import HttpError, PartialError
+from .progress import Progress
+from .sources.base import File, Record, Result
+from .units import human
+
+TRANSFER_ATTEMPTS = 3
+META_SUFFIX = ".part.meta.json"
+
+
+@dataclass
+class Options:
+    out: Path
+    concurrency: int = 3
+    retries: int = TRANSFER_ATTEMPTS
+    dry_run: bool = False
+    max_bytes: int | None = None
+    progress: Progress | None = None
+    verify_only: bool = False
+
+
+def _read_meta(path: Path) -> dict:
+    try:
+        return json.loads(path.read_text("utf-8"))
+    except (OSError, ValueError):
+        return {}
+
+
+def _write_meta(path: Path, data: dict) -> None:
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps(data), "utf-8")
+    os.replace(tmp, path)
+
+
+def _drop(*paths: Path) -> None:
+    for path in paths:
+        with contextlib.suppress(OSError):
+            path.unlink()
+
+
+def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Options) -> int:
+    """Move bytes into ``part``, resuming if possible. Returns bytes written."""
+    meta = _read_meta(meta_path) if part.exists() else {}
+    if meta.get("url") not in (None, file.url):
+        _drop(part, meta_path)
+        meta = {}
+
+    have = part.stat().st_size if part.exists() else 0
+    if file.size is not None and have > file.size:
+        _drop(part, meta_path)
+        have = 0
+
+    headers = {}
+    if have:
+        headers["Range"] = f"bytes={have}-"
+        validator = meta.get("etag") or meta.get("last_modified")
+        if validator:
+            headers["If-Range"] = validator
+
+    written = 0
+    resumed = 0
+    try:
+        resp = client.open(file.url, headers)
+    except HttpError as exc:
+        if exc.status == 416 and have:
+            return 0  # already complete
+        raise
+
+    with resp:
+        status = getattr(resp, "status", 200)
+        if have and status == 206:
+            mode, offset, resumed = "ab", have, have
+        else:
+            mode, offset = "wb", 0
+        _write_meta(
+            meta_path,
+            {
+                "url": file.url,
+                "etag": resp.headers.get("ETag"),
+                "last_modified": resp.headers.get("Last-Modified"),
+                "size": file.size,
+            },
+        )
+        declared = resp.headers.get("Content-Length")
+        with part.open(mode) as fh:
+            while True:
+                chunk = resp.read(CHUNK)
+                if not chunk:
+                    break
+                fh.write(chunk)
+                written += len(chunk)
+                if opts.progress:
+                    opts.progress.advance(len(chunk))
+                if opts.max_bytes and offset + written > opts.max_bytes:
+                    raise HttpError(0, file.url, f"exceeds --max-size ({human(opts.max_bytes)})")
+        if declared is not None and int(declared) != written:
+            raise HttpError(0, file.url, f"short read: got {written}, expected {declared}")
+    if resumed:
+        return written
+    return written
+
+
+def fetch_file(client: Client, file: File, opts: Options, taken: set[str]) -> Result:
+    name = naming.unique(naming.sanitize(file.name), taken)
+    final = opts.out / name
+    part = opts.out / (name + ".part")
+    meta_path = opts.out / (name + META_SUFFIX)
+
+    if final.exists():
+        verdict = verify.matches(final, file.checksum)
+        sized_ok = file.size is None or final.stat().st_size == file.size
+        if verdict is True or (verdict is None and sized_ok):
+            return Result(
+                file=file,
+                state="skipped",
+                path=str(final),
+                bytes=final.stat().st_size,
+                verified=verdict,
+            )
+
+    if opts.dry_run:
+        return Result(file=file, state="skipped", path=str(final), verified=None)
+
+    resumed_from = part.stat().st_size if part.exists() else 0
+    progress = opts.progress
+    if progress:
+        progress.start(name, file.size)
+    try:
+        return _attempt(client, file, part, meta_path, final, opts, resumed_from)
+    finally:
+        if progress:
+            progress.stop(name)
+
+
+def _attempt(
+    client: Client,
+    file: File,
+    part: Path,
+    meta_path: Path,
+    final: Path,
+    opts: Options,
+    resumed_from: int,
+) -> Result:
+    """Retry loop, verification and final rename for one file.
+
+    A 403 that looks like a WAF block page is retried a couple of times with
+    the partial file kept, then the file's fallback URLs are tried (a legacy
+    endpoint for the same bytes). Everything else terminal fails immediately.
+    """
+    last: Exception | None = None
+    blocked = 0
+    candidates = [file.url, *file.fallbacks]
+    done = False
+    for index, url in enumerate(candidates):
+        source = file if url == file.url else replace(file, url=url)
+        for attempt in range(opts.retries + 1):
+            try:
+                _transfer(client, source, part, meta_path, opts)
+                done = True
+                break
+            except HttpError as exc:
+                last = exc
+                if exc.status == 0 and "exceeds --max-size" in str(exc):
+                    return Result(file=file, state="failed", error=str(exc))
+                if exc.status == 403 and exc.transient and blocked < client.block_retries:
+                    blocked += 1
+                elif exc.status in (401, 403, 404, 410):
+                    break  # terminal for this URL; a fallback may still work
+            except (OSError, ConnectionError) as exc:
+                last = exc
+            if attempt < opts.retries:
+                time.sleep(min(2**attempt, 10))
+                resumed_from = part.stat().st_size if part.exists() else 0
+        if done:
+            break
+        if index < len(candidates) - 1:
+            _drop(part, meta_path)
+            resumed_from = 0
+            continue
+        _drop(part, meta_path)
+        return Result(file=file, state="failed", error=str(last))
+
+    if last is not None and not part.exists():
+        return Result(file=file, state="failed", error=str(last))
+
+    if file.size is not None and part.exists() and part.stat().st_size != file.size:
+        return Result(
+            file=file,
+            state="failed",
+            error=f"size mismatch (want {file.size})",
+            bytes=part.stat().st_size,
+        )
+
+    verdict = verify.matches(part, file.checksum)
+    if verdict is False:
+        _drop(part, meta_path)
+        try:
+            _transfer(client, file, part, meta_path, opts)
+            verdict = verify.matches(part, file.checksum)
+        except Exception as exc:  # noqa: BLE001
+            return Result(file=file, state="failed", error=f"re-download failed: {exc}")
+        if verdict is False:
+            _drop(part, meta_path)
+            return Result(file=file, state="failed", error="checksum mismatch")
+
+    os.replace(part, final)
+    _drop(meta_path)
+    return Result(
+        file=file,
+        state="downloaded",
+        path=str(final),
+        bytes=final.stat().st_size,
+        verified=verdict,
+        resumed_from=resumed_from,
+    )
+
+
+def fetch_record(
+    record: Record, opts: Options, client: Client, only: list[str] | None = None
+) -> list[Result]:
+    opts.out.mkdir(parents=True, exist_ok=True)
+    files = record.files
+    if only:
+        wanted = {n.lower() for n in only}
+        files = [
+            f
+            for f in files
+            if naming.sanitize(f.name).lower() in wanted or f.name.lower() in wanted
+        ]
+        if not files:
+            raise PartialError([])
+
+    if opts.max_bytes:
+        _enforce_total(record, opts.max_bytes)
+
+    taken: set[str] = set()
+    lock = threading.Lock()
+    results: list[Result] = []
+
+    if opts.concurrency <= 1:
+        for file in files:
+            results.append(fetch_file(client, file, opts, taken))
+        return results
+
+    with ThreadPoolExecutor(max_workers=opts.concurrency) as pool:
+        futures = {}
+        for file in files:
+            with lock:
+                futures[pool.submit(fetch_file, client, file, opts, taken)] = file
+        for future in as_completed(futures):
+            results.append(future.result())
+    order = {id(f): i for i, f in enumerate(record.files)}
+    results.sort(key=lambda r: order.get(id(r.file), 0))
+    return results
+
+
+def _enforce_total(record: Record, limit: int) -> None:
+    total = record.total_bytes
+    if total is None:
+        return
+    if total > limit:
+        from .errors import SourceError
+
+        raise SourceError(f"record is {human(total)}, over --max-size ({human(limit)})")

acadfetch/errors.py

@@ -0,0 +1,43 @@
+"""Exception types. Every failure mode gets a distinct exit code path."""
+
+from __future__ import annotations
+
+
+class AcFError(Exception):
+    """Base class for all acadfetch errors."""
+
+
+class TargetError(AcFError):
+    """The target string could not be parsed or resolved."""
+
+
+class SourceError(AcFError):
+    """A repository API returned an unusable response."""
+
+
+class HttpError(AcFError):
+    """An HTTP request failed after all retries.
+
+    ``transient`` marks a failure that is worth retrying further up, currently
+    only a 403 that came from a WAF/load-balancer block page rather than from
+    the API's own permission check.
+    """
+
+    def __init__(self, status: int, url: str, detail: str = "", transient: bool = False) -> None:
+        self.status = status
+        self.url = url
+        self.transient = transient
+        super().__init__(f"HTTP {status} for {url}{': ' + detail if detail else ''}")
+
+
+class VerifyError(AcFError):
+    """Downloaded bytes do not match the checksum published by the source."""
+
+
+class PartialError(AcFError):
+    """One or more files failed while others succeeded."""
+
+    def __init__(self, failures) -> None:  # noqa: ANN001
+        self.failures = list(failures)
+        names = ", ".join(f.name for f in self.failures)
+        super().__init__(f"{len(self.failures)} file(s) failed: {names}")

acadfetch/manifest.py

@@ -0,0 +1,89 @@
+"""manifest.json: the reproducibility artifact.
+
+The point of a manifest is that someone else (or you, in two years) can prove
+the bytes on disk are the bytes that were published, and can tell which
+license they are allowed to redistribute under.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+from . import consts, naming, verify
+from .auth import Token
+from .sources.base import Record, Result
+
+SCHEMA = 1
+
+
+def build(record: Record, results: list[Result], token: Token | None = None) -> dict:
+    by_url = {r.file.url: r for r in results}
+    return {
+        "schema": SCHEMA,
+        "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        "tool": {"name": "acadfetch", "version": consts.__version__},
+        "source": record.source,
+        "target": record.target,
+        "doi": record.doi,
+        "title": record.title,
+        "license": record.license,
+        "publisher": record.publisher,
+        "version": record.version,
+        "landing": record.landing,
+        # Provenance only: the variable name and a tail, never the secret.
+        "auth": {"env": token.env, "hint": token.hint} if token else None,
+        "files": [
+            {
+                "name": naming.sanitize(f.name),
+                "url": f.url,
+                "size": f.size,
+                "checksum": f.checksum,
+                "sha256": _sha256(by_url.get(f.url)),
+                "bytes": by_url[f.url].bytes if f.url in by_url else 0,
+                "state": by_url[f.url].state if f.url in by_url else "missing",
+                "verified": by_url[f.url].verified if f.url in by_url else None,
+            }
+            for f in record.files
+        ],
+    }
+
+
+def _sha256(result: Result | None) -> str | None:
+    if result is None or not result.path:
+        return None
+    path = Path(result.path)
+    if not path.is_file():
+        return None
+    return verify.hash_file(path, "sha256")
+
+
+def write(path: Path, data: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", "utf-8")
+    os.replace(tmp, path)
+
+
+def load(path: Path) -> dict:
+    data = json.loads(path.read_text("utf-8"))
+    if data.get("schema") != SCHEMA:
+        raise ValueError(f"unsupported manifest schema: {data.get('schema')!r}")
+    return data
+
+
+def verify_manifest(path: Path, root: Path) -> list[dict]:
+    """Re-hash every recorded file. Returns one row per file."""
+    data = load(path)
+    rows = []
+    for entry in data.get("files") or []:
+        target = root / entry["name"]
+        if not target.is_file():
+            rows.append({"name": entry["name"], "status": "missing"})
+            continue
+        actual = verify.hash_file(target, "sha256")
+        status = "ok" if actual == entry.get("sha256") else "MISMATCH"
+        rows.append({"name": entry["name"], "status": status, "sha256": actual})
+    return rows

acadfetch/naming.py

@@ -0,0 +1,88 @@
+"""Turn arbitrary repository file names into safe local paths.
+
+Repository metadata is untrusted: it can contain path separators, NULs,
+Windows reserved device names, 300-character names, and duplicates that
+would silently overwrite each other.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import unicodedata
+from urllib.parse import unquote, urlsplit
+
+MAX_STEM = 120
+MAX_SUFFIX = 20
+MAX_FULL = 180
+
+_ILLEGAL = re.compile(r'[<>:"/\\|?*\x00-\x1f\x7f]')
+_WS = re.compile(r"[ \t\n\r\f\v]+")
+
+# CON, NUL, COM1..9, LPT1..9 are unusable as file names on Windows even with
+# an extension, in every directory.
+_RESERVED = {
+    "CON",
+    "PRN",
+    "AUX",
+    "NUL",
+    *(f"COM{i}" for i in range(1, 10)),
+    *(f"LPT{i}" for i in range(1, 10)),
+}
+_RESERVED_WITH_EXT = {
+    f"{base}.{ext}" for base in _RESERVED for ext in ("txt", "tar", "gz", "zip", "csv")
+}
+
+
+_COMPRESS = {".gz", ".bz2", ".xz", ".zst", ".zip", ".7z", ".rar"}
+_EXT = re.compile(r"\.[A-Za-z0-9][A-Za-z0-9._-]*$")
+
+
+def _split_ext(name: str) -> tuple[str, str]:
+    """Split one extension, or a two-part one for ``archive.tar.gz`` style names."""
+    stem, ext = os.path.splitext(name)
+    if not ext or len(ext) > MAX_SUFFIX or not _EXT.fullmatch(ext):
+        return name, ""
+    if ext.lower() in _COMPRESS:
+        inner_stem, inner_ext = os.path.splitext(stem)
+        if inner_stem and _EXT.fullmatch(inner_ext) and len(inner_ext) <= MAX_SUFFIX:
+            return inner_stem, inner_ext + ext
+    return stem, ext
+
+
+def sanitize(name: str, fallback: str = "file") -> str:
+    """Reduce ``name`` to a single safe path component."""
+    raw = unquote(name or "")
+    raw = urlsplit(raw).path if ("://" in raw or raw.count("/") > 1) else raw
+    raw = raw.replace("\\", "/").rsplit("/", 1)[-1]
+    raw = unicodedata.normalize("NFC", raw)
+    raw = _WS.sub(" ", raw)  # tabs/newlines become spaces, not underscores
+    raw = _ILLEGAL.sub("_", raw)
+    raw = _WS.sub(" ", raw).strip()
+    raw = raw.strip(". ")
+
+    stem, ext = _split_ext(raw)
+    stem = stem.strip(". ") or fallback
+    if stem.upper() in _RESERVED or f"{stem}{ext}".upper() in _RESERVED_WITH_EXT:
+        stem = "_" + stem
+    if len(stem) > MAX_STEM:
+        stem = stem[:MAX_STEM].rstrip(". ")
+    out = stem + ext[:MAX_SUFFIX]
+    if len(out) > MAX_FULL:
+        keep = MAX_FULL - len(ext[:MAX_SUFFIX])
+        out = stem[:keep].rstrip(". ") + ext[:MAX_SUFFIX]
+    return out or fallback
+
+
+def unique(name: str, taken: set[str]) -> str:
+    """Return a name not already in ``taken`` (case-insensitively)."""
+    if name.lower() not in taken:
+        taken.add(name.lower())
+        return name
+    stem, ext = os.path.splitext(name)
+    for i in range(2, 10000):
+        candidate = f"{stem} ({i}){ext}"
+        if candidate.lower() not in taken:
+            taken.add(candidate.lower())
+            return candidate
+    raise RuntimeError(f"could not dedupe {name!r}")

acadfetch/progress.py

@@ -0,0 +1,161 @@
+"""Aggregate transfer progress.
+
+One line for the whole run, not one per file: with ``-j 3`` three downloads are
+in flight and three interleaved lines are unreadable. On a TTY the line is
+redrawn in place; when stdout is a pipe (CI, ``tee``) it degrades to occasional
+whole lines so the log stays greppable.
+"""
+
+from __future__ import annotations
+
+import sys
+import threading
+import time
+from collections import deque
+from collections.abc import Callable
+from typing import TextIO
+
+from .units import duration, human
+
+SMOOTHING_SECONDS = 1.0
+TTY_INTERVAL = 0.15
+PIPE_INTERVAL = 5.0
+MAX_WIDTH = 120
+NAME_WIDTH = 40
+
+
+def _clip(text: str, width: int) -> str:
+    return text if len(text) <= width else text[: width - 1] + "…"
+
+
+class Progress:
+    def __init__(
+        self,
+        stream: TextIO | None = None,
+        enabled: bool = True,
+        is_tty: bool | None = None,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> None:
+        self.stream = stream if stream is not None else sys.stdout
+        self.enabled = enabled
+        self.is_tty = (self.stream.isatty() if is_tty is None else is_tty) and enabled
+        self._clock = clock
+        self._lock = threading.RLock()  # _render() re-enters under _maybe_emit
+        self._active: dict[str, int] = {}
+        self._done = 0
+        self._total = 0
+        self._samples: deque[tuple[float, int]] = deque()
+        self._last_emit = 0.0
+        self._printed = 0
+        self._last_count = 0
+        self._dirty = False
+        self._wrote = False
+
+    # -- structure ---------------------------------------------------------
+    def start(self, name: str, size: int | None) -> None:
+        if not self.enabled:
+            return
+        with self._lock:
+            self._active[name] = size or 0
+            self._total += size or 0
+            self._sample()
+        self._maybe_emit(force=True)
+
+    def stop(self, name: str) -> None:
+        if not self.enabled:
+            return
+        self._maybe_emit(force=True)  # one last frame that still counts the file
+        with self._lock:
+            self._active.pop(name, None)
+
+    def advance(self, count: int) -> None:
+        if not self.enabled or count <= 0:
+            return
+        with self._lock:
+            self._done += count
+            self._sample()
+        self._maybe_emit()
+
+    # -- rendering ---------------------------------------------------------
+    def _sample(self) -> None:
+        now = self._clock()
+        self._samples.append((now, self._done))
+        while len(self._samples) > 2 and now - self._samples[0][0] > SMOOTHING_SECONDS * 4:
+            self._samples.popleft()
+        self._dirty = True
+
+    def _speed(self) -> float:
+        if len(self._samples) < 2:
+            return 0.0
+        (t0, b0), (t1, b1) = self._samples[0], self._samples[-1]
+        span = t1 - t0
+        if span < 0.2:
+            return 0.0
+        return max(0.0, (b1 - b0) / span)
+
+    def _eta(self, speed: float) -> float | None:
+        if not self._total or speed <= 0:
+            return None
+        return max(0.0, (self._total - self._done) / speed)
+
+    def _render(self) -> str:
+        speed = self._speed()
+        with self._lock:
+            active = list(self._active)
+        # After the last file stops there is nothing active; keep showing how
+        # many files the run covered instead of dropping to "0 file(s)".
+        count = len(active) or self._last_count
+        self._last_count = count
+        parts = [f"{count} file(s)", f"{human(self._done)}"]
+        if len(active) == 1:
+            parts[0] = _clip(active[0], NAME_WIDTH)
+        if self._total:
+            parts[1] = f"{human(self._done)}/{human(self._total)}"
+        if speed > 0:
+            parts.append(f"{human(speed)}/s")
+            eta = self._eta(speed)
+            if eta is not None:
+                parts.append(f"ETA {duration(eta)}")
+        line = "  ".join(parts)
+        return line[:MAX_WIDTH]
+
+    def _maybe_emit(self, force: bool = False) -> None:
+        if not self.enabled:
+            return
+        now = self._clock()
+        interval = TTY_INTERVAL if self.is_tty else PIPE_INTERVAL
+        with self._lock:
+            if not self._dirty:
+                return
+            if not force and now - self._last_emit < interval:
+                return
+            line = self._render()
+            self._last_emit = now
+            self._dirty = False
+        self._write(line, newline=not self.is_tty)
+
+    def _write(self, line: str, newline: bool) -> None:
+        try:
+            if self.is_tty:
+                pad = max(0, self._printed - len(line))
+                self.stream.write("\r" + line + " " * pad)
+                self._printed = len(line)
+            else:
+                self.stream.write(line + ("\n" if newline else ""))
+            self.stream.flush()
+            self._wrote = True
+        except (OSError, ValueError):  # stream closed under us
+            self.enabled = False
+
+    def finish(self) -> None:
+        if not self.enabled:
+            return
+        # A run where every file was already on disk never started one, so a
+        # frame here would only say "0 file(s) 0 B" right before the tally.
+        if self._wrote:
+            line = self._render()
+            self._write(line, newline=True)
+            if self.is_tty:
+                self.stream.write("\n")
+                self.stream.flush()
+        self.enabled = False