acadfetch 0.1.0__py3-none-any.whl

acadfetch/__init__.py

@@ -0,0 +1,5 @@
+"""acadfetch: resumable, rate-limited, checksum-verified bulk downloads."""
+
+from .consts import __version__
+
+__all__ = ["__version__"]

acadfetch/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

acadfetch/auth.py

@@ -0,0 +1,61 @@
+"""Repository tokens, read from the environment.
+
+Putting a token in ``--header`` leaks it into shell history and into every
+``ps`` on the box. Each adapter instead declares the variable it reads, and the
+token is attached per-source so it never travels to a host that has no business
+seeing it.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+# source name -> variable that holds its token
+TOKEN_ENV = {
+    "zenodo": "ZENODO_TOKEN",
+    "figshare": "FIGSHARE_TOKEN",
+}
+
+
+@dataclass(frozen=True)
+class Token:
+    source: str
+    env: str
+    value: str
+
+    @property
+    def headers(self) -> dict[str, str]:
+        return {"Authorization": f"Bearer {self.value}"}
+
+    @property
+    def hint(self) -> str:
+        """Enough to tell two tokens apart, not enough to use one."""
+        return f"…{self.value[-4:]}" if len(self.value) > 8 else "…"
+
+    def __str__(self) -> str:
+        return f"${self.env} ({self.hint})"
+
+
+def load(environ: Mapping[str, str] | None = None) -> dict[str, Token]:
+    """Collect every token the environment provides, keyed by source."""
+    env = os.environ if environ is None else environ
+    found: dict[str, Token] = {}
+    for source, name in TOKEN_ENV.items():
+        value = (env.get(name) or "").strip()
+        if value:
+            found[source] = Token(source=source, env=name, value=value)
+    return found
+
+
+def headers(tokens: Mapping[str, Token], source: str | None) -> dict[str, str]:
+    token = tokens.get(source or "")
+    return token.headers if token else {}
+
+
+def describe(tokens: Mapping[str, Token], source: str | None) -> str:
+    token = tokens.get(source or "")
+    if token is None:
+        return ""
+    return f"token from {token}"

acadfetch/cli.py

@@ -0,0 +1,280 @@
+"""Command line interface.
+
+Exit codes are part of the contract: 0 all good, 1 everything failed or the
+target was unusable, 2 partial success, 130 interrupted (partials are kept).
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+from . import auth, consts, manifest, naming
+from .client import Client
+from .downloader import Options, fetch_record, human
+from .errors import AcFError, PartialError, TargetError
+from .progress import Progress
+from .sources import registry
+
+EXIT_OK, EXIT_FAIL, EXIT_PARTIAL, EXIT_INTERRUPT = 0, 1, 2, 130
+_SIZE = re.compile(r"^(\d+(?:\.\d+)?)\s*([KMGT]i?B?|B?)$", re.I)
+
+
+def parse_size(text: str) -> int:
+    hit = _SIZE.match(text.strip())
+    if not hit:
+        raise argparse.ArgumentTypeError(f"bad size: {text!r}")
+    value = float(hit.group(1))
+    unit = (hit.group(2) or "").upper().rstrip("B")
+    scale = {
+        "": 1,
+        "K": 1024,
+        "KI": 1024,
+        "M": 1 << 20,
+        "MI": 1 << 20,
+        "G": 1 << 30,
+        "GI": 1 << 30,
+        "T": 1 << 40,
+        "TI": 1 << 40,
+    }[unit]
+    return int(value * scale)
+
+
+def _count(text: str) -> int:
+    try:
+        value = int(text)
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"not a whole number: {text!r}") from None
+    if value < 0:
+        raise argparse.ArgumentTypeError(f"cannot retry {value} times")
+    return value
+
+
+def cmd_sources(_: argparse.Namespace) -> int:
+    for name in registry.names():
+        print(name)
+    return EXIT_OK
+
+
+def _client(args: argparse.Namespace) -> Client:
+    tokens = {} if getattr(args, "no_token", False) else auth.load()
+    args.tokens = tokens
+    return Client(
+        rate=args.rate,
+        retries=args.retries,
+        block_retries=args.retry_403,
+        headers=_headers(args.header),
+        auth={source: token.headers for source, token in tokens.items()},
+    )
+
+
+def _headers(raw: list[str] | None) -> dict[str, str]:
+    out = {}
+    for item in raw or []:
+        key, sep, value = item.partition(":")
+        if not sep:
+            raise TargetError(f"--header needs 'Name: value', got {item!r}")
+        out[key.strip()] = value.strip()
+    return out
+
+
+def _resolve(args: argparse.Namespace, client: Client):
+    record = registry.resolve(client, args.target)
+    return record
+
+
+def _token_for(args: argparse.Namespace, record):  # noqa: ANN001, ANN202
+    return getattr(args, "tokens", {}).get(record.source)
+
+
+def cmd_manifest(args: argparse.Namespace) -> int:
+    client = _client(args)
+    record = _resolve(args, client)
+    data = manifest.build(record, [], _token_for(args, record))
+    out = Path(args.out or "manifest.json")
+    manifest.write(out, data)
+    print(f"{len(record.files)} file(s) -> {out}")
+    _print_header(record)
+    if token := _token_for(args, record):
+        print(f"auth    : token from {token}")
+    return EXIT_OK
+
+
+def _print_header(record) -> None:  # noqa: ANN001
+    print(f"title   : {record.title or '(untitled)'}")
+    if record.doi:
+        print(f"doi     : {record.doi}")
+    print(f"license : {record.license or 'UNKNOWN - check before redistributing'}")
+    total = record.total_bytes
+    print(f"files   : {len(record.files)}" + (f", {human(total)}" if total is not None else ""))
+    if record.landing:
+        print(f"landing : {record.landing}")
+
+
+def cmd_fetch(args: argparse.Namespace) -> int:
+    client = _client(args)
+    record = _resolve(args, client)
+    _print_header(record)
+    token = _token_for(args, record)
+    if token:
+        print(f"auth    : token from {token}")
+
+    out_dir = Path(args.out or f"acadfetch-out/{record.source}-{_slug(record)}")
+    progress = Progress(enabled=not args.quiet)
+    opts = Options(
+        out=out_dir,
+        concurrency=args.concurrency,
+        dry_run=args.dry_run,
+        max_bytes=args.max_size,
+        progress=progress,
+    )
+
+    if args.dry_run:
+        for file in record.files:
+            print(f"  would fetch {naming.sanitize(file.name)} ({human(file.size)})")
+        return EXIT_OK
+
+    try:
+        results = fetch_record(record, opts, client, only=args.only)
+    except PartialError:
+        print(f"no file matched --only {args.only}", file=sys.stderr)
+        return EXIT_FAIL
+    except KeyboardInterrupt:
+        progress.finish()
+        raise
+    finally:
+        progress.finish()
+
+    failures = [r for r in results if not r.ok]
+    for row in results:
+        mark = {"downloaded": "ok  ", "skipped": "skip", "failed": "FAIL"}[row.state]
+        extra = ""
+        if row.resumed_from:
+            extra = f" (resumed at {human(row.resumed_from)})"
+        if row.verified is False:
+            extra += " [CHECKSUM MISMATCH]"
+        print(f"{mark} {naming.sanitize(row.file.name)} {human(row.bytes)}{extra}")
+        if row.error:
+            print(f"     {row.error}")
+
+    manifest.write(out_dir / "manifest.json", manifest.build(record, results, token))
+    print(f"\n{len(results) - len(failures)}/{len(results)} file(s) in {out_dir}")
+    if args.manifest_out:
+        manifest.write(Path(args.manifest_out), manifest.build(record, results, token))
+
+    if failures and len(failures) == len(results):
+        return EXIT_FAIL
+    return EXIT_PARTIAL if failures else EXIT_OK
+
+
+def cmd_verify(args: argparse.Namespace) -> int:
+    root = Path(args.root or ".")
+    rows = manifest.verify_manifest(Path(args.manifest), root)
+    bad = 0
+    for row in rows:
+        if row["status"] != "ok":
+            bad += 1
+        print(f"{row['status']:>9} {row['name']}")
+    print(f"\n{len(rows) - bad}/{len(rows)} ok")
+    return EXIT_FAIL if bad else EXIT_OK
+
+
+def _slug(record) -> str:  # noqa: ANN001
+    if record.doi:
+        return naming.sanitize(record.doi.replace("/", "_"))
+    return naming.sanitize(record.target.rsplit("/", 1)[-1] or "record")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="acadfetch",
+        description="Resumable, rate-limited, checksum-verified bulk downloads from data repositories.",
+    )
+    parser.add_argument("--version", action="version", version=f"acadfetch {consts.__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    def network_flags(p: argparse.ArgumentParser) -> None:
+        p.add_argument(
+            "--rate",
+            type=float,
+            default=consts.DEFAULT_RATE,
+            help="requests per second, global (default: %(default)s)",
+        )
+        p.add_argument(
+            "--retries",
+            type=int,
+            default=consts.DEFAULT_RETRIES,
+            help="retries per request on 5xx/429/network errors (default: %(default)s)",
+        )
+        p.add_argument(
+            "--retry-403",
+            type=_count,
+            default=consts.WAF_403_RETRIES,
+            metavar="N",
+            help="retries for a WAF block page, i.e. a 403 with an HTML body; 0 makes "
+            "every 403 final (default: %(default)s)",
+        )
+        p.add_argument(
+            "--header",
+            action="append",
+            metavar="NAME:VALUE",
+            help="extra request header, repeatable (e.g. 'Authorization: Bearer <token>')",
+        )
+        p.add_argument(
+            "--no-token",
+            action="store_true",
+            help="ignore $ZENODO_TOKEN / $FIGSHARE_TOKEN",
+        )
+
+    fetch = sub.add_parser("fetch", help="download every file of a record")
+    fetch.add_argument("target", help="DOI, zenodo:<id>, figshare:<id>, arxiv:<id>, or URL")
+    fetch.add_argument("-o", "--out", help="output directory")
+    fetch.add_argument("-j", "--concurrency", type=int, default=consts.DEFAULT_CONCURRENCY)
+    fetch.add_argument("--max-size", type=parse_size, help="abort above this total, e.g. 500MB")
+    fetch.add_argument("--only", nargs="+", metavar="NAME", help="fetch only these file names")
+    fetch.add_argument("--manifest-out", help="also write manifest.json here")
+    fetch.add_argument("-n", "--dry-run", action="store_true")
+    fetch.add_argument("-q", "--quiet", action="store_true")
+    network_flags(fetch)
+    fetch.set_defaults(func=cmd_fetch)
+
+    man = sub.add_parser("manifest", help="write a manifest without downloading")
+    man.add_argument("target")
+    man.add_argument("-o", "--out")
+    network_flags(man)
+    man.set_defaults(func=cmd_manifest)
+
+    ver = sub.add_parser("verify", help="re-check a local directory against its manifest")
+    ver.add_argument("manifest")
+    ver.add_argument("--root", help="directory the manifest's names are relative to")
+    ver.set_defaults(func=cmd_verify)
+
+    src = sub.add_parser("sources", help="list supported sources")
+    src.set_defaults(func=cmd_sources)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    # Repository titles are full of non-ASCII; a cp936/cp1252 console would
+    # otherwise abort or mangle them.
+    for stream in (sys.stdout, sys.stderr):
+        if hasattr(stream, "reconfigure"):
+            stream.reconfigure(encoding="utf-8", errors="replace")
+    args = build_parser().parse_args(argv)
+    try:
+        return args.func(args)
+    except TargetError as exc:
+        print(f"acadfetch: {exc}", file=sys.stderr)
+        return EXIT_FAIL
+    except AcFError as exc:
+        print(f"acadfetch: {exc}", file=sys.stderr)
+        return EXIT_FAIL
+    except KeyboardInterrupt:
+        print("\ninterrupted - partial files kept, rerun to resume", file=sys.stderr)
+        return EXIT_INTERRUPT
+
+
+if __name__ == "__main__":
+    sys.exit(main())

acadfetch/client.py

@@ -0,0 +1,209 @@
+"""Single choke point for every HTTP request.
+
+Everything outbound goes through :class:`Client` so that rate limiting, the
+User-Agent, retry policy, and ``Retry-After`` handling cannot drift between
+adapters. Only idempotent methods are retried.
+"""
+
+from __future__ import annotations
+
+import email.utils
+import json
+import random
+import time
+import urllib.error
+import urllib.request
+from collections.abc import Mapping
+from datetime import datetime, timezone
+
+from . import consts, verify
+from .errors import HttpError
+from .ratelimit import TokenBucket
+
+RETRY_STATUS = {408, 425, 429, 500, 502, 503, 504}
+MAX_RETRY_AFTER = 120.0
+MAX_JSON = 32 << 20
+SNIPPET = 200
+
+
+def is_block_page(detail: str, content_type: str | None) -> bool:
+    """Tell a WAF/load-balancer 403 apart from the API's own permission 403.
+
+    Block pages are HTML or empty (AWS ELB, Cloudflare and friends all ship
+    one — see ``tests/fixtures/figshare_403_block.html``); a repository's real
+    permission error is JSON or plain text from the application itself. Both
+    ``api.figshare.com`` endpoints answer the HTML one intermittently, and it
+    clears within a retry or two, so it must not be terminal.
+
+    An empty body counts as a block page: retrying costs one request, while
+    misreading it as terminal costs the whole download.
+    """
+    if content_type and "html" in content_type.lower():
+        return True
+    head = detail.lstrip()[:1]
+    return not head or head == "<"
+
+
+def _blocked_detail(detail: str, retries: int) -> str:
+    """Block-page markup is noise in a one-line error; name it instead."""
+    gave_up = f"gave up after {retries} retr{'y' if retries == 1 else 'ies'}"
+    if detail.startswith("<"):
+        return f"WAF block page, {gave_up}"
+    return f"{detail} (WAF block page, {gave_up})"
+
+
+def parse_retry_after(value: str | None) -> float | None:
+    """Parse the two legal forms of Retry-After. Returns seconds, clamped."""
+    if not value:
+        return None
+    text = value.strip()
+    if text.isdigit():
+        return min(float(text), MAX_RETRY_AFTER)
+    try:
+        when = email.utils.parsedate_to_datetime(text)
+    except (TypeError, ValueError):
+        return None
+    if when is None:
+        return None
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=timezone.utc)
+    delta = (when - datetime.now(timezone.utc)).total_seconds()
+    return max(0.0, min(delta, MAX_RETRY_AFTER))
+
+
+class Client:
+    def __init__(
+        self,
+        rate: float = consts.DEFAULT_RATE,
+        retries: int = consts.DEFAULT_RETRIES,
+        timeout: float = consts.DEFAULT_TIMEOUT,
+        user_agent: str = consts.USER_AGENT,
+        headers: Mapping[str, str] | None = None,
+        auth: Mapping[str, Mapping[str, str]] | None = None,
+        block_retries: int = consts.WAF_403_RETRIES,
+    ) -> None:
+        self.bucket = TokenBucket(rate)
+        self.retries = max(0, retries)
+        self.timeout = timeout
+        self.user_agent = user_agent
+        # How often a WAF block page is worth retrying. 0 turns the feature
+        # off, which is what a run from a network the WAF likes wants.
+        self.block_retries = max(0, block_retries)
+        self.headers = dict(headers or {})
+        # Per-source credentials. The registry sets ``source`` before dispatch
+        # so a Zenodo token is never sent to figshare.
+        self.auth = {k: dict(v) for k, v in (auth or {}).items()}
+        self.source: str | None = None
+
+    # -- internals ---------------------------------------------------------
+    def _sleep(self, attempt: int, retry_after: float | None) -> None:
+        if retry_after is not None:
+            time.sleep(retry_after)
+            return
+        backoff = min(0.8 * (2**attempt), 30.0)
+        time.sleep(backoff * (0.7 + random.random() * 0.6))
+
+    def _request(
+        self, url: str, headers: Mapping[str, str] | None = None
+    ) -> urllib.request.Request:
+        merged: dict[str, str] = {}
+        if self.source and self.source in self.auth:
+            merged.update(self.auth[self.source])
+        merged["User-Agent"] = self.user_agent
+        merged["Accept-Encoding"] = "identity"
+        merged.update(self.headers)  # explicit --header wins over a token
+        if headers:
+            merged.update(headers)
+        return urllib.request.Request(url, headers=merged)
+
+    def _open(self, req: urllib.request.Request):  # noqa: ANN202
+        return urllib.request.urlopen(req, timeout=self.timeout)
+
+    def _with_retries(self, url: str, fn, headers=None, block_retries: int | None = None):  # noqa: ANN001, ANN202
+        if block_retries is None:
+            block_retries = self.block_retries
+        last: Exception | None = None
+        last_detail = ""
+        last_blocked = False
+        waf = 0
+        for attempt in range(self.retries + 1):
+            self.bucket.acquire()
+            retry_after = None
+            try:
+                return fn(self._request(url, headers))
+            except urllib.error.HTTPError as exc:
+                last = exc
+                last_detail = detail = _snippet(exc)
+                content_type = exc.headers.get("Content-Type") if exc.headers else None
+                last_blocked = exc.code == 403 and is_block_page(detail, content_type)
+                retry_after = parse_retry_after(
+                    exc.headers.get("Retry-After") if exc.headers else None
+                )
+                exc.close()
+                if exc.code in RETRY_STATUS:
+                    pass
+                elif not last_blocked:
+                    raise HttpError(exc.code, url, detail) from exc
+                elif waf >= block_retries:
+                    raise HttpError(
+                        403, url, _blocked_detail(last_detail, block_retries), transient=True
+                    ) from exc
+                else:
+                    waf += 1
+            except (urllib.error.URLError, TimeoutError, OSError) as exc:
+                last = exc
+                last_detail = str(exc)
+                last_blocked = False
+            if attempt < self.retries:
+                self._sleep(attempt, retry_after)
+        if isinstance(last, urllib.error.HTTPError):
+            detail = _blocked_detail(last_detail, waf) if last_blocked else last_detail
+            raise HttpError(last.code, url, detail, transient=True) from last
+        raise HttpError(0, url, last_detail or str(last)) from last
+
+    # -- public ------------------------------------------------------------
+    def open(self, url: str, headers: Mapping[str, str] | None = None):
+        """Open a stream. Caller must close it."""
+        return self._with_retries(url, self._open, headers)
+
+    def get_json(self, url: str, headers: Mapping[str, str] | None = None) -> dict:
+        with self.open(url, headers) as resp:
+            raw = resp.read(MAX_JSON)
+        try:
+            return json.loads(raw.decode("utf-8"))
+        except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+            raise HttpError(0, url, f"invalid JSON: {exc}") from exc
+
+    def head(self, url: str) -> dict:
+        def _head(req: urllib.request.Request):
+            req.method = "HEAD"
+            return self._open(req)
+
+        try:
+            with self._with_retries(url, _head, block_retries=0) as resp:
+                return dict(resp.headers)
+        except HttpError as exc:
+            if exc.status in (403, 405, 501):
+                return {}
+            raise
+
+
+def _snippet(exc: urllib.error.HTTPError) -> str:
+    try:
+        raw = exc.read(SNIPPET)
+    except Exception:  # noqa: BLE001
+        return ""
+    text = " ".join(raw.decode("utf-8", "replace").split())
+    return text[:SNIPPET]
+
+
+def checksum_of(headers: Mapping[str, str]) -> str | None:
+    """Prefer a strong hash header over nothing (for direct URLs)."""
+    for key in ("X-Checksum-Sha256", "X-Amz-Meta-Sha256", "Content-MD5"):
+        if key in headers:
+            algo = "sha256" if "sha256" in key.lower() else "md5"
+            return f"{algo}:{headers[key]}"
+    return None
+
+
+__all__ = ["Client", "parse_retry_after", "checksum_of", "verify"]

acadfetch/consts.py

@@ -0,0 +1,15 @@
+"""Package-wide constants and version."""
+
+__version__ = "0.1.0"
+
+USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
+
+DEFAULT_CONCURRENCY = 3
+DEFAULT_RATE = 2.0  # requests per second, global
+DEFAULT_RETRIES = 4
+# A 403 that came from a WAF/load-balancer block page clears on its own; a 403
+# from the API's own permission check does not. Cap the former low so a really
+# forbidden file still fails fast.
+WAF_403_RETRIES = 2
+DEFAULT_TIMEOUT = 60.0
+CHUNK = 1 << 16