pkgreach 0.0.2__py3-none-any.whl

pkgreach/config.py

@@ -0,0 +1,120 @@
+"""Settings: which packages, on which platforms.
+
+A ``pkgreach.toml`` has one table per platform, each a :class:`Selection`::
+
+    [pypi]
+    users = ["alice"]                 # every package these accounts maintain
+    packages = ["some-other-package"] # plus these
+    exclude = ["alice-test-*"]        # minus these (glob patterns)
+
+    [github]
+    users = ["alice", "alice-org"]    # repos of these owners (public only)
+    where = { fork = false, archived = false }   # characteristics to keep
+
+Every key is optional and unknown keys are refused, so a typo never silently
+falls back to a default. In Python, ``collect(select=...)`` adds a predicate
+over each package's metadata, for selections a TOML file cannot express.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+from typing import Any
+
+import tomllib
+
+DFLT_CONFIG_FILENAME = "pkgreach.toml"
+DFLT_DATA_DIR = (
+    Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local" / "share"))
+    / "pkgreach"
+    / "data"
+)
+#: Top-level keys that are run options, not platforms.
+RUN_KEYS = ("max_workers", "dependents")
+
+
+class SettingsError(ValueError):
+    """A settings file that is missing or wrong (a user error, not a bug)."""
+
+
+@dataclass(frozen=True)
+class Selection:
+    """Which packages of one platform to follow.
+
+    Attributes:
+        users: accounts (users, maintainers, orgs) whose packages are all followed.
+        packages: packages followed by name, in addition to the users' ones.
+        exclude: glob patterns of package names to drop (``"*-test"``).
+        where: metadata a package must match to be kept, field by field
+            (``{"fork": False}``). Fields a platform does not report never match.
+    """
+
+    users: tuple[str, ...] = ()
+    packages: tuple[str, ...] = ()
+    exclude: tuple[str, ...] = ()
+    where: Mapping[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class Settings:
+    """What to collect: a :class:`Selection` per platform, and run options.
+
+    Attributes:
+        platforms: platform name to selection; a platform absent here is skipped.
+        max_workers: concurrent requests per platform.
+        dependents: also record dependent packages and repositories
+            (from ecosyste.ms), the strongest sign of real use.
+    """
+
+    platforms: Mapping[str, Selection] = field(default_factory=dict)
+    max_workers: int = 4
+    dependents: bool = True
+
+
+def _tuple(v) -> tuple[str, ...]:
+    return (v,) if isinstance(v, str) else tuple(v)
+
+
+def _selection(platform: str, d: Mapping) -> Selection:
+    known = [f.name for f in fields(Selection)]
+    unknown = sorted(set(d) - set(known))
+    if unknown:
+        raise SettingsError(
+            f"Unknown key(s) {unknown} in [{platform}]; valid keys are {known}."
+        )
+    kw = {k: _tuple(d[k]) for k in ("users", "packages", "exclude") if k in d}
+    return Selection(**kw, where=dict(d.get("where", {})))
+
+
+def settings_from_mapping(d: Mapping, *, known_platforms=None) -> Settings:
+    """Settings from a parsed TOML document (or any nested mapping)."""
+    d = dict(d.get("pkgreach", d))
+    run = {k: d.pop(k) for k in RUN_KEYS if k in d}
+    if known_platforms is not None:
+        unknown = sorted(set(d) - set(known_platforms))
+        if unknown:
+            raise SettingsError(
+                f"Unknown platform(s) or setting(s) {unknown}; platforms are "
+                f"{sorted(known_platforms)}, settings are {list(RUN_KEYS)}."
+            )
+    return Settings(
+        platforms={name: _selection(name, sel) for name, sel in d.items()}, **run
+    )
+
+
+def load_settings(path: str | os.PathLike, *, known_platforms=None) -> Settings:
+    """Settings from a TOML file; a missing or invalid file raises :class:`SettingsError`."""
+    p = Path(path)
+    if not p.exists():
+        raise SettingsError(
+            f"No settings file at {p}. Write one (`pkgreach init`) that lists "
+            'your accounts per platform, e.g. [pypi] users = ["you"].'
+        )
+    try:
+        doc = tomllib.loads(p.read_text(encoding="utf-8"))
+    except tomllib.TOMLDecodeError as e:
+        raise SettingsError(f"{p} is not valid TOML: {e}") from None
+    return settings_from_mapping(doc, known_platforms=known_platforms)

pkgreach/http.py

@@ -0,0 +1,129 @@
+"""The HTTP transport: JSON over the standard library, polite to public APIs.
+
+Every platform takes an ``http=`` callable, ``(url, *, headers=None,
+data=None) -> parsed JSON`` (or text, for non-JSON responses; ``data`` makes it
+a POST), so tests and caches can stand in.
+The default, :func:`make_http`, uses :mod:`urllib`, sends an identifying
+``User-Agent`` (crates.io refuses requests without one), retries rate limits
+and server errors with backoff, and spaces calls to the same host.
+"""
+
+from __future__ import annotations
+
+import http.client
+import json
+import threading
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from collections.abc import Callable, Mapping
+from typing import Any
+
+Http = Callable[..., Any]
+
+DFLT_USER_AGENT = "pkgreach (+https://github.com/thorwhalen/pkgreach)"
+DFLT_TIMEOUT = 30
+DFLT_RETRIES = 6
+DFLT_BACKOFF_S = 2.0
+#: Minimum seconds between two calls to the same host: crates.io asks for one a
+#: second, pypistats.org allows 60 a minute, api.npmjs.org answers bursts with 429.
+DFLT_MIN_INTERVAL_S = {"crates.io": 1.0, "pypistats.org": 1.0, "api.npmjs.org": 2.0}
+MAX_RETRY_WAIT_S = 120.0
+_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
+
+
+class _SameHostAuthRedirect(urllib.request.HTTPRedirectHandler):
+    """Follow redirects, but never carry ``Authorization`` to another host."""
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        new = super().redirect_request(req, fp, code, msg, headers, newurl)
+        old_host = urllib.parse.urlsplit(req.full_url).hostname
+        if new is not None and urllib.parse.urlsplit(newurl).hostname != old_host:
+            new.remove_header("Authorization")
+        return new
+
+
+_OPENER = urllib.request.build_opener(_SameHostAuthRedirect)
+
+
+class HttpError(RuntimeError):
+    """An HTTP call that failed, with its status (0 for network errors)."""
+
+    def __init__(self, status: int, message: str, *, url: str = ""):
+        super().__init__(f"HTTP {status} on {url}: {message}")
+        self.status, self.message, self.url = status, message, url
+
+
+def _retry_after(e: urllib.error.HTTPError, attempt: int, backoff_s: float) -> float:
+    """Exponential backoff, longer if the server asks (npm sends ``Retry-After: 0``)."""
+    try:
+        asked = float(e.headers.get("Retry-After", ""))
+    except ValueError:
+        asked = 0.0
+    return min(max(asked, backoff_s * 2**attempt), MAX_RETRY_WAIT_S)
+
+
+def make_http(
+    *,
+    user_agent: str = DFLT_USER_AGENT,
+    timeout: float = DFLT_TIMEOUT,
+    retries: int = DFLT_RETRIES,
+    backoff_s: float = DFLT_BACKOFF_S,
+    min_interval_s: Mapping[str, float] = DFLT_MIN_INTERVAL_S,
+    headers_by_host: Mapping[str, Mapping[str, str]] | None = None,
+    sleep: Callable[[float], None] = time.sleep,
+) -> Http:
+    """A ``get(url, *, headers=None, data=None)`` returning parsed JSON (else text).
+
+    ``headers_by_host`` adds headers to every request to a host: the place for
+    credentials, so that platforms never read them from the environment.
+    """
+    last_call: dict[str, float] = {}
+    host_locks: dict[str, threading.Lock] = {}
+    locks_lock = threading.Lock()
+
+    def _space(host: str) -> None:
+        """Wait until ``host`` may be called again (hosts never wait on each other)."""
+        interval = min_interval_s.get(host, 0.0)
+        if not interval:
+            return
+        with locks_lock:
+            lock = host_locks.setdefault(host, threading.Lock())
+        with lock:
+            wait = last_call.get(host, 0.0) + interval - time.monotonic()
+            if wait > 0:
+                sleep(wait)
+            last_call[host] = time.monotonic()
+
+    def get(
+        url: str, *, headers: Mapping[str, str] | None = None, data: bytes | None = None
+    ) -> Any:
+        host = urllib.parse.urlsplit(url).hostname or ""
+        hdrs = {"User-Agent": user_agent, "Accept": "application/json"}
+        hdrs.update((headers_by_host or {}).get(host, {}))  # credentials, per host
+        hdrs.update(headers or {})
+        for attempt in range(retries + 1):
+            _space(host)
+            req = urllib.request.Request(url, headers=hdrs, data=data)
+            try:
+                with _OPENER.open(req, timeout=timeout) as resp:
+                    body = resp.read().decode("utf-8")
+            except urllib.error.HTTPError as e:
+                if e.code in _RETRY_STATUSES and attempt < retries:
+                    sleep(_retry_after(e, attempt, backoff_s))
+                    continue
+                raise HttpError(e.code, e.reason or "", url=url) from None
+            except (urllib.error.URLError, OSError, http.client.HTTPException) as e:
+                # network trouble, including resets and truncated reads
+                if attempt < retries:
+                    sleep(backoff_s * 2**attempt)
+                    continue
+                raise HttpError(0, str(e), url=url) from None
+            try:
+                return json.loads(body)
+            except json.JSONDecodeError:
+                return body
+        raise AssertionError("unreachable")  # pragma: no cover
+
+    return get

pkgreach/platforms/__init__.py

@@ -0,0 +1,103 @@
+"""Platforms: where packages are published, and how to read their numbers.
+
+A :class:`Platform` says how to list an account's packages, how to fetch one
+package's numbers, and how the report should present them. :data:`PLATFORMS`
+holds the built-in ones; pass your own mapping to :func:`pkgreach.collect`
+and :func:`pkgreach.summarize` to add one (conda-forge, Docker Hub, Hugging
+Face, ...). Nothing else changes: the analysis and the report read only the
+fields below, never a platform's name.
+
+``list_packages(user, *, http)`` yields metadata records with at least a
+``package`` key. Other plain keys (``fork``, ``role``, ...) can be matched by
+a selection's ``where``. A listing that already carries the numbers may put
+them under ``_info`` and ``_metrics``, so that ``fetch`` needs no further call
+(GitHub does); keys starting with ``_`` are never matched by ``where``.
+
+``fetch(record, *, http, today, since)`` returns a dict with any of:
+
+- ``downloads``: ``[{"date", "series", "count"}, ...]`` daily counts from
+  ``since`` (``None``: as far back as the platform keeps them);
+- ``history_from``: the first day the source covered (ISO date), so absent
+  days after it read as zero rather than unknown;
+- ``metrics``: ``{name: int}`` current values (stars, versions, total downloads);
+- ``info``: metadata for the ``packages`` table (``url``, ``repository``,
+  ``description``, ``latest_version``, ``created``).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Mapping
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from pkgreach.platforms import crates, github, npm, pypi
+
+
+@dataclass(frozen=True)
+class Platform:
+    """How to list and read the packages of one publishing platform.
+
+    Attributes:
+        name: the key in ``pkgreach.toml`` and in the tables' ``platform`` column.
+        list_packages, fetch: see the module docstring.
+        title: how the report names it.
+        kind: ``"daily"`` if ``fetch`` returns daily downloads (the report shows
+            windows, growth and the noise floor), ``"snapshot"`` if it returns
+            current values only (the report shows ``headline`` metrics).
+        headline: the metrics a snapshot platform's table and card show, the
+            first one summed on the card.
+        preferred_series: download series to analyse, best first.
+        ecosystem: the package ecosystem, as a purl type (``pypi``, ``npm``,
+            ``cargo``), for dependents lookups; ``""`` for none.
+        default_where: the ``where`` that ``pkgreach init`` writes for it.
+        normalize: canonical form of a package name (PyPI: PEP 503).
+        prune_unlisted: remove packages no longer listed from the history's
+            current tables (for GitHub: a repo made private must not stay in a
+            public history).
+    """
+
+    name: str
+    list_packages: Callable[..., Iterable[dict]]
+    fetch: Callable[..., dict[str, Any]]
+    title: str = ""
+    kind: Literal["daily", "snapshot"] = "daily"
+    headline: tuple[str, ...] = ()
+    preferred_series: tuple[str, ...] = ("all",)
+    ecosystem: str = ""
+    default_where: Mapping[str, Any] = field(default_factory=dict)
+    normalize: Callable[[str], str] = str
+    prune_unlisted: bool = False
+
+
+PLATFORMS: dict[str, Platform] = {
+    p.name: p
+    for p in [
+        Platform(
+            "pypi",
+            pypi.list_packages,
+            pypi.fetch,
+            title="PyPI",
+            preferred_series=("without_mirrors", "all"),
+            ecosystem="pypi",
+            normalize=pypi.normalize,
+        ),
+        Platform("npm", npm.list_packages, npm.fetch, title="npm", ecosystem="npm"),
+        Platform(
+            "crates",
+            crates.list_packages,
+            crates.fetch,
+            title="crates.io",
+            ecosystem="cargo",
+        ),
+        Platform(
+            "github",
+            github.list_packages,
+            github.fetch,
+            title="GitHub",
+            kind="snapshot",
+            headline=("stars", "forks", "open_issues"),
+            default_where={"fork": False, "archived": False},
+            prune_unlisted=True,
+        ),
+    ]
+}

pkgreach/platforms/crates.py

@@ -0,0 +1,67 @@
+"""crates.io: daily downloads (last 90 days), totals and metadata.
+
+- **Listing** an account's crates: ``/api/v1/users/{login}`` gives the user id,
+  then ``/api/v1/crates?user_id={id}`` (100 per page).
+- **Downloads**: ``/api/v1/crates/{name}/downloads``: per version and day for
+  the last 90 days, plus ``extra_downloads`` for the older versions; summed
+  per day.
+- **Metadata** and all-time totals: ``/api/v1/crates/{name}``.
+
+crates.io requires an identifying User-Agent and at most one request a second
+(see :mod:`pkgreach.http`, which does both).
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Iterator
+from datetime import date
+
+API = "https://crates.io/api/v1"
+PAGE_SIZE = 100
+
+
+def list_packages(user: str, *, http) -> Iterator[dict]:
+    """The crates ``user`` (a crates.io login) owns."""
+    uid = http(f"{API}/users/{user}")["user"]["id"]
+    page = 1
+    while True:
+        got = http(f"{API}/crates?user_id={uid}&per_page={PAGE_SIZE}&page={page}")
+        for c in got.get("crates", []):
+            yield {"package": c["name"]}
+        if not got.get("meta", {}).get("next_page"):
+            return
+        page += 1
+
+
+def fetch(record: dict, *, http, today: date, since: date | None = None) -> dict:
+    """Daily downloads, totals and metadata of one crate."""
+    name = record["package"]
+    got = http(f"{API}/crates/{name}/downloads")
+    per_day: Counter = Counter()
+    for r in got.get("version_downloads", []) + got.get("meta", {}).get(
+        "extra_downloads", []
+    ):
+        per_day[r["date"]] += r["downloads"]
+    lo = since.isoformat() if since else ""
+    crate = http(f"{API}/crates/{name}")
+    c = crate.get("crate", {})
+    return {
+        "downloads": [
+            {"date": d, "series": "all", "count": n}
+            for d, n in sorted(per_day.items())
+            if d >= lo
+        ],
+        "history_from": min(per_day, default=""),
+        "metrics": {
+            "total_downloads": c.get("downloads"),
+            "versions": len(crate.get("versions") or []) or None,
+        },
+        "info": {
+            "url": f"https://crates.io/crates/{name}",
+            "repository": c.get("repository") or "",
+            "description": (c.get("description") or "").strip(),
+            "latest_version": c.get("max_stable_version") or c.get("max_version") or "",
+            "created": (c.get("created_at") or "")[:10],
+        },
+    }

pkgreach/platforms/ecosystems.py

@@ -0,0 +1,36 @@
+"""Dependents, from ecosyste.ms: the strongest public sign that a package is used.
+
+A download can be a mirror or a CI cache miss; a dependent is another package
+or repository that declares yours in its manifest. ecosyste.ms (open data,
+CC-BY-SA 4.0, no key needed) indexes them for every major registry:
+``GET /api/v1/registries/{registry}/packages/{name}`` carries
+``dependent_packages_count`` and ``dependent_repos_count``.
+"""
+
+from __future__ import annotations
+
+import urllib.parse
+
+from pkgreach.http import HttpError
+
+API = "https://packages.ecosyste.ms/api/v1"
+#: Ecosystem (purl type) to ecosyste.ms registry name.
+REGISTRIES = {"pypi": "pypi.org", "npm": "npmjs.org", "cargo": "crates.io"}
+
+
+def dependents(ecosystem: str, package: str, *, http) -> dict[str, int]:
+    """``{"dependent_packages": n, "dependent_repos": m}``; ``{}`` if not indexed."""
+    registry = REGISTRIES.get(ecosystem)
+    if registry is None:
+        return {}
+    name = urllib.parse.quote(package, safe="")
+    try:
+        d = http(f"{API}/registries/{registry}/packages/{name}")
+    except HttpError as e:
+        if e.status == 404:  # not indexed (yet): unknown, not zero
+            return {}
+        raise
+    return {
+        "dependent_packages": d.get("dependent_packages_count"),
+        "dependent_repos": d.get("dependent_repos_count"),
+    }

pkgreach/platforms/github.py

@@ -0,0 +1,84 @@
+"""GitHub: public repositories, with stars, forks and open issues.
+
+Only **public** repositories are followed, whatever the token can see: this
+report is meant to be publishable. Traffic (views, clones, referrers) needs
+push access and is private; that is ghtraffic's job
+(https://github.com/thorwhalen/ghtraffic), kept in a private repository.
+
+Listing uses ``GET /users/{owner}/repos`` (users and organizations alike), one
+call per 100 repos, and the listing already carries the numbers, so fetching
+costs no further call. A token only raises the rate limit (60 calls an hour
+without one); it reaches the requests through the transport
+(``make_http(headers_by_host=auth_headers(token))``), never from here.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping
+from datetime import date
+
+API = "https://api.github.com"
+HOST = "api.github.com"
+PAGE_SIZE = 100
+TOKEN_ENV_VARS = ("GITHUB_TOKEN", "GH_TOKEN")
+_HEADERS = {
+    "Accept": "application/vnd.github+json",
+    "X-GitHub-Api-Version": "2022-11-28",
+}
+
+
+def auth_headers(token: str | None) -> dict[str, Mapping[str, str]]:
+    """``headers_by_host`` for :func:`pkgreach.http.make_http`: the token, for GitHub only."""
+    return {HOST: {"Authorization": f"Bearer {token}"}} if token else {}
+
+
+def token_from_env(environ: Mapping[str, str]) -> str:
+    """The first of ``GITHUB_TOKEN``, ``GH_TOKEN`` set in ``environ``, or ``""``."""
+    return next((environ[v] for v in TOKEN_ENV_VARS if environ.get(v)), "")
+
+
+def _record(r: dict) -> dict:
+    return {
+        "package": r["full_name"],
+        "private": r.get("private", False),
+        "fork": r.get("fork", False),
+        "archived": r.get("archived", False),
+        "language": r.get("language") or "",
+        "_info": {
+            "url": r.get("html_url", ""),
+            "repository": r.get("html_url", ""),
+            "description": r.get("description") or "",
+            "created": (r.get("created_at") or "")[:10],
+        },
+        "_metrics": {
+            "stars": r.get("stargazers_count"),
+            "forks": r.get("forks_count"),
+            "open_issues": r.get("open_issues_count"),  # issues + pull requests
+        },
+    }
+
+
+def list_packages(user: str, *, http) -> Iterator[dict]:
+    """The public repos owned by ``user`` (a user or an organization)."""
+    page = 1
+    while True:
+        got = http(
+            f"{API}/users/{user}/repos?type=owner&per_page={PAGE_SIZE}&page={page}",
+            headers=_HEADERS,
+        )
+        for r in got:
+            if not r.get("private", False):
+                yield _record(r)
+        if len(got) < PAGE_SIZE:
+            return
+        page += 1
+
+
+def fetch(record: dict, *, http, today: date, since: date | None = None) -> dict:
+    """Stars, forks and open issues of one public repo."""
+    if "_metrics" not in record:  # named in `packages`, not listed
+        r = http(f"{API}/repos/{record['package']}", headers=_HEADERS)
+        if r.get("private", False):
+            raise ValueError("private repository: not followed (public report)")
+        record = _record(r)
+    return {"metrics": dict(record["_metrics"]), "info": dict(record["_info"])}

pkgreach/platforms/npm.py

@@ -0,0 +1,90 @@
+"""npm: daily downloads from api.npmjs.org, listing from the registry search.
+
+- **Listing** an account's packages: the registry search,
+  ``registry.npmjs.org/-/v1/search?text=maintainer:{user}`` (250 per page).
+- **Downloads**: ``api.npmjs.org/downloads/range/{start}:{end}/{name}``,
+  daily, at most the last 18 months (a longer range is clipped), scoped names
+  included. The first run takes all 18 months; later runs take what is new.
+  A package never downloaded yet answers 404 there, which reads as zero.
+- **Metadata**: from the search record (version, description, repository).
+"""
+
+from __future__ import annotations
+
+import urllib.parse
+from collections.abc import Iterator
+from datetime import date, timedelta
+
+from pkgreach.http import HttpError
+
+SEARCH_URL = "https://registry.npmjs.org/-/v1/search?text={q}&size={n}&from={start}"
+RANGE_URL = "https://api.npmjs.org/downloads/range/{start}:{end}/{name}"
+LATEST_URL = "https://registry.npmjs.org/{name}/latest"
+PAGE_SIZE = 250
+#: The download API serves the last 18 months; ask for a little more.
+MAX_HISTORY_DAYS = 560
+#: What the download API actually keeps (a longer range is clipped to this).
+SERVED_DAYS = 540
+
+
+def _clean_repo(url: str) -> str:
+    url = (url or "").removeprefix("git+")
+    url = url.replace("git://", "https://").replace("ssh://git@", "https://")
+    return url.removesuffix(".git") if url.startswith("https://") else ""
+
+
+def list_packages(user: str, *, http) -> Iterator[dict]:
+    """The packages ``user`` maintains on npm."""
+    q = urllib.parse.quote(f"maintainer:{user}")
+    start = 0
+    while True:
+        got = http(SEARCH_URL.format(q=q, n=PAGE_SIZE, start=start))
+        objects = got.get("objects", [])
+        for o in objects:
+            p = o["package"]
+            yield {
+                "package": p["name"],
+                "_info": {
+                    "url": f"https://www.npmjs.com/package/{p['name']}",
+                    "repository": _clean_repo(p.get("links", {}).get("repository")),
+                    "description": p.get("description") or "",
+                    "latest_version": p.get("version") or "",
+                },
+            }
+        start += len(objects)
+        if not objects or start >= got.get("total", 0):
+            return
+
+
+def fetch(record: dict, *, http, today: date, since: date | None = None) -> dict:
+    """Daily downloads and metadata of one package."""
+    name = record["package"]
+    start = since or today - timedelta(days=MAX_HISTORY_DAYS)
+    try:
+        got = http(RANGE_URL.format(start=start, end=today, name=name))
+    except HttpError as e:
+        if e.status != 404:
+            raise
+        # "package not found" here means: published, but never downloaded yet
+        served = max(start, today - timedelta(days=SERVED_DAYS))
+        got = {"start": served.isoformat(), "downloads": []}
+    info = record.get("_info")
+    if info is None:  # named in `packages`, not listed
+        latest = http(LATEST_URL.format(name=name))
+        repo = latest.get("repository")
+        info = {
+            "url": f"https://www.npmjs.com/package/{name}",
+            "repository": _clean_repo(
+                repo.get("url") if isinstance(repo, dict) else repo
+            ),
+            "description": latest.get("description") or "",
+            "latest_version": latest.get("version") or "",
+        }
+    return {
+        "downloads": [
+            {"date": d["day"], "series": "all", "count": d["downloads"]}
+            for d in got.get("downloads", [])
+        ],
+        "history_from": got.get("start", ""),  # the min with earlier runs is kept
+        "info": info,
+    }