pkgreach 0.0.2__py3-none-any.whl

pkgreach/__init__.py

@@ -0,0 +1,53 @@
+"""Track how your packages are really used, across PyPI, npm, crates.io and GitHub.
+
+Registries keep a few months of daily download counts at most, and none of
+them separates people from robots. ``pkgreach`` collects the counts every week
+into a git-friendly history, adds the signals a robot does not create
+(dependent packages and repositories, stars, forks), and builds a static
+report that puts each package against a noise floor.
+
+>>> from pkgreach import collect, load_settings, build_site, Files  # doctest: +SKIP
+>>> collect(load_settings("pkgreach.toml"), store=Files("data"))   # doctest: +SKIP
+>>> build_site(Files("data"), Files("site"))                       # doctest: +SKIP
+
+Seams (each one keyword argument, with a working default):
+
+- ``collect(platforms=)``: which publishing platforms (:data:`PLATFORMS`).
+- :class:`Selection` per platform, plus ``collect(select=)``: which packages.
+- ``collect(dependents=)``: where dependents come from (ecosyste.ms).
+- ``build_site(analyze=, render=)``: what is computed and how it is shown.
+- ``store`` / ``out``: where the history and the site live (any
+  ``MutableMapping[str, str]``: a folder by default).
+"""
+
+from ghtraffic.tables import Files
+
+from pkgreach.analyze import summarize
+from pkgreach.collect import RunReport, collect, select_packages, write_manifest
+from pkgreach.config import Selection, Settings, load_settings, settings_from_mapping
+from pkgreach.http import HttpError, make_http
+from pkgreach.platforms import PLATFORMS, Platform
+from pkgreach.report import build_site, render_html
+from pkgreach.scaffold import init_repo
+from pkgreach.tables import TABLES
+
+__all__ = [
+    "Files",
+    "HttpError",
+    "PLATFORMS",
+    "Platform",
+    "RunReport",
+    "Selection",
+    "Settings",
+    "TABLES",
+    "build_site",
+    "collect",
+    "init_repo",
+    "load_settings",
+    "make_http",
+    "render_html",
+    "select_packages",
+    "settings_from_mapping",
+    "summarize",
+    "write_manifest",
+]

pkgreach/__main__.py

@@ -0,0 +1,36 @@
+# PYTHON_ARGCOMPLETE_OK
+"""Command line: ``pkgreach collect | report | init``, one command per function
+of :data:`pkgreach.tools.COMMANDS` (dispatched by ``cw``).
+
+``pkgreach collect --config pkgreach.toml --data-dir data`` merges this week's
+numbers into the history; ``pkgreach report --data-dir data --out site``
+writes the static report; ``pkgreach init . --pypi you --github you`` sets up
+a repository. Without ``--data-dir``, data goes to ``~/.local/share/pkgreach/data``.
+"""
+
+from __future__ import annotations
+
+import sys
+
+import cw
+
+from pkgreach.config import SettingsError
+from pkgreach.tools import COMMANDS, CollectionFailed
+
+#: Keep injection-only parameters off the command line.
+_CLI_CONFIG = {"collect": {"environ": cw.HIDE}}
+
+
+def main(argv=None) -> int:
+    """Entry point of the ``pkgreach`` command; returns the exit code."""
+    try:
+        return cw.dispatch(
+            COMMANDS, argv=argv, config=_CLI_CONFIG, convention=cw.MODERN
+        )
+    except (CollectionFailed, SettingsError) as e:
+        print(f"pkgreach: {e}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pkgreach/analyze.py

@@ -0,0 +1,242 @@
+"""From the history to a summary: windowed totals, growth, and a noise floor.
+
+:func:`summarize` turns the tables into one JSON-ready dict, which the report
+renders and anyone can consume (``summary.json``). It is the default of the
+``analyze=`` seam of :func:`pkgreach.report.build_site`.
+
+Download counts are mostly not people. Mirrors, CI pipelines, scanners and
+bots download every release of every package, so a package nobody uses still
+gets a steady trickle. The summary therefore puts each package against a
+**noise floor**: the 25th percentile of 30-day downloads among the packages
+you follow on that platform that were downloaded at all (an estimate of what
+an unused package of yours gets). ``x_floor`` is a package's 30-day downloads over that floor; dependents
+and stars, which a bot does not create, are reported next to it.
+
+On PyPI the ``without_mirrors`` series (from pypistats.org) is used when it
+exists, since mirror traffic is pure noise.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from collections.abc import Mapping, MutableMapping
+from datetime import date, timedelta
+from statistics import quantiles
+from typing import Any
+
+from pkgreach.platforms import PLATFORMS, Platform
+from pkgreach.tables import LAST_RUN_KEY, TABLES, read_table
+
+#: Windows (days) the summary totals over.
+WINDOWS = (7, 30, 90, 365)
+DFLT_FLOOR_QUANTILE = 0.25
+#: Where the numbers come from, and on what terms (ecosyste.ms is CC BY-SA 4.0).
+ATTRIBUTION = {
+    "downloads": "pypistats.org, api.npmjs.org, crates.io",
+    "dependents": "ecosyste.ms (https://ecosyste.ms), CC BY-SA 4.0 "
+    "(https://creativecommons.org/licenses/by-sa/4.0/), aggregated",
+    "github": "GitHub REST API, public repositories only",
+}
+SPARK_WEEKS = 26
+
+
+def _daily(
+    store: MutableMapping, prefs_of: Mapping[str, tuple[str, ...]]
+) -> dict[tuple[str, str], dict[str, int]]:
+    """``{(platform, package): {date: count}}`` in one series per platform.
+
+    The series is the platform's most preferred one present for any package,
+    then used for all of them: zero days are not stored, so a package that a
+    preferred series counts as zero (all its PyPI downloads are mirrors) must
+    read as zero, not fall back to a noisier series.
+    """
+    by_series: dict[tuple[str, str, str], dict[str, int]] = defaultdict(dict)
+    for r in read_table(store, TABLES["downloads"]):
+        key = (r["platform"], r["package"], r["series"])
+        by_series[key][str(r["date"])] = int(r["count"])
+    present: dict[str, set[str]] = defaultdict(set)
+    for platform, _, series in by_series:
+        present[platform].add(series)
+    chosen = {}
+    for platform, series_set in present.items():
+        prefs = prefs_of.get(platform, ("all",))
+        chosen[platform] = next(
+            (x for x in prefs if x in series_set), sorted(series_set)[0]
+        )
+    return {
+        (platform, package): days
+        for (platform, package, series), days in by_series.items()
+        if series == chosen[platform]
+    }
+
+
+def _window(days: Mapping[str, int], end: date, n: int) -> int:
+    start = (end - timedelta(days=n - 1)).isoformat()
+    stop = end.isoformat()
+    return sum(c for d, c in days.items() if start <= d <= stop)
+
+
+def _weekly(days: Mapping[str, int], end: date, weeks: int) -> list[int]:
+    return [
+        _window(days, end - timedelta(days=7 * (weeks - 1 - i)), 7)
+        for i in range(weeks)
+    ]
+
+
+def _metrics(store: MutableMapping, today: date, delta_days: int = 30):
+    """``{(platform, package): {metric: {"value", "change_30d"}}}``."""
+    hist: dict[tuple, list[tuple[str, int]]] = defaultdict(list)
+    for r in read_table(store, TABLES["metrics"]):
+        if r["value"] != "":
+            hist[(r["platform"], r["package"], r["metric"])].append(
+                (str(r["date"]), int(r["value"]))
+            )
+    cutoff = (today - timedelta(days=delta_days)).isoformat()
+    out: dict[tuple[str, str], dict[str, dict]] = defaultdict(dict)
+    for (platform, package, metric), pts in hist.items():
+        pts.sort()
+        value = pts[-1][1]
+        before = [v for d, v in pts if d <= cutoff]
+        out[(platform, package)][metric] = {
+            "value": value,
+            "change_30d": value - before[-1] if before else None,
+        }
+    return out
+
+
+def _floor(values: list[int], q: float) -> int:
+    values = sorted(values)
+    if not values:
+        return 0
+    if len(values) < 4:
+        return values[0]
+    idx = min(99, max(1, round(q * 100))) - 1
+    return int(quantiles(values, n=100, method="inclusive")[idx])
+
+
+def _current(packages: Mapping[tuple[str, str], Mapping]) -> set[tuple[str, str]]:
+    """The packages the latest run of their platform still followed.
+
+    A package dropped from the selection, deleted, or (for a repo) made private
+    leaves the report, though its past rows stay in the history.
+    """
+    last_run: dict[str, str] = {}
+    for (platform, _), r in packages.items():
+        last_run[platform] = max(last_run.get(platform, ""), r.get("last_seen", ""))
+    return {k for k, r in packages.items() if r.get("last_seen", "") == last_run[k[0]]}
+
+
+def _ends(packages, daily) -> dict[str, date]:
+    """Per platform, the last complete day: its last day with downloads, but
+    never the day of its last run (crates.io serves the partial current day,
+    pypistats and npm stop a day or two earlier)."""
+    last_run: dict[str, str] = {}
+    for (platform, _), r in packages.items():
+        last_run[platform] = max(last_run.get(platform, ""), r.get("last_seen", ""))
+    last_day: dict[str, str] = {}
+    for (platform, _), days in daily.items():
+        if days:
+            last_day[platform] = max(last_day.get(platform, ""), max(days))
+    ends = {}
+    for platform, d in last_day.items():
+        end = date.fromisoformat(d)
+        if last_run.get(platform):
+            end = min(end, date.fromisoformat(last_run[platform]) - timedelta(days=1))
+        ends[platform] = end
+    return ends
+
+
+def _run_errors(store: MutableMapping) -> int | None:
+    try:
+        return len(json.loads(store[LAST_RUN_KEY]).get("errors", {}))
+    except (KeyError, ValueError):
+        return None
+
+
+def summarize(
+    store: MutableMapping,
+    *,
+    platforms: Mapping[str, Platform] = PLATFORMS,
+    today: date | None = None,
+    floor_quantile: float = DFLT_FLOOR_QUANTILE,
+) -> dict[str, Any]:
+    """A JSON-ready summary of the history in ``store``.
+
+    ``platforms`` supplies how each platform is presented (title, kind,
+    headline metrics, preferred download series); a platform in the history
+    but not in ``platforms`` is shown as a daily-downloads platform.
+    """
+
+    def spec(name: str) -> Platform:
+        return platforms.get(name) or Platform(name, list, dict, title=name)
+
+    packages = {
+        (r["platform"], r["package"]): r for r in read_table(store, TABLES["packages"])
+    }
+    known = {name for name, _ in packages}
+    daily = _daily(store, {n: spec(n).preferred_series for n in known})
+    ends = _ends(packages, daily)
+    as_of = today or max(ends.values(), default=date.today())
+    metrics = _metrics(store, as_of)
+    rows = []
+    for key in sorted(_current(packages)):
+        platform, package = key
+        end = today or ends.get(platform, as_of)
+        days = daily.get(key, {})
+        info = packages.get(key, {})
+        row = {
+            "platform": platform,
+            "package": package,
+            "url": info.get("url", ""),
+            "repository": info.get("repository", ""),
+            "description": info.get("description", ""),
+            "latest_version": info.get("latest_version", ""),
+            "history_from": info.get("history_from", ""),
+            "last_seen": info.get("last_seen", ""),
+            "downloads": {f"{n}d": _window(days, end, n) for n in WINDOWS},
+            "downloads_prev_30d": _window(days, end - timedelta(days=30), 30),
+            "downloads_collected": sum(days.values()),
+            "has_downloads": bool(days or info.get("history_from")),
+            "weekly": _weekly(days, end, SPARK_WEEKS) if days else [],
+            "metrics": metrics.get(key, {}),
+        }
+        prev = row["downloads_prev_30d"]
+        row["growth_30d"] = (
+            round(row["downloads"]["30d"] / prev - 1, 3) if prev else None
+        )
+        rows.append(row)
+
+    by_platform: dict[str, dict] = {}
+    for platform in sorted({r["platform"] for r in rows}):
+        p = spec(platform)
+        mine = [r for r in rows if r["platform"] == platform]
+        # packages with no download at all say nothing about the noise level
+        with_dl = [r["downloads"]["30d"] for r in mine if r["downloads"]["30d"]]
+        floor = _floor(with_dl, floor_quantile)
+        for r in mine:
+            r["x_floor"] = (
+                round(r["downloads"]["30d"] / floor, 1)
+                if floor and r["has_downloads"]
+                else None
+            )
+        by_platform[platform] = {
+            "title": p.title or platform,
+            "kind": p.kind,
+            "headline": list(p.headline),
+            "packages": len(mine),
+            "downloads_30d": sum(r["downloads"]["30d"] for r in mine),
+            "downloads_prev_30d": sum(r["downloads_prev_30d"] for r in mine),
+            "noise_floor_30d": floor if with_dl else None,
+            "metric_totals": {
+                m: sum(r["metrics"].get(m, {}).get("value", 0) for r in mine)
+                for m in sorted({m for r in mine for m in r["metrics"]})
+            },
+        }
+    return {
+        "as_of": as_of.isoformat(),
+        "last_run_errors": _run_errors(store),
+        "attribution": ATTRIBUTION,
+        "platforms": by_platform,
+        "packages": rows,
+    }

pkgreach/collect.py

@@ -0,0 +1,301 @@
+"""The collection run: select packages per platform, fetch, merge into the history.
+
+:func:`collect` is the one orchestrator. Every table merges idempotently, so a
+cold start and a weekly update are the same call, and running it twice in a
+row changes nothing. One package failing never sinks a run: its error goes to
+``last_run.json`` and the rest is collected.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from collections.abc import Callable, Mapping, MutableMapping
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import asdict, dataclass, field
+from datetime import date, datetime, timedelta, timezone
+from fnmatch import fnmatch
+from typing import Any
+
+from pkgreach.config import DFLT_DATA_DIR, Selection, Settings
+from pkgreach.http import Http, make_http
+from pkgreach.platforms import PLATFORMS, Platform
+from pkgreach.platforms.ecosystems import dependents as ecosystems_dependents
+from pkgreach.tables import (
+    DATA_LICENSE,
+    Files,
+    dump_csv,
+    parse_csv,
+    read_table,
+    table_files,
+    write_rows,
+    DATA_LICENSE_KEY,
+    LAST_RUN_KEY,
+    MANIFEST_KEY,
+    SCHEMA_VERSION,
+    TABLES,
+)
+
+_log = logging.getLogger("pkgreach")
+#: Days re-fetched before the last collected day: registries revise recent days.
+DFLT_OVERLAP_DAYS = 7
+
+Select = Callable[[str, Mapping[str, Any]], bool]
+Dependents = Callable[..., dict[str, int]]
+
+
+@dataclass
+class RunReport:
+    """What a run did, for logs and ``last_run.json``."""
+
+    started_at: str
+    duration_s: float = 0.0
+    packages: dict[str, int] = field(default_factory=dict)
+    errors: dict[str, str] = field(default_factory=dict)
+    files_changed: int = 0
+
+    def summary(self) -> str:
+        """One line for the log."""
+        per = ", ".join(f"{p} {n}" for p, n in self.packages.items()) or "nothing"
+        return (
+            f"pkgreach: {per} packages; {len(self.errors)} errors, "
+            f"{self.files_changed} files changed in {self.duration_s:.1f}s"
+        )
+
+
+def _matches(record: Mapping, where: Mapping) -> bool:
+    return all(
+        not k.startswith("_")
+        and k in record
+        and str(record[k]).lower() == str(v).lower()
+        for k, v in where.items()
+    )
+
+
+def select_packages(
+    platform: Platform,
+    selection: Selection,
+    *,
+    http: Http,
+    select: Select | None = None,
+    on_error: Callable[[str, Exception], None] = lambda *_: None,
+) -> dict[str, dict]:
+    """The packages a selection names, as ``{name: metadata record}``.
+
+    The users' packages that match ``where``, plus the named ones (named
+    explicitly, so ``where`` does not apply to them), minus ``exclude``
+    patterns, keeping those ``select`` accepts. Names are normalized by the
+    platform (PyPI: ``Flask_Login`` and ``flask-login`` are one package).
+    """
+    norm = platform.normalize
+    records: dict[str, dict] = {}
+    for user in selection.users:
+        try:
+            for r in platform.list_packages(user, http=http):
+                if _matches(r, selection.where):
+                    records.setdefault(
+                        norm(r["package"]), {**r, "package": norm(r["package"])}
+                    )
+        except Exception as e:  # noqa: BLE001 - one account never sinks the run
+            on_error(f"{platform.name}:@{user}", e)
+    for name in selection.packages:
+        records.setdefault(norm(name), {"package": norm(name)})
+    return {
+        name: r
+        for name, r in sorted(records.items())
+        if not any(fnmatch(name, pat) for pat in selection.exclude)
+        and (select is None or select(platform.name, r))
+    }
+
+
+def _last_dates(store: MutableMapping, platform: str) -> dict[str, str]:
+    """The last day with downloads, per package, from the last two year files."""
+    files = table_files(store, TABLES["downloads"])[-2:]
+    years = {f.split("/")[-1][:4] for f in files}
+    last: dict[str, str] = {}
+    for r in read_table(store, TABLES["downloads"], years=years):
+        if r["platform"] == platform:
+            last[r["package"]] = max(last.get(r["package"], ""), str(r["date"]))
+    return last
+
+
+def _prune(store: MutableMapping, platform: str, *, keep: set[str]) -> int:
+    """Remove a platform's packages that are no longer followed from the
+    ``packages`` and ``metrics`` tables (a repo made private must disappear
+    from a public history). Returns the number of files changed."""
+    changed = 0
+    for name in ("packages", "metrics"):
+        spec = TABLES[name]
+        for path in table_files(store, spec):
+            text = store[path]
+            rows = parse_csv(spec, text)
+            kept = [
+                r for r in rows if r["platform"] != platform or r["package"] in keep
+            ]
+            if len(kept) != len(rows):
+                store[path] = dump_csv(spec, kept)
+                changed += 1
+    return changed
+
+
+def write_manifest(store: MutableMapping, *, updated: str) -> dict:
+    """Write ``manifest.json``: the schema and the files of every table."""
+    manifest = {
+        "schema_version": SCHEMA_VERSION,
+        "updated": updated,
+        "tables": {
+            name: {
+                "files": table_files(store, spec),
+                "columns": list(spec.columns),
+                "key": list(spec.key),
+                "description": spec.description,
+            }
+            for name, spec in TABLES.items()
+        },
+    }
+    text = json.dumps(manifest, indent=2) + "\n"
+    if store.get(MANIFEST_KEY) != text:
+        store[MANIFEST_KEY] = text
+    return manifest
+
+
+def collect(
+    settings: Settings,
+    *,
+    store: MutableMapping | None = None,
+    platforms: Mapping[str, Platform] = PLATFORMS,
+    http: Http | None = None,
+    select: Select | None = None,
+    dependents: Dependents = ecosystems_dependents,
+    now: Callable[[], datetime] = lambda: datetime.now(timezone.utc),
+    log: Callable[[str], None] = _log.info,
+) -> RunReport:
+    """Run one collection into ``store`` and return its report.
+
+    Args:
+        settings: what to collect (:func:`~pkgreach.config.load_settings`).
+        store: where the history lives; default ``~/.local/share/pkgreach/data``.
+        platforms: name to :class:`~pkgreach.platforms.Platform`; add one here
+            to support a new registry without touching anything else.
+        http: the transport (default :func:`~pkgreach.http.make_http`).
+        select: ``(platform, metadata) -> keep?``, on top of the settings.
+        dependents: ``(ecosystem, package, *, http) -> {metric: count}``; the
+            default asks ecosyste.ms for dependent packages and repositories.
+        now: the clock; log: where the one-line summary goes.
+    """
+    store = Files(DFLT_DATA_DIR) if store is None else store
+    http = http or make_http()
+    t0 = time.monotonic()
+    started = now()
+    today = started.date()
+    report = RunReport(started_at=started.isoformat(timespec="seconds"))
+    rows: dict[str, list[dict]] = {name: [] for name in TABLES}
+    old_packages = {
+        (r["platform"], r["package"]): r for r in read_table(store, TABLES["packages"])
+    }
+    unknown = sorted(set(settings.platforms) - set(platforms))
+    if unknown:
+        raise ValueError(f"Unknown platform(s) {unknown}; known: {sorted(platforms)}")
+
+    def on_error(where: str, e: Exception) -> None:
+        report.errors[where] = f"{type(e).__name__}: {e}"
+
+    for pname, selection in settings.platforms.items():
+        platform = platforms[pname]
+        chosen = select_packages(
+            platform, selection, http=http, select=select, on_error=on_error
+        )
+        last = _last_dates(store, pname)
+
+        def one(item: tuple[str, dict]):
+            name, record = item
+            since = None
+            if name in last:
+                since = date.fromisoformat(last[name]) - timedelta(DFLT_OVERLAP_DAYS)
+            got = platform.fetch(record, http=http, today=today, since=since)
+            if settings.dependents and platform.ecosystem:
+                try:  # a dependents outage never costs the downloads
+                    deps = dependents(platform.ecosystem, name, http=http)
+                    got["metrics"] = {**deps, **got.get("metrics", {})}
+                except Exception as e:  # noqa: BLE001 - recorded, run goes on
+                    got["dependents_error"] = e
+            return got
+
+        def safely(item):
+            try:
+                return item[0], one(item), None
+            except Exception as e:  # noqa: BLE001 - recorded per package
+                return item[0], None, e
+
+        n_ok = n_err = 0
+        listing_ok = not any(k.startswith(f"{pname}:@") for k in report.errors)
+        with ThreadPoolExecutor(max_workers=max(1, settings.max_workers)) as pool:
+            for name, got, err in pool.map(safely, chosen.items()):
+                old = old_packages.get((pname, name), {})
+                if err is not None:
+                    on_error(f"{pname}:{name}", err)
+                    n_err += 1
+                    # still followed: keep it current, with what we knew
+                    rows["packages"].append(
+                        {
+                            **old,
+                            "platform": pname,
+                            "package": name,
+                            "first_seen": old.get("first_seen") or today.isoformat(),
+                            "last_seen": today.isoformat(),
+                        }
+                    )
+                    continue
+                n_ok += 1
+                if "dependents_error" in got:
+                    on_error(f"{pname}:{name}:dependents", got["dependents_error"])
+                rows["downloads"] += [
+                    {"platform": pname, "package": name, **r}
+                    for r in got.get("downloads", [])
+                    if r["count"]
+                ]
+                rows["metrics"] += [
+                    {"date": today.isoformat(), "platform": pname, "package": name}
+                    | {"metric": m, "value": v}
+                    for m, v in got.get("metrics", {}).items()
+                    if v is not None
+                ]
+                fetched_from = got.get("history_from", "")
+                rows["packages"].append(
+                    {
+                        **got.get("info", {}),
+                        "platform": pname,
+                        "package": name,
+                        "history_from": min(
+                            filter(None, [old.get("history_from"), fetched_from]),
+                            default="",
+                        ),
+                        "first_seen": old.get("first_seen") or today.isoformat(),
+                        "last_seen": today.isoformat(),
+                    }
+                )
+        report.packages[pname] = n_ok
+        if platform.prune_unlisted and listing_ok:
+            report.files_changed += _prune(store, pname, keep=set(chosen))
+        rows["runs"].append(
+            {
+                "started_at": report.started_at,
+                "platform": pname,
+                "packages": n_ok,
+                "errors": n_err,
+            }
+        )
+
+    report.duration_s = round(time.monotonic() - t0, 1)
+    for r in rows["runs"]:
+        r["duration_s"] = report.duration_s
+    for name, spec in TABLES.items():
+        if rows[name]:
+            report.files_changed += write_rows(store, spec, rows[name])
+    store[LAST_RUN_KEY] = json.dumps(asdict(report), indent=2) + "\n"
+    if store.get(DATA_LICENSE_KEY) != DATA_LICENSE:
+        store[DATA_LICENSE_KEY] = DATA_LICENSE
+    write_manifest(store, updated=today.isoformat())
+    log(report.summary())
+    return report