pypi-tracker 0.1.0__py3-none-any.whl

pypi_tracker/__init__.py

@@ -0,0 +1,22 @@
+"""Collect, permanently preserve, aggregate and export PyPI download statistics."""
+
+from .aggregation import find_gaps, monthly, rolling_window, summarize, weekly, yearly
+from .config import ReportSettings
+from .errors import ConfigError, HistoryError, PartialFailure, PyPIStatsError, TrackerError
+from .tracker import Tracker
+
+__all__ = [
+    "ConfigError",
+    "HistoryError",
+    "PartialFailure",
+    "PyPIStatsError",
+    "ReportSettings",
+    "Tracker",
+    "TrackerError",
+    "find_gaps",
+    "monthly",
+    "rolling_window",
+    "summarize",
+    "weekly",
+    "yearly",
+]

pypi_tracker/aggregation.py

@@ -0,0 +1,130 @@
+"""Pure calculations over daily observations. Missing days are never treated as zero."""
+
+import calendar
+from collections.abc import Callable, Iterator
+from datetime import date, timedelta
+
+from .storage import History
+
+_ONE_DAY = timedelta(days=1)
+
+
+def tally(history: History, start: date, end: date) -> dict:
+    """Total an inclusive date range. `downloads` is None when no day in it was recorded."""
+    days = (end - start).days + 1
+    values = [history[day] for day in _dates(start, end) if day in history]
+    return {
+        "start": start.isoformat(),
+        "end": end.isoformat(),
+        "days": days,
+        "downloads": sum(values) if values else None,
+        "recorded_days": len(values),
+        "missing_days": days - len(values),
+        "complete": len(values) == days,
+    }
+
+
+def rolling_window(history: History, days: int) -> dict:
+    """Tally the `days` days ending at the latest recorded date, not at today's date.
+
+    `observations` has one entry per day in the window, so charts always get `days` points.
+    A day with no data has `downloads: None`. With no history at all the list is empty.
+    """
+    if days < 1:
+        raise ValueError(f"days must be positive, got {days}")
+    if not history:
+        return {
+            "start": None,
+            "end": None,
+            "days": days,
+            "downloads": None,
+            "recorded_days": 0,
+            "missing_days": days,
+            "complete": False,
+            "observations": [],
+        }
+    end = max(history)
+    start = end - _ONE_DAY * (days - 1)
+    return {
+        **tally(history, start, end),
+        "observations": [
+            {"date": day.isoformat(), "downloads": history.get(day)} for day in _dates(start, end)
+        ],
+    }
+
+
+def weekly(history: History) -> list[dict]:
+    """ISO calendar weeks, Monday through Sunday."""
+    return [
+        {"week": "{}-W{:02d}".format(*start.isocalendar()[:2]), **tally(history, start, end)}
+        for start, end in _spans(history, _week_span)
+    ]
+
+
+def monthly(history: History) -> list[dict]:
+    return [
+        {"month": f"{start:%Y-%m}", **tally(history, start, end)}
+        for start, end in _spans(history, _month_span)
+    ]
+
+
+def yearly(history: History) -> list[dict]:
+    return [
+        {"year": start.year, **tally(history, start, end)}
+        for start, end in _spans(history, _year_span)
+    ]
+
+
+def find_gaps(history: History) -> list[dict]:
+    """Runs of unrecorded days between the earliest and latest observations."""
+    days = sorted(history)
+    return [
+        {
+            "start": (before + _ONE_DAY).isoformat(),
+            "end": (after - _ONE_DAY).isoformat(),
+            "days": (after - before).days - 1,
+        }
+        for before, after in zip(days, days[1:])
+        if after - before > _ONE_DAY
+    ]
+
+
+def summarize(history: History) -> dict:
+    """Overall coverage. Totals are "recorded": history may not reach back to a package's start."""
+    gaps = find_gaps(history)
+    return {
+        "earliest_date": min(history).isoformat() if history else None,
+        "latest_date": max(history).isoformat() if history else None,
+        "recorded_downloads": sum(history.values()),
+        "recorded_days": len(history),
+        "missing_days": sum(gap["days"] for gap in gaps),
+        "gaps": gaps,
+    }
+
+
+def _dates(start: date, end: date) -> list[date]:
+    return [start + _ONE_DAY * offset for offset in range((end - start).days + 1)]
+
+
+def _spans(history: History, span: Callable[[date], tuple[date, date]]) -> Iterator[tuple[date, date]]:
+    """Every period from the earliest to latest observation, including periods with no data."""
+    if not history:
+        return
+    day, last = min(history), max(history)
+    while day <= last:
+        start, end = span(day)
+        yield start, end
+        day = end + _ONE_DAY
+
+
+def _week_span(day: date) -> tuple[date, date]:
+    monday = day - _ONE_DAY * day.weekday()
+    return monday, monday + _ONE_DAY * 6
+
+
+def _month_span(day: date) -> tuple[date, date]:
+    return day.replace(day=1), day.replace(day=calendar.monthrange(day.year, day.month)[1])
+
+
+def _year_span(day: date) -> tuple[date, date]:
+    return date(day.year, 1, 1), date(day.year, 12, 31)

pypi_tracker/cli.py

@@ -0,0 +1,117 @@
+"""The `pypi-tracker` command line. All real work happens in Tracker."""
+
+import argparse
+import sys
+from collections.abc import Sequence
+from importlib.metadata import version
+
+from .config import resolve_path
+from .errors import ConfigError, TrackerError
+from .tracker import Tracker
+
+EXAMPLE_CONFIG = """\
+packages = ["example-package"]
+
+[reports]
+rolling_days = [7, 30, 90, 180]
+calendar = ["weekly", "monthly", "yearly"]
+historical = ["daily"]
+summary = true
+
+[storage]
+directory = "data/"
+
+[output]
+directory = "public/"
+"""
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+    try:
+        return args.command(args)
+    except (TrackerError, OSError) as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="pypi-tracker", description="Preserve PyPI download history and export static JSON."
+    )
+    # shortcut: needs installed package metadata, so the CLI won't start from a bare PYTHONPATH checkout.
+    parser.add_argument("--version", action="version", version=f"%(prog)s {version('pypi-tracker')}")
+    commands = parser.add_subparsers(required=True)
+
+    source = argparse.ArgumentParser(add_help=False)
+    source.add_argument("--config", help="TOML configuration file")
+    source.add_argument("--data-dir", help="directory of historical CSV files")
+    source.add_argument(
+        "--package", action="append", help="package to process, instead of the config's (repeatable)"
+    )
+    output = argparse.ArgumentParser(add_help=False)
+    output.add_argument("--output-dir", help="directory receiving JSON reports")
+
+    def add(name: str, handler, help: str, parents: list) -> argparse.ArgumentParser:
+        command = commands.add_parser(name, help=help, parents=parents)
+        command.set_defaults(command=handler)
+        return command
+
+    add("collect", _collect, "fetch observations and merge them into CSV storage", [source])
+    add("build", _build, "generate JSON reports from stored history", [source, output])
+    add("sync", _sync, "collect, then build", [source, output])
+    add("validate", _validate, "check the historical dataset", [source])
+
+    init = add("init", _init, "write an example configuration file", [])
+    init.add_argument("--output", default="pypi-tracker.toml", help="where to write it")
+    return parser
+
+
+def _tracker(args: argparse.Namespace) -> Tracker:
+    output_dir = getattr(args, "output_dir", None)
+    if args.config:
+        return Tracker.from_config(
+            args.config, data_dir=args.data_dir, output_dir=output_dir, packages=args.package
+        )
+    paths = {"data_dir": args.data_dir, "output_dir": output_dir}
+    return Tracker(args.package or (), **{key: value for key, value in paths.items() if value})
+
+
+def _collect(args: argparse.Namespace) -> int:
+    tracker = _tracker(args)
+    changed = tracker.collect()
+    print(f"collected {len(tracker.packages)} package(s), {len(changed)} changed")
+    return 0
+
+
+def _build(args: argparse.Namespace) -> int:
+    written = _tracker(args).build()
+    print(f"built reports, {len(written)} file(s) changed")
+    return 0
+
+
+def _sync(args: argparse.Namespace) -> int:
+    changed, written = _tracker(args).sync()
+    print(f"synced: {len(changed)} package(s) changed, {len(written)} report file(s) changed")
+    return 0
+
+
+def _validate(args: argparse.Namespace) -> int:
+    results = _tracker(args).validate()
+    for package, result in results.items():
+        print(f"{package}: {result['recorded_days']} recorded day(s)")
+        for problem in result["problems"]:
+            print(f"  problem: {problem}")
+        for gap in result["gaps"]:
+            print(f"  coverage gap: {gap['start']} to {gap['end']} ({gap['days']} day(s))")
+    return 1 if any(result["problems"] for result in results.values()) else 0
+
+
+def _init(args: argparse.Namespace) -> int:
+    path = resolve_path(args.output)
+    if path.exists():
+        raise ConfigError(f"{path} already exists")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(EXAMPLE_CONFIG, encoding="utf-8")
+    print(f"wrote {path}")
+    return 0

pypi_tracker/client.py

@@ -0,0 +1,78 @@
+"""Read daily download counts from the PyPIStats `/overall` endpoint."""
+
+import http.client
+import json
+import time
+from datetime import UTC, date, datetime
+from email.utils import parsedate_to_datetime
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+from .errors import PyPIStatsError
+
+URL = "https://pypistats.org/api/packages/{package}/overall?mirrors=false"
+CATEGORY = "without_mirrors"
+TIMEOUT_SECONDS = 30
+MAX_ATTEMPTS = 4
+MAX_WAIT_SECONDS = 120
+
+
+def fetch_overall(package: str) -> dict[date, int]:
+    """Return all daily observations PyPIStats has for a package."""
+    request = Request(URL.format(package=package), headers={"User-Agent": "pypi-tracker"})
+    problem = ""
+    for attempt in range(1, MAX_ATTEMPTS + 1):
+        wait = 2**attempt
+        try:
+            with urlopen(request, timeout=TIMEOUT_SECONDS) as response:
+                body = response.read()
+        except HTTPError as error:
+            if error.code == 404:
+                raise PyPIStatsError("package not found on PyPIStats") from error
+            if error.code != 429 and error.code < 500:
+                raise PyPIStatsError(f"PyPIStats returned HTTP {error.code}") from error
+            problem = f"PyPIStats returned HTTP {error.code}"
+            wait = _retry_after(error.headers.get("Retry-After")) or wait
+        except (URLError, OSError, http.client.HTTPException) as error:
+            problem = f"network error: {error}"
+        else:
+            return parse_overall(body)
+
+        if wait > MAX_WAIT_SECONDS:
+            raise PyPIStatsError(f"{problem}; server asked to wait {wait:.0f} seconds")
+        if attempt < MAX_ATTEMPTS:
+            time.sleep(wait)
+    raise PyPIStatsError(f"{problem} (after {MAX_ATTEMPTS} attempts)")
+
+
+def parse_overall(body: bytes) -> dict[date, int]:
+    """Turn a response body into {date: downloads}, rejecting anything suspicious."""
+    observations: dict[date, int] = {}
+    try:
+        for row in json.loads(body)["data"]:
+            if row["category"] != CATEGORY:
+                continue
+            day = date.fromisoformat(row["date"])
+            downloads = row["downloads"]
+            if type(downloads) is not int or downloads < 0:
+                raise ValueError(f"invalid download count {downloads!r} for {day}")
+            if day in observations:
+                raise ValueError(f"duplicate observation for {day}")
+            observations[day] = downloads
+    except (ValueError, KeyError, TypeError) as error:
+        raise PyPIStatsError(f"malformed PyPIStats response: {error!r}") from error
+    return observations
+
+
+def _retry_after(header: str | None) -> float | None:
+    """Parse a Retry-After header, which is either seconds or an HTTP date."""
+    if not header:
+        return None
+    try:
+        return max(float(header), 0)
+    except ValueError:
+        pass
+    try:
+        return max((parsedate_to_datetime(header) - datetime.now(UTC)).total_seconds(), 0)
+    except (ValueError, TypeError):
+        return None

pypi_tracker/config.py

@@ -0,0 +1,124 @@
+"""Load and validate the TOML configuration file."""
+
+import re
+import tomllib
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+from .errors import ConfigError
+
+DEFAULT_DATA_DIR = "data"
+DEFAULT_OUTPUT_DIR = "public"
+CALENDAR_REPORTS = ("weekly", "monthly", "yearly")
+HISTORICAL_REPORTS = ("daily",)
+
+# Names become file paths, so this also blocks path traversal.
+_PACKAGE_NAME = re.compile(r"[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?")
+
+
+@dataclass(frozen=True)
+class ReportSettings:
+    """Which reports `build` generates. Every report is on by default."""
+
+    rolling_days: Sequence[int] = (7, 30, 90, 180)
+    calendar: Sequence[str] = CALENDAR_REPORTS
+    historical: Sequence[str] = HISTORICAL_REPORTS
+    summary: bool = True
+
+    def __post_init__(self):
+        _require_list("reports.rolling_days", self.rolling_days)
+        for days in self.rolling_days:
+            if type(days) is not int or days < 1:
+                raise ConfigError(f"reports.rolling_days must be positive integers, got {days!r}")
+        _require_choices("reports.calendar", self.calendar, CALENDAR_REPORTS)
+        _require_choices("reports.historical", self.historical, HISTORICAL_REPORTS)
+        if type(self.summary) is not bool:
+            raise ConfigError(f"reports.summary must be true or false, got {self.summary!r}")
+
+
+@dataclass(frozen=True)
+class Config:
+    packages: list[str]
+    reports: ReportSettings
+    data_dir: Path | None
+    output_dir: Path | None
+
+
+def normalize_packages(names: Iterable[str]) -> list[str]:
+    """Validate names and apply PEP 503 normalization, so `Foo_Bar` and `foo-bar` share one CSV."""
+    packages = []
+    for name in names:
+        if not isinstance(name, str) or not _PACKAGE_NAME.fullmatch(name):
+            raise ConfigError(f"invalid package name: {name!r}")
+        packages.append(re.sub(r"[-_.]+", "-", name).lower())
+    return list(dict.fromkeys(packages))
+
+
+def resolve_path(value: str | Path, base: Path | None = None) -> Path:
+    """Expand `~` and make a path absolute. Relative paths start at `base` (default: the cwd)."""
+    return ((base or Path.cwd()) / Path(value).expanduser()).resolve()
+
+
+def load_config(path: str | Path) -> Config:
+    path = resolve_path(path)
+    try:
+        raw = tomllib.loads(path.read_text(encoding="utf-8"))
+    except OSError as error:
+        raise ConfigError(f"cannot read config file {path}: {error}") from error
+    except (tomllib.TOMLDecodeError, UnicodeDecodeError) as error:
+        raise ConfigError(f"invalid TOML in {path}: {error}") from error
+    try:
+        return _parse(raw, base=path.parent)
+    except ConfigError as error:
+        raise ConfigError(f"{path}: {error}") from error
+
+
+def _parse(raw: dict, base: Path) -> Config:
+    _reject_unknown_keys("top level", raw, {"packages", "reports", "storage", "output"})
+
+    packages = raw.get("packages") or []
+    _require_list("packages", packages)
+
+    reports = _section(raw, "reports", {"rolling_days", "calendar", "historical", "summary"})
+    return Config(
+        packages=normalize_packages(packages),
+        reports=ReportSettings(**reports),
+        data_dir=_directory(raw, "storage", base),
+        output_dir=_directory(raw, "output", base),
+    )
+
+
+def _directory(raw: dict, key: str, base: Path) -> Path | None:
+    directory = _section(raw, key, {"directory"}).get("directory")
+    if directory is None:
+        return None
+    if not isinstance(directory, str):
+        raise ConfigError(f"{key}.directory must be a path, got {directory!r}")
+    return resolve_path(directory, base)
+
+
+def _section(raw: dict, key: str, allowed: set[str]) -> dict:
+    section = raw.get(key) or {}
+    if not isinstance(section, dict):
+        raise ConfigError(f"'{key}' must be a mapping")
+    _reject_unknown_keys(key, section, allowed)
+    return section
+
+
+def _reject_unknown_keys(where: str, mapping: dict, allowed: set[str]) -> None:
+    unknown = sorted(str(key) for key in set(mapping) - allowed)
+    if unknown:
+        raise ConfigError(f"unknown key(s) in {where}: {', '.join(unknown)}")
+
+
+def _require_list(name: str, value: object) -> None:
+    if not isinstance(value, (list, tuple)):
+        raise ConfigError(f"{name} must be a list, got {value!r}")
+
+
+def _require_choices(name: str, value: object, choices: tuple[str, ...]) -> None:
+    _require_list(name, value)
+    for item in value:
+        if item not in choices:
+            raise ConfigError(f"{name}: unknown report {item!r}; choose from {', '.join(choices)}")

pypi_tracker/errors.py

@@ -0,0 +1,26 @@
+"""Exceptions raised by pypi-tracker."""
+
+
+class TrackerError(Exception):
+    """Base class for every expected pypi-tracker failure."""
+
+
+class ConfigError(TrackerError):
+    """The configuration or a package name is invalid."""
+
+
+class PyPIStatsError(TrackerError):
+    """PyPIStats could not provide usable data for a package."""
+
+
+class HistoryError(TrackerError):
+    """A historical CSV file is missing or invalid."""
+
+
+class PartialFailure(TrackerError):
+    """One or more packages failed; the others were still processed."""
+
+    def __init__(self, action: str, failures: dict[str, str]):
+        self.failures = failures
+        details = "\n".join(f"  {package}: {reason}" for package, reason in failures.items())
+        super().__init__(f"{action} failed for {len(failures)} package(s):\n{details}")

pypi_tracker/reports.py

@@ -0,0 +1,87 @@
+"""Shape aggregates into report payloads and publish them as static JSON files."""
+
+import json
+from pathlib import Path
+
+from . import aggregation
+from .config import ReportSettings
+from .storage import History, write_atomic
+
+_CALENDAR = {
+    "weekly": aggregation.weekly,
+    "monthly": aggregation.monthly,
+    "yearly": aggregation.yearly,
+}
+
+
+def rolling_report(package: str, history: History, days: int) -> dict:
+    window = aggregation.rolling_window(history, days)
+    return {
+        "package": package,
+        "report": f"last-{days}-days",
+        "as_of": _as_of(history),
+        "period": {"start": window["start"], "end": window["end"], "days": days},
+        "downloads": window["downloads"],
+        "recorded_days": window["recorded_days"],
+        "missing_days": window["missing_days"],
+        "complete": window["complete"],
+        "observations": window["observations"],
+    }
+
+
+def calendar_report(package: str, history: History, name: str) -> dict:
+    """`name` is weekly, monthly or yearly."""
+    return {
+        "package": package,
+        "report": name,
+        "as_of": _as_of(history),
+        "periods": _CALENDAR[name](history),
+    }
+
+
+def summary_report(package: str, history: History) -> dict:
+    return {
+        "package": package,
+        "report": "summary",
+        "as_of": _as_of(history),
+        **aggregation.summarize(history),
+    }
+
+
+def daily_reports(package: str, history: History) -> dict[int, dict]:
+    """One payload per calendar year that has observations."""
+    years: dict[int, dict] = {}
+    for day, downloads in sorted(history.items()):
+        year = years.setdefault(
+            day.year, {"package": package, "year": day.year, "observations": []}
+        )
+        year["observations"].append({"date": day.isoformat(), "downloads": downloads})
+    return years
+
+
+def write_reports(
+    package: str, history: History, output_dir: Path, settings: ReportSettings
+) -> list[Path]:
+    """Write a package's selected reports and return the files that actually changed."""
+    files: dict[str, str] = {}
+    if settings.summary:
+        files["summary.json"] = _readable(summary_report(package, history))
+    for days in settings.rolling_days:
+        files[f"last-{days}-days.json"] = _readable(rolling_report(package, history, days))
+    for name in settings.calendar:
+        files[f"{name}.json"] = _readable(calendar_report(package, history, name))
+    if "daily" in settings.historical:
+        for year, payload in daily_reports(package, history).items():
+            files[f"daily/{year}.json"] = json.dumps(payload, separators=(",", ":")) + "\n"
+
+    root = output_dir / "packages" / package
+    return [root / name for name, text in files.items() if write_atomic(root / name, text)]
+
+
+def _as_of(history: History) -> str | None:
+    """The actual date of the newest data, so stale reports can be recognized as stale."""
+    return max(history).isoformat() if history else None
+
+
+def _readable(payload: dict) -> str:
+    return json.dumps(payload, indent=2) + "\n"

pypi_tracker/storage.py

@@ -0,0 +1,96 @@
+"""Historical CSV files (the source of truth) and atomic file writes."""
+
+import csv
+import os
+import re
+from datetime import date
+from pathlib import Path
+
+from .errors import HistoryError
+
+History = dict[date, int]
+
+_HEADER = ["date", "downloads"]
+
+
+def write_atomic(path: Path, text: str) -> bool:
+    """Replace a file's contents in one step. Returns False if it already had this content."""
+    data = text.encode("utf-8")
+    if path.exists() and path.read_bytes() == data:
+        return False
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
+    try:
+        with temporary.open("wb") as file:
+            file.write(data)
+            file.flush()
+            os.fsync(file.fileno())
+        os.replace(temporary, path)
+    finally:
+        temporary.unlink(missing_ok=True)
+    return True
+
+
+def scan_csv(path: Path) -> tuple[History, list[str]]:
+    """Read a history file, returning its valid observations and every problem found."""
+    if not path.exists():
+        return {}, ["file not found"]
+
+    history: History = {}
+    problems: list[str] = []
+    previous: date | None = None
+    with path.open(newline="", encoding="utf-8", errors="replace") as file:
+        reader = csv.reader(file)
+        try:
+            header = next(reader, None)
+            if header != _HEADER:
+                problems.append(f"line 1: expected header date,downloads, got {header}")
+            for row in reader:
+                where = f"line {reader.line_num}"
+                if len(row) != 2:
+                    problems.append(f"{where}: expected 2 columns, got {row}")
+                    continue
+                day = _parse_date(row[0])
+                if day is None:
+                    problems.append(f"{where}: malformed date {row[0]!r}")
+                    continue
+                if not re.fullmatch(r"-?[0-9]+", row[1]):
+                    problems.append(f"{where}: malformed download count {row[1]!r}")
+                    continue
+                downloads = int(row[1])
+                if downloads < 0:
+                    problems.append(f"{where}: negative download count {downloads}")
+                    continue
+                if day in history:
+                    problems.append(f"{where}: duplicate observation for {day}")
+                    continue
+                if previous and day < previous:
+                    problems.append(f"{where}: {day} is out of order (after {previous})")
+                history[day] = downloads
+                previous = day
+        except csv.Error as error:
+            problems.append(f"line {reader.line_num}: {error}")
+    return history, problems
+
+
+def read_history(path: Path) -> History:
+    """Read a history file, refusing to return partial data from an invalid one."""
+    history, problems = scan_csv(path)
+    if problems:
+        raise HistoryError(f"{path}: " + "; ".join(problems))
+    return history
+
+
+def write_history(path: Path, history: History) -> bool:
+    """Write observations in date order. Returns False if the file was already up to date."""
+    rows = "".join(f"{day.isoformat()},{downloads}\n" for day, downloads in sorted(history.items()))
+    return write_atomic(path, "date,downloads\n" + rows)
+
+
+def _parse_date(text: str) -> date | None:
+    try:
+        day = date.fromisoformat(text)
+    except ValueError:
+        return None
+    # fromisoformat also accepts forms such as 20260101; stored dates must be YYYY-MM-DD.
+    return day if day.isoformat() == text else None

pypi_tracker/tracker.py

@@ -0,0 +1,133 @@
+"""The public Python API: collect, build, sync and validate."""
+
+from collections.abc import Iterable
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Self
+
+from . import aggregation, reports
+from .client import fetch_overall
+from .config import (
+    DEFAULT_DATA_DIR,
+    DEFAULT_OUTPUT_DIR,
+    ReportSettings,
+    load_config,
+    normalize_packages,
+    resolve_path,
+)
+from .errors import ConfigError, PartialFailure, TrackerError
+from .storage import History, read_history, scan_csv, write_history
+
+
+class Tracker:
+    def __init__(
+        self,
+        packages: Iterable[str] = (),
+        data_dir: str | Path = DEFAULT_DATA_DIR,
+        output_dir: str | Path = DEFAULT_OUTPUT_DIR,
+        reports: ReportSettings = ReportSettings(),
+    ):
+        self.packages = normalize_packages(packages)
+        self.data_dir = resolve_path(data_dir)
+        self.output_dir = resolve_path(output_dir)
+        self.reports = reports
+
+    @classmethod
+    def from_config(
+        cls,
+        path: str | Path,
+        data_dir: str | Path | None = None,
+        output_dir: str | Path | None = None,
+        packages: Iterable[str] | None = None,
+    ) -> Self:
+        """Load a config file. Explicit arguments win over its values, which win over defaults."""
+        config = load_config(path)
+        return cls(
+            packages=packages or config.packages,
+            data_dir=data_dir or config.data_dir or DEFAULT_DATA_DIR,
+            output_dir=output_dir or config.output_dir or DEFAULT_OUTPUT_DIR,
+            reports=config.reports,
+        )
+
+    def collect(self) -> list[str]:
+        """Merge PyPIStats data into each package's CSV and return the packages that changed.
+
+        Raises PartialFailure after trying every package if any of them failed.
+        """
+        if not self.packages:
+            raise ConfigError("no packages to collect: use --package or list them in a config file")
+        changed, failures = [], {}
+        for package in self.packages:
+            try:
+                if self._collect_one(package):
+                    changed.append(package)
+            except (TrackerError, OSError) as error:
+                failures[package] = str(error)
+        if failures:
+            raise PartialFailure("collection", failures)
+        return changed
+
+    def build(self, packages: Iterable[str] | None = None) -> list[Path]:
+        """Generate JSON reports from local CSVs and return the files that changed.
+
+        Never touches the network. Raises PartialFailure after trying every package.
+        """
+        written, failures = [], {}
+        for package in self._selected() if packages is None else packages:
+            try:
+                written += reports.write_reports(
+                    package, self.history(package), self.output_dir, self.reports
+                )
+            except (TrackerError, OSError) as error:
+                failures[package] = str(error)
+        if failures:
+            raise PartialFailure("report generation", failures)
+        return written
+
+    def sync(self) -> tuple[list[str], list[Path]]:
+        """Collect, then build. Returns (changed packages, changed report files).
+
+        If collection fails for a package, the others are still published, its existing reports
+        are left alone (their `as_of` shows the data's age), and PartialFailure is raised.
+        """
+        try:
+            changed = self.collect()
+        except PartialFailure as failure:
+            self.build(package for package in self.packages if package not in failure.failures)
+            raise
+        return changed, self.build()
+
+    def history(self, package: str) -> History:
+        """Stored observations for a package as {date: downloads}."""
+        return read_history(self._csv_path(package))
+
+    def validate(self) -> dict[str, dict]:
+        """Check each CSV. `problems` are invalid data; `gaps` are only missing coverage."""
+        results = {}
+        for package in self._selected():
+            history, problems = scan_csv(self._csv_path(package))
+            results[package] = {
+                "problems": problems,
+                "gaps": aggregation.find_gaps(history),
+                "recorded_days": len(history),
+            }
+        return results
+
+    def _collect_one(self, package: str) -> bool:
+        path = self._csv_path(package)
+        # Read first: a damaged CSV must stop us before anything can overwrite it.
+        existing = read_history(path) if path.exists() else {}
+        today = datetime.now(UTC).date()
+        fetched = {day: count for day, count in fetch_overall(package).items() if day < today}
+        # Upstream wins on overlap (corrections); dates it no longer returns are kept (retention).
+        return write_history(path, existing | fetched)
+
+    def _csv_path(self, package: str) -> Path:
+        return self.data_dir / f"{package}.csv"
+
+    def _selected(self) -> list[str]:
+        """Configured packages, or else every package with history in the data directory."""
+        packages = self.packages or sorted(path.stem for path in self.data_dir.glob("*.csv"))
+        if not packages:
+            raise ConfigError(f"no packages configured and no CSV files found in {self.data_dir}")
+        return packages

pypi_tracker-0.1.0.dist-info/METADATA

@@ -0,0 +1,107 @@
+Metadata-Version: 2.4
+Name: pypi-tracker
+Version: 0.1.0
+Summary: Permanently preserve PyPI download history and export it as static JSON reports.
+License-Expression: MIT
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Dynamic: license-file
+
+# pypi-tracker
+
+pypi-tracker is a lightweight tool for collecting and preserving PyPI package download statistics. It maintains historical data beyond [PyPIStats](https://pypistats.org)' retention window. It generates static, chart-ready JSON reports that can be used to visualize download trends, track package adoption, and power dashboards or other applications without a backend.
+
+
+## Try it
+
+```bash
+pip install pypi-tracker
+
+
+pypi-tracker sync --package dj-control-room
+```
+
+You now have the full history in `data/dj-control-room.csv` and reports in
+`public/packages/dj-control-room/`, such as `last-30-days.json`:
+
+```json
+{
+  "package": "dj-control-room",
+  "report": "last-30-days",
+  "as_of": "2026-11-01",
+  "downloads": 38071,
+  "complete": true,
+  "observations": [
+    {"date": "2026-10-03", "downloads": 510},
+    {"date": "2026-10-04", "downloads": 757}
+  ]
+}
+```
+
+The `observations` list has one entry per day in the window (shortened here). A day with no data is `null`,
+never `0`, and `complete` tells you whether any day is missing.
+
+## Commands
+
+| Command | What it does |
+|---|---|
+| `init` | Creates a starter config file. |
+| `sync` | Runs `collect`, then `build`. This is the one to schedule. |
+| `collect` | Fetches new data from PyPIStats and merges it into the CSVs. |
+| `build` | Writes the JSON report files to publish. |
+| `validate` | Checks the CSVs for bad data and missing dates. |
+
+## Tracking several packages
+
+You will typically want to keep a config file where you specify the packages you want to track.
+
+```bash
+pypi-tracker init  # generates a new pypi-tracker.toml file in the current directory
+```
+
+A complete config file will have the sections seen below. Note that the `pypi-tracker` also
+allows for cli args that can supersede these config values; though typically it is much
+easier to simply rely on a config file.
+```toml
+# pypi-tracker.toml
+packages = [
+    "dj-control-room",
+    "dj-control-room-base",
+    "dj-redis-panel",
+    "dj-cache-panel",
+    "dj-celery-panel",
+    "dj-urls-panel",
+    "dj-signals-panel",
+]
+
+[reports]
+rolling_days = [7, 30, 90, 180]
+calendar = ["weekly", "monthly", "yearly"]
+historical = ["daily"]
+summary = true
+
+[storage]
+directory = "data/"
+
+[output]
+directory = "public/"
+```
+
+You can update both reports and historical data by running a single `sync` command.
+```bash
+pypi-tracker sync --config pypi-tracker.toml
+```
+
+## Keep your CSVs
+
+Run `sync` on a schedule and commit `data/`, because **the CSV files are your history**. If you lose them, or
+stop collecting for longer than PyPIStats keeps data, those days are gone. History also can't reach back
+further than PyPIStats did on your first run, so totals are "recorded" downloads, not lifetime downloads.
+
+## More
+
+The [guide](docs/guide.md) covers configuration, every report format, the Python API, and GitHub Actions and
+Cloudflare Pages setup.

pypi_tracker-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+pypi_tracker/__init__.py,sha256=Zf4emouK-WZRAqVCYUKgrM2CvByyWcM3yiBwkkCdpgE,582
+pypi_tracker/aggregation.py,sha256=lnEXkrcjMx5fRdnO9Z_gUucuJ3Q80yrdbJ5Gatyaajc,4234
+pypi_tracker/cli.py,sha256=O61WVXG2t35qojNadUpxdB4MjGhZ76H0j3wpBcGGL3s,4207
+pypi_tracker/client.py,sha256=SEYw9iGbrr1pyIN7EZ7RNDl8OdSd7BYz7tjfzECmP9o,3045
+pypi_tracker/config.py,sha256=I2OZTxKIy6r3wmv40oFJSxXhlbGgwXrGZgGC_S6vkxU,4554
+pypi_tracker/errors.py,sha256=jExiEhD3sfYBA_RwYN2_bHkqxO0ClVeldaHRyGsSVrM,812
+pypi_tracker/reports.py,sha256=1GDmnK7GoryGQNr0z-rxGlBAYPxm0RM4QS9ky4Ur3Q4,3043
+pypi_tracker/storage.py,sha256=pIkKSsyK5AwXwSFE69hCo_AoAV9K7PswN71zSHfrDGc,3549
+pypi_tracker/tracker.py,sha256=eHXHrE-OeBKOqA34xkDHQjvTPejz7vM00V-UqWTfIN8,5294
+pypi_tracker-0.1.0.dist-info/licenses/LICENSE,sha256=qCZ1uHegA5eg57GpTtj5Gx4bzmQsdHnt8BcDOQErnX0,1071
+pypi_tracker-0.1.0.dist-info/METADATA,sha256=d5nRzkMg_urhL1w4cwPbXXM3UHuxUZ1EnpxUrv7rPAM,3226
+pypi_tracker-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+pypi_tracker-0.1.0.dist-info/entry_points.txt,sha256=LEG-ZRmSo8fSv6wNDxEuuD3lxMyzPLBdZEfLqME_99o,55
+pypi_tracker-0.1.0.dist-info/top_level.txt,sha256=WbD0M5xUqG8IQpjJLL1PzYmD_TxATYaoNCJBpLeyXs0,13
+pypi_tracker-0.1.0.dist-info/RECORD,,

pypi_tracker-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pypi_tracker-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pypi-tracker = pypi_tracker.cli:main

pypi_tracker-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Yasser Toruño
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pypi_tracker-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+pypi_tracker