ghtraffic 0.0.2__py3-none-any.whl

ghtraffic/__init__.py

@@ -0,0 +1,41 @@
+"""Accumulate GitHub traffic and insights across all your repos and orgs.
+
+GitHub keeps repository traffic (views, clones, referrers, popular paths) for
+14 days only. ``ghtraffic`` collects it on a schedule and merges it into a
+growing, git-friendly history of CSV files, together with star, fork and
+watcher counts, and ships a static dashboard to explore it.
+
+>>> from ghtraffic import collect, load_settings, Files   # doctest: +SKIP
+>>> collect(load_settings(owners=["@me"]), store=Files("~/traffic"))  # doctest: +SKIP
+
+Main pieces:
+
+- :func:`collect`: one idempotent run (cold start and update are the same call).
+- :func:`load_settings` / :class:`Settings`: which owners and repos.
+- :data:`TABLES`: the on-disk schema; :class:`Files`: the default store.
+- :func:`init_data_repo`: write the workflow + config into a data repository.
+"""
+
+from ghtraffic.collect import RunReport, collect, write_manifest
+from ghtraffic.config import Settings, load_settings
+from ghtraffic.github import GitHubError, make_graphql, make_rest, resolve_token
+from ghtraffic.scaffold import init_data_repo
+from ghtraffic.tables import TABLES, Files, TableSpec, merge_rows, read_table
+
+__all__ = [
+    "Files",
+    "GitHubError",
+    "RunReport",
+    "Settings",
+    "TABLES",
+    "TableSpec",
+    "collect",
+    "init_data_repo",
+    "load_settings",
+    "make_graphql",
+    "make_rest",
+    "merge_rows",
+    "read_table",
+    "resolve_token",
+    "write_manifest",
+]

ghtraffic/__main__.py

@@ -0,0 +1,134 @@
+"""Command line: ``ghtraffic collect | init | guard | demo | manifest``.
+
+``python -m ghtraffic collect --data-dir .`` is what the GitHub Action runs.
+Without ``--data-dir``, data goes to ``~/.local/share/ghtraffic/data`` (never
+the current directory, so a local run cannot end up committed by accident).
+Every :class:`~ghtraffic.config.Settings` field is also a ``collect`` flag.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import typing
+from dataclasses import fields
+from pathlib import Path
+
+from ghtraffic.collect import collect, has_private_data, write_manifest
+from ghtraffic.config import (
+    DFLT_CONFIG_FILENAME,
+    DFLT_DATA_DIR,
+    Settings,
+    load_settings,
+)
+from ghtraffic.scaffold import DFLT_CRON, default_action_ref, init_data_repo
+from ghtraffic.tables import Files
+
+
+def _add_settings_flags(parser: argparse.ArgumentParser) -> None:
+    """One flag per Settings field, so a new setting needs no CLI edit."""
+    hints = typing.get_type_hints(Settings)
+    for f in fields(Settings):
+        flag = "--" + f.name.replace("_", "-")
+        kind = hints[f.name]
+        if kind is bool:
+            parser.add_argument(flag, action=argparse.BooleanOptionalAction)
+        elif typing.get_origin(kind) is tuple:
+            parser.add_argument(flag, action="append", help="repeatable")
+        else:
+            parser.add_argument(flag, type=kind)
+
+
+def _parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="ghtraffic", description=__doc__.splitlines()[0])
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    c = sub.add_parser("collect", help="fetch traffic and merge it into the data dir")
+    c.add_argument("--data-dir", default=str(DFLT_DATA_DIR))
+    c.add_argument("--config", help=f"default: <data-dir>/{DFLT_CONFIG_FILENAME}")
+    c.add_argument(
+        "--fail-on-errors",
+        action="store_true",
+        help="exit 1 when some repo or owner failed (never for missing access)",
+    )
+    _add_settings_flags(c)
+
+    i = sub.add_parser("init", help="write the workflow and config into a data repo")
+    i.add_argument("dir", nargs="?", default=".")
+    i.add_argument("--owner", action="append", dest="owners")
+    i.add_argument("--cron", default=DFLT_CRON)
+    i.add_argument("--action-ref", help="default: this ghtraffic's release tag")
+    i.add_argument("--overwrite", action="store_true")
+
+    g = sub.add_parser(
+        "guard", help="exit 1 if private data would be committed to a public repo"
+    )
+    g.add_argument("--data-dir", default=".")
+    g.add_argument("--repo", required=True, help="owner/name of the data repo")
+
+    d = sub.add_parser(
+        "demo", help="write a synthetic data dir (for the demo dashboard)"
+    )
+    d.add_argument("--data-dir", required=True)
+    d.add_argument("--days", type=int, default=420)
+
+    m = sub.add_parser("manifest", help="rewrite manifest.json from the files present")
+    m.add_argument("--data-dir", default=str(DFLT_DATA_DIR))
+    return p
+
+
+def _guard(data_dir: str, repo: str) -> int:
+    from ghtraffic.github import make_rest
+
+    if not make_rest()(f"/repos/{repo}").get("private", False) and has_private_data(
+        Files(data_dir)
+    ):
+        print(
+            f"ghtraffic: REFUSING to commit: {repo} is public and the history "
+            "holds private repos. Make the data repo private, or exclude private "
+            "repos, or set the action input allow-public: true.",
+            file=sys.stderr,
+        )
+        return 1
+    return 0
+
+
+def main(argv=None) -> int:
+    """Entry point of the ``ghtraffic`` command."""
+    args = _parser().parse_args(argv)
+    if args.cmd == "collect":
+        data_dir = Path(args.data_dir)
+        overrides = {f.name: getattr(args, f.name) for f in fields(Settings)}
+        settings = load_settings(
+            args.config or data_dir / DFLT_CONFIG_FILENAME, **overrides
+        )
+        report = collect(settings, store=Files(data_dir), log=print)
+        failed = report.errors or report.owner_errors
+        return 1 if (args.fail_on_errors and failed) else 0
+    if args.cmd == "init":
+        written = init_data_repo(
+            args.dir,
+            owners=args.owners or ("@me",),
+            cron=args.cron,
+            action_ref=args.action_ref or default_action_ref(),
+            overwrite=args.overwrite,
+        )
+        print("\n".join(written) or "nothing written (files exist; --overwrite)")
+        return 0
+    if args.cmd == "guard":
+        return _guard(args.data_dir, args.repo)
+    if args.cmd == "demo":
+        from ghtraffic.demo import make_demo_data
+
+        make_demo_data(Files(args.data_dir), n_days=args.days)
+        return 0
+    if args.cmd == "manifest":
+        from datetime import date
+
+        write_manifest(Files(args.data_dir), updated=date.today().isoformat())
+        return 0
+    return 2  # pragma: no cover
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ghtraffic/collect.py

@@ -0,0 +1,266 @@
+"""The collection run: list repos, fetch what GitHub still holds, merge it in.
+
+:func:`collect` is the one orchestrator. It is safe to run any number of times
+a day and in any order: every table merges idempotently (see
+:mod:`ghtraffic.tables`), so the cold start (the first 14-day backfill) and the
+daily update are the same call.
+
+Repos the token cannot push to have no readable traffic: they are counted and
+listed in ``last_run.json``, never treated as a failure.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from collections.abc import Callable, Iterable, MutableMapping
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import asdict, dataclass, field
+from functools import partial
+from datetime import date, datetime, timezone
+
+from ghtraffic.config import DFLT_DATA_DIR, Settings
+from ghtraffic.github import GitHubError, GraphQL, Rest, make_graphql, make_rest
+from ghtraffic.repos import list_repos
+from ghtraffic.tables import (
+    MANIFEST_KEY,
+    SCHEMA_VERSION,
+    TABLES,
+    Files,
+    read_table,
+    table_files,
+    write_rows,
+)
+from ghtraffic.traffic import fetch_daily, fetch_popular, star_history
+
+LAST_RUN_KEY = "last_run.json"
+_log = logging.getLogger("ghtraffic")
+#: HTTP statuses meaning "this token cannot read this repo's traffic".
+NO_ACCESS_STATUSES = frozenset({401, 403, 404})
+
+
+@dataclass
+class RunReport:
+    """What a run did, for logs and ``last_run.json``."""
+
+    started_at: str
+    duration_s: float = 0.0
+    repos_listed: int = 0
+    repos_collected: int = 0
+    snapshots: int = 0
+    no_access: list[str] = field(default_factory=list)
+    errors: dict[str, str] = field(default_factory=dict)
+    owner_errors: dict[str, str] = field(default_factory=dict)
+    files_changed: int = 0
+
+    def summary(self) -> str:
+        """One human line, with no repo names (safe for public logs)."""
+        return (
+            f"ghtraffic: {self.repos_listed} repos listed, "
+            f"{self.repos_collected} collected ({self.snapshots} snapshots), "
+            f"{len(self.no_access)} without push access, "
+            f"{len(self.errors)} errors, {len(self.owner_errors)} owner errors, "
+            f"{self.files_changed} files changed in {self.duration_s:.1f}s"
+        )
+
+
+def _last_snapshots(store: MutableMapping, today: date) -> dict[str, str]:
+    years = {str(today.year), str(today.year - 1)}
+    last: dict[str, str] = {}
+    for r in read_table(store, TABLES["windows"], years=years):
+        last[r["repo"]] = max(last.get(r["repo"], ""), str(r["fetched"]))
+    return last
+
+
+def _snapshot_due(last: str | None, today: date, every_days: int) -> bool:
+    if not last:
+        return True
+    return (today - date.fromisoformat(last)).days >= every_days
+
+
+def write_manifest(store: MutableMapping, *, updated: str) -> dict:
+    """Write ``manifest.json``: the schema and the files of every table."""
+    manifest = {
+        "schema_version": SCHEMA_VERSION,
+        "updated": updated,
+        "tables": {
+            name: {
+                "files": table_files(store, spec),
+                "columns": list(spec.columns),
+                "key": list(spec.key),
+                "description": spec.description,
+            }
+            for name, spec in TABLES.items()
+        },
+    }
+    text = json.dumps(manifest, indent=2) + "\n"
+    if store.get(MANIFEST_KEY) != text:
+        store[MANIFEST_KEY] = text
+    return manifest
+
+
+def _safely(fn: Callable, repo: str):
+    """``(repo, fn(repo), None)``, or ``(repo, None, error)``: one repo never sinks a run."""
+    try:
+        return repo, fn(repo), None
+    except Exception as e:  # noqa: BLE001 - any failure is recorded per repo
+        return repo, None, e
+
+
+def _repo_row(r: dict, last_seen: str) -> dict:
+    return {
+        **{c: r.get(c, "") for c in TABLES["repos"].columns},
+        "last_seen": last_seen,
+    }
+
+
+def collect(
+    settings: Settings,
+    *,
+    store: MutableMapping | None = None,
+    rest: Rest | None = None,
+    graphql: GraphQL | None = None,
+    list_repos: Callable[..., Iterable[dict]] = list_repos,
+    star_history: Callable[..., list[dict]] = star_history,
+    now: Callable[[], datetime] = lambda: datetime.now(timezone.utc),
+    log: Callable[[str], None] = _log.info,
+) -> RunReport:
+    """Run one collection into ``store`` and return its report.
+
+    Args:
+        settings: what to collect (:func:`~ghtraffic.config.load_settings`).
+        store: where the history lives; default ``~/.local/share/ghtraffic/data``.
+        rest, graphql: GitHub transports; default urllib with an env token.
+        list_repos: ``(owners, *, graphql, rest, exclude, include_forks,
+            include_archived, on_error) -> records`` with the keys of
+            :data:`~ghtraffic.repos.REPO_FIELDS`.
+        star_history: ``(repo, *, graphql) -> stars rows``, for the backfill.
+        now: the clock; log: where the one-line summary goes.
+    """
+    store = Files(DFLT_DATA_DIR) if store is None else store
+    rest = rest or make_rest()
+    graphql = graphql or make_graphql()
+    t0 = time.monotonic()
+    started = now()
+    today = started.date()
+    fetched = today.isoformat()
+    report = RunReport(started_at=started.isoformat(timespec="seconds"))
+    workers = max(1, settings.max_workers)
+
+    def on_owner_error(owner: str, e: Exception) -> None:
+        report.owner_errors[owner] = str(e)
+
+    repos = list(
+        list_repos(
+            settings.owners,
+            graphql=graphql,
+            rest=rest,
+            exclude=settings.exclude,
+            include_forks=settings.include_forks,
+            include_archived=settings.include_archived,
+            on_error=on_owner_error,
+        )
+    )
+    report.repos_listed = len(repos)
+    rows: dict[str, list[dict]] = {name: [] for name in TABLES}
+    rows["repos"] = [_repo_row(r, fetched) for r in repos]
+
+    # star history first, so today's live values (below) win a same-day merge
+    if settings.backfill_stars:
+        done = {
+            r["repo"]
+            for r in read_table(store, TABLES["backfills"])
+            if r["kind"] == "stars"
+        }
+        todo = [
+            r["repo"]
+            for r in repos
+            if r["stars"] and r["can_push"] and r["repo"] not in done
+        ]
+        backfill = partial(star_history, graphql=graphql)
+        with ThreadPoolExecutor(max_workers=workers) as pool:
+            for repo, got, err in pool.map(lambda r: _safely(backfill, r), todo):
+                if err is not None:
+                    report.errors[repo] = f"star backfill: {err}"  # retried next run
+                else:
+                    rows["stars"] += got
+                    rows["backfills"].append(
+                        {"repo": repo, "kind": "stars", "date": fetched}
+                    )
+    rows["stars"] += [
+        {"date": fetched, **{k: r[k] for k in ("repo", "stars", "forks", "watchers")}}
+        for r in repos
+    ]
+
+    # traffic: only where the token can push
+    readable = [r["repo"] for r in repos if r["can_push"]]
+    report.no_access = sorted(r["repo"] for r in repos if not r["can_push"])
+    last_snap = _last_snapshots(store, today)
+    due = {
+        repo: _snapshot_due(last_snap.get(repo), today, settings.snapshot_every_days)
+        for repo in readable
+    }
+
+    def fetch(repo: str) -> dict[str, list[dict]]:
+        got = fetch_daily(repo, rest=rest, fetched=fetched)
+        if not due[repo]:
+            got.pop("windows")
+            return got
+        return {**got, **fetch_popular(repo, rest=rest, fetched=fetched)}
+
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        for repo, got, err in pool.map(lambda r: _safely(fetch, r), readable):
+            if err is not None:
+                if (
+                    isinstance(err, GitHubError)
+                    and err.status in NO_ACCESS_STATUSES
+                    and not (err.is_rate_limit or err.is_policy_block)
+                ):
+                    report.no_access.append(repo)
+                else:
+                    report.errors[repo] = f"{type(err).__name__}: {err}"
+                continue
+            report.repos_collected += 1
+            report.snapshots += bool(due[repo])
+            for name, table_rows in got.items():
+                rows[name] += table_rows
+    report.no_access.sort()
+
+    for name, spec in TABLES.items():
+        if name != "runs" and rows[name]:
+            report.files_changed += write_rows(store, spec, rows[name])
+
+    report.duration_s = round(time.monotonic() - t0, 1)
+    report.files_changed += write_rows(
+        store,
+        TABLES["runs"],
+        [
+            {
+                "started_at": report.started_at,
+                "duration_s": report.duration_s,
+                "repos_listed": report.repos_listed,
+                "repos_collected": report.repos_collected,
+                "repos_no_access": len(report.no_access),
+                "snapshots": report.snapshots,
+                "errors": len(report.errors) + len(report.owner_errors),
+            }
+        ],
+    )
+    store[LAST_RUN_KEY] = json.dumps(asdict(report), indent=2) + "\n"
+    write_manifest(store, updated=fetched)
+    log(report.summary())
+    return report
+
+
+def has_private_data(store: MutableMapping) -> bool:
+    """Whether the history holds anything about a private repo."""
+    return any(r["private"] == "true" for r in read_table(store, TABLES["repos"]))
+
+
+def window_days(store: MutableMapping) -> dict[str, int]:
+    """How many distinct days each daily table holds (a quick health check)."""
+    return {
+        name: len({r["date"] for r in read_table(store, TABLES[name])})
+        for name in ("views", "clones")
+    }

ghtraffic/config.py

@@ -0,0 +1,74 @@
+"""Collection settings: which owners, which repos, how often to snapshot.
+
+Settings come from a TOML file (``ghtraffic.toml`` at the data directory's root
+by default), under a ``[ghtraffic]`` table or at top level, overridden by
+keyword arguments. Every field has a default, so an empty file (or none)
+collects every repo of the authenticated user and their organizations.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, fields
+from pathlib import Path
+
+import tomllib
+
+DFLT_CONFIG_FILENAME = "ghtraffic.toml"
+DFLT_DATA_DIR = (
+    Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local" / "share"))
+    / "ghtraffic"
+    / "data"
+)
+
+
+@dataclass(frozen=True)
+class Settings:
+    """What to collect, and how.
+
+    Attributes:
+        owners: user/org logins; ``"@me"`` means you and every org you belong to.
+        exclude: ``owner/repo`` fnmatch patterns to skip (``"acme/*"`` skips an owner).
+        include_forks: collect forks too.
+        include_archived: collect archived repos too.
+        snapshot_every_days: days between snapshots of the 14-day aggregates
+            (referrers, paths, window totals). 14 (the default) tiles the window:
+            snapshots never overlap, so their counts can be added up.
+        backfill_stars: on first sight of a repo, rebuild its star history from
+            stargazer dates (one GraphQL call per 100 stars).
+        max_workers: concurrent requests while fetching traffic.
+    """
+
+    owners: tuple[str, ...] = ("@me",)
+    exclude: tuple[str, ...] = ()
+    include_forks: bool = True
+    include_archived: bool = True
+    snapshot_every_days: int = 14
+    backfill_stars: bool = True
+    max_workers: int = 8
+
+
+def _from_mapping(d: dict) -> Settings:
+    d = dict(d.get("ghtraffic", d))
+    known = [f.name for f in fields(Settings)]
+    unknown = sorted(set(d) - set(known))
+    if unknown:
+        raise ValueError(
+            f"Unknown ghtraffic setting(s) {unknown}; valid keys are {known}."
+        )
+    for k in ("owners", "exclude"):
+        if isinstance(d.get(k), str):
+            d[k] = (d[k],)
+        elif k in d:
+            d[k] = tuple(d[k])
+    return Settings(**d)
+
+
+def load_settings(path: str | os.PathLike | None = None, **overrides) -> Settings:
+    """Settings from a TOML file (missing file: defaults), then ``overrides``."""
+    s = Settings()
+    if path is not None and Path(path).exists():
+        s = _from_mapping(tomllib.loads(Path(path).read_text(encoding="utf-8")))
+    overrides = {k: v for k, v in overrides.items() if v is not None}
+    base = {f.name: getattr(s, f.name) for f in fields(Settings)}
+    return _from_mapping({**base, **overrides})