github-actions-ingester 0.1.0__py3-none-any.whl

github_actions_ingester/collector.py

@@ -0,0 +1,338 @@
+"""Ingestion cycle: GitHub → PostgreSQL, incrementally.
+
+One cycle does, in order:
+
+  1. **Inventory** (every ``repo_refresh_seconds``): list the repositories
+     in scope, then the workflow files of each, upsert both. Optionally
+     read each workflow YAML from the default branch to record its cron
+     schedules.
+  2. **Runs**, per repository: list runs created since
+     ``cursor - lookback`` (``now - backfill_days`` on the first cycle),
+     upsert. Runs whose status/updated_at changed are flagged for a jobs
+     refresh by the store itself.
+  3. **Jobs**: fetch the jobs of every run flagged above and upsert them;
+     the run's ``completed_at`` is derived from the last job to finish.
+  4. **Stale open runs**: runs still open but older than the lookback
+     window (long queues, multi-hour jobs) are refreshed one by one,
+     bounded by ``max_open_run_refresh``.
+  5. **Gauges**: table counts and scheduled-workflow liveness.
+
+Failures in one repository are logged and counted; the cycle carries on
+with the next one. Only a rate-limit exhaustion aborts the whole cycle.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Callable, Iterable
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+import structlog
+
+from .config import Settings
+from .github import (
+    GitHubAPIError,
+    GitHubClient,
+    GitHubRateLimitError,
+    Repository,
+    WorkflowRun,
+)
+from .metrics import Metrics
+from .store import Store
+from .workflow_schedule import expected_interval_seconds, parse_schedules
+
+logger = structlog.get_logger(__name__)
+
+CONCLUSIONS = (
+    "success",
+    "failure",
+    "cancelled",
+    "skipped",
+    "timed_out",
+    "action_required",
+    "neutral",
+    "stale",
+    "startup_failure",
+)
+_RUN_BATCH = 500
+
+
+def _batched(items: Iterable[WorkflowRun], size: int) -> Iterable[list[WorkflowRun]]:
+    batch: list[WorkflowRun] = []
+    for item in items:
+        batch.append(item)
+        if len(batch) >= size:
+            yield batch
+            batch = []
+    if batch:
+        yield batch
+
+
+class Collector:
+    def __init__(
+        self,
+        client: GitHubClient,
+        store: Store,
+        metrics: Metrics,
+        settings: Settings,
+        clock: Callable[[], datetime] | None = None,
+    ) -> None:
+        self._client = client
+        self._store = store
+        self._metrics = metrics
+        self._settings = settings
+        self._clock = clock or (lambda: datetime.now(UTC))
+        self._repos: list[Repository] = []
+        self._inventory_at: datetime | None = None
+        self.cycles = 0
+
+    # -- inventory ----------------------------------------------------------
+
+    def _discover_repositories(self) -> list[Repository]:
+        s = self._settings
+        found: dict[int, Repository] = {}
+        orgs = {o.lower() for o in s.org_list()}
+        if orgs:
+            if s.uses_app():
+                for repo in self._client.list_installation_repositories():
+                    if repo.owner.lower() in orgs:
+                        found[repo.id] = repo
+            else:
+                for org in s.org_list():
+                    for repo in self._client.list_org_repositories(org):
+                        found[repo.id] = repo
+        for full_name in s.repo_list():
+            if any(r.full_name.lower() == full_name.lower() for r in found.values()):
+                continue
+            try:
+                repo = self._client.get_repository(full_name)
+            except GitHubAPIError as exc:
+                logger.warning("inventory.repo_unreachable", repo=full_name, error=str(exc))
+                self._metrics.errors_total.labels(stage="inventory").inc()
+                continue
+            found[repo.id] = repo
+        repos = []
+        for repo in found.values():
+            if s.is_excluded(repo.full_name):
+                continue
+            if repo.archived and not s.include_archived:
+                continue
+            repos.append(repo)
+        repos.sort(key=lambda r: r.full_name.lower())
+        return repos
+
+    def refresh_inventory(self) -> None:
+        repos = self._discover_repositories()
+        self._store.upsert_repositories(repos)
+        workflows_total = 0
+        for repo in repos:
+            try:
+                workflows = list(self._client.list_workflows(repo))
+            except GitHubAPIError as exc:
+                logger.warning("inventory.workflows_failed", repo=repo.full_name, error=str(exc))
+                self._metrics.errors_total.labels(stage="inventory").inc()
+                continue
+            workflows_total += self._store.upsert_workflows(workflows)
+        self._repos = repos
+        self._inventory_at = self._clock()
+        self._metrics.repositories.set(len(repos))
+        self._metrics.workflows.set(workflows_total)
+        logger.info("inventory.refreshed", repositories=len(repos), workflows=workflows_total)
+        if self._settings.sync_schedules:
+            self.sync_schedules()
+
+    def sync_schedules(self) -> None:
+        older_than = self._clock() - timedelta(seconds=self._settings.schedule_refresh_seconds)
+        rows = self._store.workflows_needing_schedule_sync(older_than)
+        synced = 0
+        for row in rows:
+            repo = Repository(
+                id=int(row["repository_id"]),
+                owner="",
+                name="",
+                full_name=str(row["full_name"]),
+                default_branch=str(row["default_branch"]),
+                private=False,
+                archived=False,
+                html_url="",
+            )
+            path = str(row["path"])
+            # Dynamic workflows (e.g. "dynamic/pages/pages-build-deployment")
+            # have no file in the tree; record an empty schedule.
+            crons: list[str] = []
+            if path.startswith(".github/workflows/"):
+                try:
+                    text = self._client.get_file_text(repo, path, repo.default_branch)
+                except GitHubAPIError as exc:
+                    logger.warning(
+                        "schedules.read_failed", repo=repo.full_name, path=path, error=str(exc)
+                    )
+                    self._metrics.errors_total.labels(stage="schedules").inc()
+                    continue
+                crons = parse_schedules(text) if text else []
+            interval = expected_interval_seconds(crons) if crons else None
+            self._store.set_workflow_schedules(int(row["id"]), crons, interval)
+            synced += 1
+        if rows:
+            logger.info("schedules.synced", workflows=synced, of=len(rows))
+
+    def _inventory_stale(self) -> bool:
+        if self._inventory_at is None:
+            return True
+        age = (self._clock() - self._inventory_at).total_seconds()
+        return age >= self._settings.repo_refresh_seconds
+
+    # -- runs / jobs -------------------------------------------------------------
+
+    def ingest_repository(self, repo: Repository) -> int:
+        s = self._settings
+        now = self._clock()
+        cursor = self._store.get_cursor(repo.id)
+        if cursor is None:
+            since = now - timedelta(days=s.backfill_days)
+            logger.info("runs.backfill", repo=repo.full_name, since=since.isoformat())
+        else:
+            since = cursor - timedelta(minutes=s.lookback_minutes)
+
+        written = 0
+        for batch in _batched(self._client.list_runs(repo, since, now), _RUN_BATCH):
+            written += self._store.upsert_runs(batch)
+        if written:
+            self._metrics.runs_upserted_total.labels(repository=repo.full_name).inc(written)
+
+        # Open runs that fell out of the window: refresh them individually.
+        stale = self._store.open_runs_before(repo.id, since, s.max_open_run_refresh)
+        for run_id in stale:
+            try:
+                run = self._client.get_run(repo.full_name, run_id)
+            except GitHubAPIError as exc:
+                if exc.status == 404:
+                    continue
+                raise
+            self._store.upsert_runs([run])
+
+        jobs_written = 0
+        for run_id in self._store.runs_needing_jobs(repo.id):
+            try:
+                jobs = list(self._client.list_jobs(repo, run_id, s.jobs_filter))
+            except GitHubAPIError as exc:
+                if exc.status == 404:
+                    # Run deleted between listing and jobs fetch; mark as synced.
+                    self._store.upsert_jobs(run_id, [])
+                    continue
+                raise
+            jobs_written += self._store.upsert_jobs(run_id, jobs)
+        if jobs_written:
+            self._metrics.jobs_upserted_total.labels(repository=repo.full_name).inc(jobs_written)
+
+        self._store.set_cursor(repo.id, now, written)
+        logger.info(
+            "runs.ingested",
+            repo=repo.full_name,
+            runs=written,
+            jobs=jobs_written,
+            refreshed_open=len(stale),
+            since=since.isoformat(),
+        )
+        return written
+
+    # -- gauges ------------------------------------------------------------------
+
+    def update_gauges(self) -> None:
+        m = self._metrics
+        counts = self._store.counts()
+        m.stored_runs.set(counts.get("runs", 0))
+        m.stored_jobs.set(counts.get("jobs", 0))
+        m.open_runs.set(counts.get("open_runs", 0))
+        m.repositories.set(counts.get("repositories", 0))
+        m.workflows.set(counts.get("workflows", 0))
+
+        rows = self._store.scheduled_workflow_status()
+        m.scheduled_last_run_timestamp_seconds.clear()
+        m.scheduled_interval_seconds.clear()
+        m.scheduled_last_conclusion.clear()
+        m.scheduled_workflows.set(len(rows))
+        for row in rows:
+            labels: dict[str, Any] = {
+                "repository": row["repository"],
+                "workflow": row["path"],
+                "workflow_name": row["name"],
+            }
+            last = row.get("last_scheduled_run_at")
+            m.scheduled_last_run_timestamp_seconds.labels(**labels).set(
+                last.timestamp() if isinstance(last, datetime) else 0
+            )
+            interval = row.get("interval_seconds")
+            m.scheduled_interval_seconds.labels(**labels).set(
+                float(interval) if interval is not None else 0
+            )
+            last_conclusion = row.get("last_conclusion")
+            for c in CONCLUSIONS:
+                m.scheduled_last_conclusion.labels(**labels, conclusion=c).set(
+                    1 if c == last_conclusion else 0
+                )
+
+    def _update_rate_limit_gauges(self) -> None:
+        rl = self._client.rate_limit
+        self._metrics.github_rate_limit_remaining.set(rl.remaining)
+        self._metrics.github_rate_limit_limit.set(rl.limit)
+        self._metrics.github_rate_limit_reset_timestamp_seconds.set(rl.reset_at)
+
+    # -- cycle ---------------------------------------------------------------------
+
+    def run_cycle(self) -> str:
+        """Run one full cycle; returns ``ok``, ``partial`` or ``error``."""
+        m = self._metrics
+        started = time.monotonic()
+        result = "ok"
+        try:
+            if self._inventory_stale():
+                self.refresh_inventory()
+            failures = 0
+            for repo in self._repos:
+                t0 = time.monotonic()
+                try:
+                    self.ingest_repository(repo)
+                except GitHubRateLimitError:
+                    raise
+                except Exception as exc:
+                    failures += 1
+                    m.errors_total.labels(stage="repository").inc()
+                    logger.error("runs.repository_failed", repo=repo.full_name, error=str(exc))
+                finally:
+                    m.repository_cycle_duration_seconds.observe(time.monotonic() - t0)
+                self._update_rate_limit_gauges()
+            self.update_gauges()
+            if failures:
+                result = "partial"
+        except GitHubRateLimitError as exc:
+            logger.error("cycle.rate_limited", error=str(exc))
+            m.errors_total.labels(stage="rate_limit").inc()
+            result = "error"
+        except Exception as exc:
+            logger.exception("cycle.failed", error=str(exc))
+            m.errors_total.labels(stage="cycle").inc()
+            result = "error"
+        finally:
+            self._update_rate_limit_gauges()
+        elapsed = time.monotonic() - started
+        now_ts = time.time()
+        m.cycle_duration_seconds.observe(elapsed)
+        m.cycles_total.labels(result=result).inc()
+        m.last_cycle_timestamp_seconds.set(now_ts)
+        if result != "error":
+            m.last_success_timestamp_seconds.set(now_ts)
+            m.up.set(1)
+            m.ready.set(1)
+        else:
+            m.up.set(0)
+        self.cycles += 1
+        logger.info("cycle.done", result=result, seconds=round(elapsed, 2), cycle=self.cycles)
+        return result
+
+    def run_forever(self, stop: threading.Event) -> None:
+        while not stop.is_set():
+            self.run_cycle()
+            stop.wait(self._settings.poll_interval_seconds)

github_actions_ingester/config.py

@@ -0,0 +1,290 @@
+"""Configuration — env-var driven via pydantic-settings.
+
+Env vars are prefixed ``GHA_`` so ``GHA_GITHUB_TOKEN`` / ``GHA_DATABASE_URL``
+/ ``GHA_POLL_INTERVAL_SECONDS`` etc. are picked up automatically.
+
+Two credential shapes are accepted, pick ONE:
+
+  - a classic / fine-grained personal access token (``GHA_GITHUB_TOKEN``)
+  - a GitHub App (``GHA_GITHUB_APP_ID`` + ``GHA_GITHUB_APP_PRIVATE_KEY`` or
+    ``..._PRIVATE_KEY_FILE``, optionally ``GHA_GITHUB_APP_INSTALLATION_ID``)
+
+The App is the recommended shape for an organization: least privilege
+(``Actions: read`` + ``Metadata: read`` + optional ``Contents: read`` for
+schedule discovery), no human account in the loop, and installation
+tokens rotate hourly on their own.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import re
+from pathlib import Path
+from typing import Any
+
+from pydantic import Field, field_validator, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+_REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
+_OWNER_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
+
+
+def _split_csv(value: str) -> list[str]:
+    return [item.strip() for item in value.split(",") if item.strip()]
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_prefix="GHA_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+    # --- GitHub auth (pick one) ---
+    github_token: str = Field(
+        default="",
+        description="Personal access token (classic or fine-grained) with "
+        "`actions:read` on the target repositories. Mutually exclusive with "
+        "the GitHub App settings.",
+    )
+    github_app_id: str = Field(
+        default="",
+        description="GitHub App ID (or client ID). Requires "
+        "github_app_private_key / github_app_private_key_file.",
+    )
+    github_app_private_key: str = Field(
+        default="",
+        description="PEM-encoded private key of the GitHub App. Newlines may "
+        "be literal or escaped as `\\n` (both are accepted).",
+    )
+    github_app_private_key_file: str = Field(
+        default="",
+        description="Path to the PEM private key. Alternative to the inline "
+        "value — pair with a Secret volume mount in Kubernetes.",
+    )
+    github_app_installation_id: str = Field(
+        default="",
+        description="Installation ID of the App. When empty the ingester "
+        "lists /app/installations and picks the one matching the first "
+        "entry in `orgs` (or the only installation, if there is one).",
+    )
+    github_api_base: str = Field(
+        default="https://api.github.com",
+        description="Override the API base URL — set to "
+        "https://ghe.example.com/api/v3 for GitHub Enterprise Server.",
+    )
+
+    # --- Scope ---
+    orgs: str = Field(
+        default="",
+        description="Comma-separated organizations (or user accounts) whose "
+        "repositories are ingested. Every repository the credential can see "
+        "in each org is included unless excluded below.",
+    )
+    repos: str = Field(
+        default="",
+        description="Comma-separated explicit repositories as `owner/name`. "
+        "Combined with `orgs` (union).",
+    )
+    exclude_repos: str = Field(
+        default="",
+        description="Comma-separated glob patterns matched against "
+        "`owner/name` (fnmatch): e.g. `acme/legacy-*,acme/sandbox`.",
+    )
+    include_archived: bool = Field(
+        default=False,
+        description="Also ingest archived repositories. They rarely run "
+        "workflows, so skipping them saves API budget.",
+    )
+
+    # --- Database ---
+    database_url: str = Field(
+        ...,
+        description="PostgreSQL connection URL (libpq form): "
+        "postgresql://user:pass@host:5432/dbname?sslmode=require. The role "
+        "needs CREATE on the database the first time (schema bootstrap) "
+        "and plain read/write afterwards.",
+    )
+    database_schema: str = Field(
+        default="gha",
+        description="Schema that holds every ingester table. Created on first start if missing.",
+    )
+    database_connect_timeout_seconds: int = Field(default=10, ge=1)
+
+    # --- Collection ---
+    poll_interval_seconds: int = Field(
+        default=300,
+        ge=30,
+        description="Seconds between ingestion cycles. Each cycle costs "
+        "roughly 1 request per repository (runs listing) + 1 per run that "
+        "changed since the previous cycle (jobs).",
+    )
+    backfill_days: int = Field(
+        default=30,
+        ge=1,
+        le=3660,
+        description="How far back the FIRST cycle for a repository goes. "
+        "Later cycles are incremental. Raising it later only affects "
+        "repositories that have no cursor yet.",
+    )
+    lookback_minutes: int = Field(
+        default=180,
+        ge=1,
+        description="Every cycle re-lists runs created within this window "
+        "before the cursor, so runs that changed status (queued → in "
+        "progress → completed) are refreshed even if a cycle was missed.",
+    )
+    repo_refresh_seconds: int = Field(
+        default=3600,
+        ge=60,
+        description="How often the repository + workflow inventory is "
+        "re-listed. New repositories / workflow files show up after at "
+        "most this delay.",
+    )
+    max_open_run_refresh: int = Field(
+        default=200,
+        ge=0,
+        description="Upper bound on individual `GET /runs/{id}` refreshes "
+        "per cycle for runs still open outside the lookback window (long "
+        "queues, multi-hour jobs). 0 disables.",
+    )
+    jobs_filter: str = Field(
+        default="all",
+        description="`all` ingests jobs from every run attempt (matches "
+        "GitHub billing); `latest` keeps only the last attempt.",
+    )
+    sync_schedules: bool = Field(
+        default=True,
+        description="Read each workflow file from the default branch and "
+        "record its `on.schedule` cron expressions (needs `Contents: read`). "
+        "Powers the scheduled-workflow liveness metrics.",
+    )
+    schedule_refresh_seconds: int = Field(default=21600, ge=300)
+
+    # --- API pacing ---
+    api_rate_limit_rps: float = Field(
+        default=5.0,
+        gt=0.0,
+        description="Max requests per second to the GitHub API. The "
+        "documented secondary limit is ~900 points/min for REST; 5 rps is "
+        "comfortably below it.",
+    )
+    api_min_remaining: int = Field(
+        default=200,
+        ge=0,
+        description="When the primary rate-limit `remaining` drops under "
+        "this value the ingester pauses until the limit resets instead of "
+        "burning the last requests other tools may need.",
+    )
+    api_timeout_seconds: float = Field(default=30.0, gt=0.0)
+    api_max_retries: int = Field(default=4, ge=0)
+
+    # --- Server ---
+    listen_host: str = Field(default="0.0.0.0", description="Bind address for /metrics.")
+    listen_port: int = Field(default=9619, ge=1, le=65535)
+
+    # --- Logging ---
+    log_level: str = Field(default="info")
+    log_format: str = Field(default="json", description="json or console")
+
+    # ------------------------------------------------------------------
+    # Validation
+    # ------------------------------------------------------------------
+
+    @field_validator("log_level")
+    @classmethod
+    def _normalize_level(cls, v: str) -> str:
+        v = v.lower()
+        if v not in {"debug", "info", "warning", "warn", "error"}:
+            raise ValueError(f"log_level must be debug/info/warning/error (got {v!r})")
+        return "warning" if v == "warn" else v
+
+    @field_validator("log_format")
+    @classmethod
+    def _normalize_format(cls, v: str) -> str:
+        v = v.lower()
+        if v not in {"json", "console"}:
+            raise ValueError(f"log_format must be json or console (got {v!r})")
+        return v
+
+    @field_validator("jobs_filter")
+    @classmethod
+    def _jobs_filter(cls, v: str) -> str:
+        v = v.lower()
+        if v not in {"all", "latest"}:
+            raise ValueError(f"jobs_filter must be all or latest (got {v!r})")
+        return v
+
+    @field_validator("database_schema")
+    @classmethod
+    def _schema_ident(cls, v: str) -> str:
+        if not re.fullmatch(r"[a-z_][a-z0-9_]{0,62}", v):
+            raise ValueError(
+                "database_schema must be a plain lowercase identifier (letters, digits, _)"
+            )
+        return v
+
+    @field_validator("github_api_base")
+    @classmethod
+    def _strip_slash(cls, v: str) -> str:
+        return v.rstrip("/")
+
+    @model_validator(mode="after")
+    def _check_auth_and_scope(self) -> Settings:
+        has_token = bool(self.github_token)
+        has_app = bool(self.github_app_id)
+        if has_token and has_app:
+            raise ValueError("set either GHA_GITHUB_TOKEN or GHA_GITHUB_APP_ID, not both")
+        if not has_token and not has_app:
+            raise ValueError("no GitHub credential: set GHA_GITHUB_TOKEN or GHA_GITHUB_APP_ID")
+        if has_app and not (self.github_app_private_key or self.github_app_private_key_file):
+            raise ValueError(
+                "GHA_GITHUB_APP_ID needs GHA_GITHUB_APP_PRIVATE_KEY or "
+                "GHA_GITHUB_APP_PRIVATE_KEY_FILE"
+            )
+        for org in self.org_list():
+            if not _OWNER_RE.match(org):
+                raise ValueError(f"orgs entry {org!r} is not a valid GitHub login")
+        for repo in self.repo_list():
+            if not _REPO_RE.match(repo):
+                raise ValueError(f"repos entry {repo!r} must look like owner/name")
+        if not self.org_list() and not self.repo_list():
+            raise ValueError("nothing to ingest: set GHA_ORGS and/or GHA_REPOS")
+        return self
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def org_list(self) -> list[str]:
+        return _split_csv(self.orgs)
+
+    def repo_list(self) -> list[str]:
+        return _split_csv(self.repos)
+
+    def exclude_patterns(self) -> list[str]:
+        return _split_csv(self.exclude_repos)
+
+    def is_excluded(self, full_name: str) -> bool:
+        name = full_name.lower()
+        return any(fnmatch.fnmatchcase(name, pat.lower()) for pat in self.exclude_patterns())
+
+    def app_private_key_pem(self) -> str:
+        """Return the PEM text, reading the file when configured.
+
+        Inline values often arrive with ``\\n`` escapes (Helm ``--set``,
+        some secret managers); normalize them back to real newlines.
+        """
+        if self.github_app_private_key_file:
+            return Path(self.github_app_private_key_file).read_text(encoding="utf-8")
+        return self.github_app_private_key.replace("\\n", "\n")
+
+    def uses_app(self) -> bool:
+        return bool(self.github_app_id)
+
+
+def load_settings(**overrides: Any) -> Settings:
+    """Load settings, applying optional overrides on top of env."""
+    return Settings(**overrides)