github-actions-ingester 0.1.0__py3-none-any.whl

github_actions_ingester/metrics.py

@@ -0,0 +1,184 @@
+"""Prometheus metric definitions.
+
+Everything is registered on an explicit ``CollectorRegistry`` (plus the
+standard process/platform/GC collectors) so ``/metrics`` renders exactly
+what this ingester publishes.
+
+Two families:
+
+  ``gha_ingester_*``      — introspection: cycle timing, API budget, rows
+                            written, errors. Alert on these to know the
+                            ingester itself is healthy.
+  ``gha_scheduled_workflow_*`` — one series per workflow with a cron
+                            ``on.schedule``: when it last ran and how often
+                            it should. Alert on these to catch a scheduled
+                            workflow that silently stopped firing.
+
+The analytical data (minutes, success rate, queue time) lives in
+PostgreSQL, not here: Grafana reads it with the SQL datasource.
+"""
+
+from __future__ import annotations
+
+from prometheus_client import (
+    GC_COLLECTOR,
+    PLATFORM_COLLECTOR,
+    PROCESS_COLLECTOR,
+    CollectorRegistry,
+    Counter,
+    Gauge,
+    Histogram,
+    Info,
+)
+
+NAMESPACE = "gha_ingester"
+
+WORKFLOW_LABELS = ("repository", "workflow", "workflow_name")
+
+
+class Metrics:
+    def __init__(self, registry: CollectorRegistry | None = None) -> None:
+        self.registry = registry or CollectorRegistry(auto_describe=True)
+        self.registry.register(PROCESS_COLLECTOR)
+        self.registry.register(PLATFORM_COLLECTOR)
+        self.registry.register(GC_COLLECTOR)
+
+        # ---- Liveness of the ingester itself ----
+        self.build_info = Info(
+            f"{NAMESPACE}_build",
+            "Build information of the running ingester.",
+            registry=self.registry,
+        )
+        self.up = Gauge(
+            f"{NAMESPACE}_up",
+            "1 when the last ingestion cycle finished without a fatal error, else 0.",
+            registry=self.registry,
+        )
+        self.ready = Gauge(
+            f"{NAMESPACE}_ready",
+            "1 once the database schema is bootstrapped and the first cycle completed.",
+            registry=self.registry,
+        )
+        self.cycles_total = Counter(
+            f"{NAMESPACE}_cycles_total",
+            "Ingestion cycles, by outcome.",
+            ("result",),
+            registry=self.registry,
+        )
+        self.cycle_duration_seconds = Histogram(
+            f"{NAMESPACE}_cycle_duration_seconds",
+            "Wall-clock duration of an ingestion cycle.",
+            registry=self.registry,
+            buckets=(1, 5, 15, 30, 60, 120, 300, 600, 1800),
+        )
+        self.last_cycle_timestamp_seconds = Gauge(
+            f"{NAMESPACE}_last_cycle_timestamp_seconds",
+            "Unix time of the end of the last cycle (any outcome).",
+            registry=self.registry,
+        )
+        self.last_success_timestamp_seconds = Gauge(
+            f"{NAMESPACE}_last_success_timestamp_seconds",
+            "Unix time of the end of the last SUCCESSFUL cycle.",
+            registry=self.registry,
+        )
+        self.errors_total = Counter(
+            f"{NAMESPACE}_errors_total",
+            "Errors during ingestion, by stage.",
+            ("stage",),
+            registry=self.registry,
+        )
+
+        # ---- GitHub API budget ----
+        self.github_requests_total = Counter(
+            f"{NAMESPACE}_github_requests_total",
+            "Requests sent to the GitHub REST API, by HTTP status.",
+            ("status",),
+            registry=self.registry,
+        )
+        self.github_rate_limit_remaining = Gauge(
+            f"{NAMESPACE}_github_rate_limit_remaining",
+            "Requests left in the current primary rate-limit window "
+            "(X-RateLimit-Remaining of the last response).",
+            registry=self.registry,
+        )
+        self.github_rate_limit_limit = Gauge(
+            f"{NAMESPACE}_github_rate_limit_limit",
+            "Size of the primary rate-limit window (X-RateLimit-Limit).",
+            registry=self.registry,
+        )
+        self.github_rate_limit_reset_timestamp_seconds = Gauge(
+            f"{NAMESPACE}_github_rate_limit_reset_timestamp_seconds",
+            "Unix time when the primary rate-limit window resets.",
+            registry=self.registry,
+        )
+
+        # ---- What was ingested ----
+        self.repositories = Gauge(
+            f"{NAMESPACE}_repositories",
+            "Repositories currently in scope (after exclusions).",
+            registry=self.registry,
+        )
+        self.workflows = Gauge(
+            f"{NAMESPACE}_workflows",
+            "Workflow files known across all repositories.",
+            registry=self.registry,
+        )
+        self.runs_upserted_total = Counter(
+            f"{NAMESPACE}_runs_upserted_total",
+            "Workflow runs written (inserted or updated), by repository.",
+            ("repository",),
+            registry=self.registry,
+        )
+        self.jobs_upserted_total = Counter(
+            f"{NAMESPACE}_jobs_upserted_total",
+            "Workflow jobs written (inserted or updated), by repository.",
+            ("repository",),
+            registry=self.registry,
+        )
+        self.open_runs = Gauge(
+            f"{NAMESPACE}_open_runs",
+            "Runs stored that are not yet completed (queued / in progress).",
+            registry=self.registry,
+        )
+        self.stored_runs = Gauge(
+            f"{NAMESPACE}_stored_runs",
+            "Total workflow runs stored in the database.",
+            registry=self.registry,
+        )
+        self.stored_jobs = Gauge(
+            f"{NAMESPACE}_stored_jobs",
+            "Total workflow jobs stored in the database.",
+            registry=self.registry,
+        )
+        self.repository_cycle_duration_seconds = Histogram(
+            f"{NAMESPACE}_repository_cycle_duration_seconds",
+            "Time spent ingesting one repository within a cycle.",
+            registry=self.registry,
+            buckets=(0.5, 1, 2, 5, 10, 30, 60, 300),
+        )
+
+        # ---- Scheduled workflow liveness ----
+        self.scheduled_last_run_timestamp_seconds = Gauge(
+            "gha_scheduled_workflow_last_run_timestamp_seconds",
+            "Unix time of the most recent run triggered by `schedule` for the workflow.",
+            WORKFLOW_LABELS,
+            registry=self.registry,
+        )
+        self.scheduled_interval_seconds = Gauge(
+            "gha_scheduled_workflow_interval_seconds",
+            "Longest gap between two consecutive cron fires of the workflow (all schedules merged).",
+            WORKFLOW_LABELS,
+            registry=self.registry,
+        )
+        self.scheduled_last_conclusion = Gauge(
+            "gha_scheduled_workflow_last_conclusion",
+            "1 for the conclusion of the last completed scheduled run "
+            "(success/failure/cancelled/...), 0 for the others.",
+            (*WORKFLOW_LABELS, "conclusion"),
+            registry=self.registry,
+        )
+        self.scheduled_workflows = Gauge(
+            "gha_scheduled_workflows",
+            "Active workflows that declare at least one cron schedule.",
+            registry=self.registry,
+        )

github_actions_ingester/migrations/0001_initial.sql

@@ -0,0 +1,140 @@
+-- 0001_initial: core tables, indexes and the compatibility views.
+--
+-- Every statement runs inside the schema selected by GHA_DATABASE_SCHEMA
+-- (search_path is set by the migration runner), so nothing here is
+-- schema-qualified.
+
+CREATE TABLE IF NOT EXISTS repositories (
+    id              BIGINT PRIMARY KEY,
+    owner           TEXT NOT NULL,
+    name            TEXT NOT NULL,
+    full_name       TEXT NOT NULL UNIQUE,
+    default_branch  TEXT NOT NULL DEFAULT 'main',
+    private         BOOLEAN NOT NULL DEFAULT FALSE,
+    archived        BOOLEAN NOT NULL DEFAULT FALSE,
+    html_url        TEXT NOT NULL DEFAULT '',
+    first_seen_at   TIMESTAMPTZ NOT NULL DEFAULT now(),
+    last_seen_at    TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE TABLE IF NOT EXISTS workflows (
+    id              BIGINT PRIMARY KEY,
+    repository_id   BIGINT NOT NULL REFERENCES repositories (id) ON DELETE CASCADE,
+    name            TEXT NOT NULL,
+    path            TEXT NOT NULL,
+    state           TEXT NOT NULL DEFAULT '',
+    html_url        TEXT NOT NULL DEFAULT '',
+    schedules       TEXT[] NOT NULL DEFAULT '{}',
+    -- Longest gap between two consecutive scheduled firings (all crons merged).
+    schedule_interval_seconds DOUBLE PRECISION,
+    schedules_synced_at TIMESTAMPTZ,
+    created_at      TIMESTAMPTZ,
+    updated_at      TIMESTAMPTZ,
+    first_seen_at   TIMESTAMPTZ NOT NULL DEFAULT now(),
+    last_seen_at    TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+CREATE INDEX IF NOT EXISTS workflows_repository_idx ON workflows (repository_id);
+
+CREATE TABLE IF NOT EXISTS workflow_runs (
+    id              BIGINT PRIMARY KEY,
+    repository_id   BIGINT NOT NULL REFERENCES repositories (id) ON DELETE CASCADE,
+    workflow_id     BIGINT NOT NULL,
+    run_number      INTEGER NOT NULL DEFAULT 0,
+    run_attempt     INTEGER NOT NULL DEFAULT 1,
+    name            TEXT NOT NULL DEFAULT '',
+    display_title   TEXT NOT NULL DEFAULT '',
+    event           TEXT NOT NULL DEFAULT '',
+    status          TEXT NOT NULL DEFAULT '',
+    conclusion      TEXT,
+    head_branch     TEXT,
+    head_sha        TEXT NOT NULL DEFAULT '',
+    actor           TEXT NOT NULL DEFAULT '',
+    triggering_actor TEXT NOT NULL DEFAULT '',
+    created_at      TIMESTAMPTZ NOT NULL,
+    updated_at      TIMESTAMPTZ,
+    run_started_at  TIMESTAMPTZ,
+    -- Derived on write: MAX(jobs.completed_at) once every job finished, or
+    -- updated_at when the run is completed and no job reported a time.
+    completed_at    TIMESTAMPTZ,
+    html_url        TEXT NOT NULL DEFAULT '',
+    jobs_synced_at  TIMESTAMPTZ,
+    ingested_at     TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+CREATE INDEX IF NOT EXISTS workflow_runs_repo_created_idx
+    ON workflow_runs (repository_id, created_at DESC);
+CREATE INDEX IF NOT EXISTS workflow_runs_workflow_created_idx
+    ON workflow_runs (workflow_id, created_at DESC);
+CREATE INDEX IF NOT EXISTS workflow_runs_created_idx ON workflow_runs (created_at DESC);
+CREATE INDEX IF NOT EXISTS workflow_runs_open_idx
+    ON workflow_runs (repository_id, created_at) WHERE status <> 'completed';
+
+CREATE TABLE IF NOT EXISTS workflow_jobs (
+    id              BIGINT PRIMARY KEY,
+    run_id          BIGINT NOT NULL REFERENCES workflow_runs (id) ON DELETE CASCADE,
+    repository_id   BIGINT NOT NULL,
+    run_attempt     INTEGER NOT NULL DEFAULT 1,
+    name            TEXT NOT NULL DEFAULT '',
+    status          TEXT NOT NULL DEFAULT '',
+    conclusion      TEXT,
+    runner_name     TEXT,
+    runner_group_name TEXT,
+    labels          TEXT[] NOT NULL DEFAULT '{}',
+    created_at      TIMESTAMPTZ,
+    started_at      TIMESTAMPTZ,
+    completed_at    TIMESTAMPTZ,
+    steps           INTEGER NOT NULL DEFAULT 0,
+    html_url        TEXT NOT NULL DEFAULT '',
+    ingested_at     TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+CREATE INDEX IF NOT EXISTS workflow_jobs_run_idx ON workflow_jobs (run_id);
+CREATE INDEX IF NOT EXISTS workflow_jobs_repo_started_idx
+    ON workflow_jobs (repository_id, started_at DESC);
+
+-- One row per repository: where the incremental listing resumes from.
+CREATE TABLE IF NOT EXISTS ingest_cursors (
+    repository_id   BIGINT PRIMARY KEY REFERENCES repositories (id) ON DELETE CASCADE,
+    runs_created_since TIMESTAMPTZ NOT NULL,
+    last_cycle_at   TIMESTAMPTZ NOT NULL DEFAULT now(),
+    last_cycle_runs INTEGER NOT NULL DEFAULT 0
+);
+
+-- ---------------------------------------------------------------------------
+-- Compatibility views: the column set expected by Grafana dashboard 24157
+-- ("GitHub Actions insights"). Point its Postgres datasource at this schema
+-- and every panel works unchanged.
+-- ---------------------------------------------------------------------------
+
+CREATE OR REPLACE VIEW minion_repositories AS
+SELECT id, full_name, owner, name, archived
+FROM repositories;
+
+CREATE OR REPLACE VIEW minion_workflow_files AS
+SELECT id, repository_id, name, path
+FROM workflows;
+
+CREATE OR REPLACE VIEW minion_workflow_runs AS
+SELECT
+    r.id,
+    r.repository_id,
+    r.workflow_id AS workflow_file_id,
+    r.event,
+    r.status,
+    r.conclusion,
+    r.head_branch,
+    r.created_at,
+    COALESCE(r.run_started_at, r.created_at) AS started_at,
+    r.completed_at
+FROM workflow_runs r;
+
+CREATE OR REPLACE VIEW minion_workflow_jobs AS
+SELECT
+    j.id,
+    j.run_id,
+    j.name,
+    j.status,
+    j.conclusion,
+    j.runner_name,
+    j.created_at,
+    j.started_at,
+    j.completed_at
+FROM workflow_jobs j;

github_actions_ingester/ratelimit.py

@@ -0,0 +1,39 @@
+"""Client-side pacing for GitHub API calls.
+
+Leaky bucket: each ``acquire()`` blocks until at least ``1/rps`` seconds
+have passed since the previous acquire. GitHub enforces a primary limit
+(5000 req/h for Apps and PATs) and secondary limits on bursts; pacing at
+a few requests per second keeps the ingester well inside both while the
+primary-limit guard in the client handles the rest.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+
+
+class RateLimiter:
+    """Thread-safe leaky-bucket pacing helper."""
+
+    def __init__(self, rps: float) -> None:
+        if rps <= 0:
+            raise ValueError("rps must be > 0")
+        self._min_interval = 1.0 / rps
+        self._last = 0.0
+        self._lock = threading.Lock()
+
+    def acquire(self) -> None:
+        """Block until the next request slot is allowed.
+
+        The lock guards the slot reservation; the wait happens outside it
+        so concurrent acquirers queue up in order instead of serializing
+        on the lock for the whole sleep.
+        """
+        with self._lock:
+            now = time.monotonic()
+            scheduled_at = max(now, self._last + self._min_interval)
+            self._last = scheduled_at
+        wait = scheduled_at - time.monotonic()
+        if wait > 0:
+            time.sleep(wait)

github_actions_ingester/server.py

@@ -0,0 +1,115 @@
+"""HTTP server — Prometheus /metrics + health endpoints.
+
+stdlib ``http.server`` only: the ingester serves one scraper and a couple
+of probes, nothing that justifies an async stack.
+
+  /metrics   Prometheus exposition
+  /healthz   liveness — 200 while the HTTP server answers
+  /readyz    readiness — 200 once the schema is bootstrapped and the first
+             cycle finished; 503 before that so a rollout waits for the
+             database bootstrap instead of declaring victory early
+  /          index
+"""
+
+from __future__ import annotations
+
+import http.server
+import threading
+import urllib.parse
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+import structlog
+from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
+
+if TYPE_CHECKING:
+    from prometheus_client import CollectorRegistry
+
+logger = structlog.get_logger(__name__)
+
+_INDEX_HTML = """\
+<!doctype html>
+<html lang="en">
+<head><title>github-actions-ingester</title></head>
+<body>
+  <h1>github-actions-ingester</h1>
+  <p>Ingests GitHub Actions workflow runs and jobs into PostgreSQL and
+    exposes ingester health as Prometheus metrics.</p>
+  <ul>
+    <li><a href="/metrics">/metrics</a> — Prometheus exposition
+      (<code>gha_ingester_*</code>, <code>gha_scheduled_workflow_*</code>)</li>
+    <li><a href="/healthz">/healthz</a> — liveness</li>
+    <li><a href="/readyz">/readyz</a> — readiness (503 until the schema is
+      bootstrapped and the first cycle completed)</li>
+  </ul>
+  <p>The analytical data (minutes, success rate, queue wait) is in the
+    database; query it with the Grafana PostgreSQL datasource.</p>
+  <p><a href="https://github.com/danielgines/github-actions-ingester">github.com/danielgines/github-actions-ingester</a></p>
+</body>
+</html>
+"""
+
+
+def make_handler(
+    registry: CollectorRegistry, is_ready: Callable[[], bool]
+) -> type[http.server.BaseHTTPRequestHandler]:
+    class Handler(http.server.BaseHTTPRequestHandler):
+        def log_message(self, format: str, *args: object) -> None:
+            return
+
+        def _send(self, code: int, body: bytes, content_type: str) -> None:
+            self.send_response(code)
+            self.send_header("Content-Type", content_type)
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+
+        def do_GET(self) -> None:
+            path = urllib.parse.urlparse(self.path).path
+            if path == "/metrics":
+                self._send(200, generate_latest(registry), CONTENT_TYPE_LATEST)
+                return
+            if path == "/healthz":
+                self._send(200, b"ok\n", "text/plain; charset=utf-8")
+                return
+            if path == "/readyz":
+                if is_ready():
+                    self._send(200, b"ready\n", "text/plain; charset=utf-8")
+                else:
+                    self._send(503, b"not ready\n", "text/plain; charset=utf-8")
+                return
+            if path in ("/", "/index.html"):
+                self._send(200, _INDEX_HTML.encode(), "text/html; charset=utf-8")
+                return
+            self._send(404, b"not found\n", "text/plain; charset=utf-8")
+
+    return Handler
+
+
+class MetricsServer:
+    """Threaded HTTP server with a controllable lifetime."""
+
+    def __init__(
+        self, host: str, port: int, registry: CollectorRegistry, is_ready: Callable[[], bool]
+    ) -> None:
+        self._httpd = http.server.ThreadingHTTPServer(
+            (host, port), make_handler(registry, is_ready)
+        )
+        self._thread = threading.Thread(
+            target=self._httpd.serve_forever, name="github-actions-ingester-http", daemon=True
+        )
+        self._host = host
+        self._port = port
+
+    @property
+    def port(self) -> int:
+        return int(self._httpd.server_address[1])
+
+    def start(self) -> None:
+        self._thread.start()
+        logger.info("http.listening", host=self._host, port=self.port)
+
+    def stop(self) -> None:
+        self._httpd.shutdown()
+        self._httpd.server_close()
+        logger.info("http.stopped")