github-actions-ingester 0.2.0__tar.gz → 0.2.2__tar.gz

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/CHANGELOG.md

@@ -4,6 +4,33 @@ All notable changes to this project are documented here. The format follows
 [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project
 adheres to [Semantic Versioning](https://semver.org/).
 
+## [0.2.2] — 2026-09-05
+
+### Fixed
+- The GitHub rate-limit gauges were refreshed once per repository, so during
+  a long first backfill of a busy repository they could lag by twenty
+  minutes while the real budget ran out. They now follow every response,
+  as the README already said they did.
+- `runs_upserted_total` and `jobs_upserted_total` move as rows are written
+  instead of once when the repository finishes.
+
+### Added
+- `jobs.progress` log line every 250 runs during the jobs phase, which was
+  silent for the whole duration of a large backfill.
+
+## [0.2.1] — 2026-09-05
+
+### Added
+- Chart: `deploymentAnnotations`, for a Reloader annotation on the
+  Deployment.
+
+### Fixed
+- A read role with a name different from the schema could not resolve the
+  views by bare name, and the Grafana PostgreSQL datasource has no
+  search_path setting. After granting read access the ingester now sets
+  the database's default `search_path` to the schema (needs the ingester
+  role to own the database; skipped with a warning otherwise).
+
 ## [0.2.0] — 2026-09-05
 
 ### Added

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: github-actions-ingester
-Version: 0.2.0
+Version: 0.2.2
 Summary: Standalone GitHub Actions ingester — pulls workflows, runs and jobs from the GitHub API into PostgreSQL for Grafana dashboards, FinOps and alerting. Prometheus /metrics for ingester health and scheduled-workflow liveness.
 Project-URL: Homepage, https://github.com/danielgines/github-actions-ingester
 Project-URL: Repository, https://github.com/danielgines/github-actions-ingester
@@ -162,7 +162,11 @@ table and view after each migration run, plus a default privilege so
 tables added by later migrations are readable too. Point Grafana at the
 database with one of those roles and it never needs the ingester's
 credentials. The roles themselves are created by whoever manages the
-database (the ingester never runs `CREATE ROLE`).
+database (the ingester never runs `CREATE ROLE`). In the same step the
+ingester sets the database's default `search_path` to the schema, so a
+Grafana datasource (which cannot send one) resolves the views by bare
+name; that statement needs the ingester's role to own the database, and
+is skipped with a warning otherwise.
 
 ### Tables and views
 

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/README.md

@@ -99,7 +99,11 @@ table and view after each migration run, plus a default privilege so
 tables added by later migrations are readable too. Point Grafana at the
 database with one of those roles and it never needs the ingester's
 credentials. The roles themselves are created by whoever manages the
-database (the ingester never runs `CREATE ROLE`).
+database (the ingester never runs `CREATE ROLE`). In the same step the
+ingester sets the database's default `search_path` to the schema, so a
+Grafana datasource (which cannot send one) resolves the views by bare
+name; that statement needs the ingester's role to own the database, and
+is skipped with a warning otherwise.
 
 ### Tables and views
 

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "github-actions-ingester"
-version = "0.2.0"
+version = "0.2.2"
 description = "Standalone GitHub Actions ingester — pulls workflows, runs and jobs from the GitHub API into PostgreSQL for Grafana dashboards, FinOps and alerting. Prometheus /metrics for ingester health and scheduled-workflow liveness."
 readme = "README.md"
 requires-python = ">=3.11"

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/src/github_actions_ingester/__init__.py

@@ -6,4 +6,4 @@ its own, and exposes Prometheus metrics about the ingestion itself and
 about the liveness of scheduled workflows.
 """
 
-__version__ = "0.2.0"
+__version__ = "0.2.2"

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/src/github_actions_ingester/__main__.py

@@ -26,7 +26,7 @@ from . import __version__
 from .app_manifest import convert_code, manifest_form_url, manifest_json, write_private_key
 from .collector import Collector
 from .config import Settings, load_settings
-from .github import AppAuth, GitHubAPIError, GitHubClient, TokenAuth
+from .github import AppAuth, GitHubAPIError, GitHubClient, RateLimitState, TokenAuth
 from .metrics import Metrics
 from .ratelimit import RateLimiter
 from .server import MetricsServer
@@ -77,6 +77,16 @@ def _build_client(settings: Settings, metrics: Metrics | None = None) -> GitHubC
         if metrics is not None:
             metrics.github_requests_total.labels(status=str(status)).inc()
 
+    # Refresh the budget gauges on every response, not once per repository:
+    # a busy repository keeps the collector inside one ingest_repository()
+    # call for many minutes, and an alert on the remaining budget must not
+    # wait for it to finish.
+    def on_rate_limit(rl: RateLimitState) -> None:
+        if metrics is not None:
+            metrics.github_rate_limit_remaining.set(rl.remaining)
+            metrics.github_rate_limit_limit.set(rl.limit)
+            metrics.github_rate_limit_reset_timestamp_seconds.set(rl.reset_at)
+
     return GitHubClient(
         auth=auth,
         base_url=settings.github_api_base,
@@ -85,6 +95,7 @@ def _build_client(settings: Settings, metrics: Metrics | None = None) -> GitHubC
         min_remaining=settings.api_min_remaining,
         max_retries=settings.api_max_retries,
         on_request=on_request,
+        on_rate_limit=on_rate_limit,
     )
 
 

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/src/github_actions_ingester/collector.py

@@ -56,6 +56,7 @@ CONCLUSIONS = (
     "stale",
     "startup_failure",
 )
+_JOBS_PROGRESS_EVERY = 250
 _RUN_BATCH = 500
 
 
@@ -198,9 +199,10 @@ class Collector:
 
         written = 0
         for batch in _batched(self._client.list_runs(repo, since, now), _RUN_BATCH):
-            written += self._store.upsert_runs(batch)
-        if written:
-            self._metrics.runs_upserted_total.labels(repository=repo.full_name).inc(written)
+            n = self._store.upsert_runs(batch)
+            written += n
+            if n:
+                self._metrics.runs_upserted_total.labels(repository=repo.full_name).inc(n)
 
         # Open runs that fell out of the window: refresh them individually.
         stale = self._store.open_runs_before(repo.id, since, s.max_open_run_refresh)
@@ -213,8 +215,12 @@ class Collector:
                 raise
             self._store.upsert_runs([run])
 
+        # One request per run: on a first backfill of a busy repository this
+        # loop runs for many minutes, so it reports progress on the way and
+        # moves the counters as it goes instead of once at the end.
         jobs_written = 0
-        for run_id in self._store.runs_needing_jobs(repo.id):
+        pending = self._store.runs_needing_jobs(repo.id)
+        for done, run_id in enumerate(pending, start=1):
             try:
                 jobs = list(self._client.list_jobs(repo, run_id, s.jobs_filter))
             except GitHubAPIError as exc:
@@ -223,9 +229,18 @@ class Collector:
                     self._store.upsert_jobs(run_id, [])
                     continue
                 raise
-            jobs_written += self._store.upsert_jobs(run_id, jobs)
-        if jobs_written:
-            self._metrics.jobs_upserted_total.labels(repository=repo.full_name).inc(jobs_written)
+            n = self._store.upsert_jobs(run_id, jobs)
+            jobs_written += n
+            if n:
+                self._metrics.jobs_upserted_total.labels(repository=repo.full_name).inc(n)
+            if done % _JOBS_PROGRESS_EVERY == 0 and done < len(pending):
+                logger.info(
+                    "jobs.progress",
+                    repo=repo.full_name,
+                    runs_done=done,
+                    runs_total=len(pending),
+                    jobs=jobs_written,
+                )
 
         self._store.set_cursor(repo.id, now, written)
         logger.info(

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/src/github_actions_ingester/github.py

@@ -338,6 +338,7 @@ class GitHubClient:
         min_remaining: int = 200,
         max_retries: int = 4,
         on_request: Callable[[int], None] | None = None,
+        on_rate_limit: Callable[[RateLimitState], None] | None = None,
         sleep: Callable[[float], None] = time.sleep,
         clock: Callable[[], float] = time.time,
     ) -> None:
@@ -355,6 +356,7 @@ class GitHubClient:
         self._min_remaining = min_remaining
         self._max_retries = max_retries
         self._on_request = on_request
+        self._on_rate_limit = on_rate_limit
         self._sleep = sleep
         self._clock = clock
         self.rate_limit = RateLimitState()
@@ -404,6 +406,8 @@ class GitHubClient:
                 self._sleep(min(2**attempt, 30))
                 continue
             self.rate_limit.update(resp.headers)
+            if self._on_rate_limit is not None:
+                self._on_rate_limit(self.rate_limit)
             if self._on_request is not None:
                 self._on_request(resp.status_code)
             if resp.status_code < 400:

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/src/github_actions_ingester/store.py

@@ -200,8 +200,45 @@ class Store:
                 )
                 granted.append(role)
         logger.info("store.read_access_granted", roles=granted, schema=self._schema)
+        self._set_database_search_path()
         return granted
 
+    def _set_database_search_path(self) -> None:
+        """Make the schema resolve by bare name for every session on the database.
+
+        The ingester sets ``search_path`` on its own connections, and a role
+        named like the schema gets it for free (``"$user"`` is first in the
+        default path), which is why the Compose example works without this.
+        A read role with a different name (``grafana`` reading schema
+        ``gha``) resolves ``minion_workflow_runs`` to nothing, and the
+        Grafana PostgreSQL datasource has no search_path setting.
+
+        ``ALTER DATABASE ... SET`` needs the database owner. When the ingester
+        is not the owner this logs and moves on: the grants above still hold,
+        and the operator can run the statement once by hand.
+        """
+        conn = self.connect()
+        try:
+            with conn.transaction(), conn.cursor() as cur:
+                cur.execute("SELECT current_database() AS db")
+                row = cur.fetchone()
+                database = row["db"] if row else ""
+                cur.execute(
+                    sql.SQL("ALTER DATABASE {} SET search_path TO {}, public").format(
+                        sql.Identifier(database), sql.Identifier(self._schema)
+                    )
+                )
+        except psycopg.errors.InsufficientPrivilege:
+            conn.rollback()
+            logger.warning(
+                "store.search_path_not_set",
+                schema=self._schema,
+                hint="ingester role does not own the database; run "
+                "ALTER DATABASE <db> SET search_path TO <schema>, public as the owner",
+            )
+            return
+        logger.info("store.search_path_set", schema=self._schema)
+
     def schema_version(self) -> int | None:
         """Highest applied migration, or None when the schema was never bootstrapped."""
         conn = self.connect()

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/tests/test_collector.py

@@ -195,6 +195,37 @@ def test_first_cycle_backfills_everything(migrated_store: Store, clock: list[dat
     assert _gauge(m, "gha_scheduled_workflow_last_conclusion", **labels, conclusion="failure") == 0
 
 
+def test_jobs_phase_reports_progress_and_moves_counters(
+    migrated_store: Store, clock: list[datetime], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """A long jobs phase logs every N runs and increments the counter per run,
+    so a stalled backfill is visible before ingest_repository() returns."""
+    import github_actions_ingester.collector as mod
+
+    gh = FakeGitHub()
+    gh.runs[1] = [run(300 + i, 1, 10) for i in range(5)]
+    for i in range(5):
+        gh.jobs[300 + i] = [job(3000 + i, 300 + i, 1)]
+    monkeypatch.setattr(mod, "_JOBS_PROGRESS_EVERY", 2)
+    events: list[dict[str, Any]] = []
+    seen_counter: list[float | None] = []
+    c, m = make(migrated_store, gh, clock)
+
+    def spy(event: str, **kw: Any) -> None:
+        if event == "jobs.progress":
+            events.append(kw)
+            seen_counter.append(
+                _gauge(m, "gha_ingester_jobs_upserted_total", repository="acme/web")
+            )
+
+    monkeypatch.setattr(mod.logger, "info", spy)
+    c.run_cycle()
+    assert [(e["runs_done"], e["runs_total"]) for e in events] == [(2, 5), (4, 5)]
+    # The counter had already moved when the progress line was written.
+    assert seen_counter == [2.0, 4.0]
+    assert _gauge(m, "gha_ingester_jobs_upserted_total", repository="acme/web") == 5.0
+
+
 def test_second_cycle_is_incremental(migrated_store: Store, clock: list[datetime]) -> None:
     gh = FakeGitHub()
     c, _ = make(migrated_store, gh, clock)

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/tests/test_github.py

@@ -330,6 +330,24 @@ def test_on_request_hook_receives_status(httpx_mock: HTTPXMock) -> None:
     assert seen == [404]
 
 
+def test_on_rate_limit_hook_fires_on_every_response(httpx_mock: HTTPXMock) -> None:
+    seen: list[int] = []
+    httpx_mock.add_response(
+        url=f"{API}/a",
+        json={},
+        headers={"X-RateLimit-Limit": "5000", "X-RateLimit-Remaining": "4000"},
+    )
+    httpx_mock.add_response(
+        url=f"{API}/b",
+        json={},
+        headers={"X-RateLimit-Limit": "5000", "X-RateLimit-Remaining": "3999"},
+    )
+    c = _client(on_rate_limit=lambda rl: seen.append(rl.remaining))
+    c.get("/a")
+    c.get("/b")
+    assert seen == [4000, 3999]
+
+
 # -- pagination / listings ---------------------------------------------------------
 
 

{github_actions_ingester-0.2.0 → github_actions_ingester-0.2.2}/tests/test_store.py

@@ -250,7 +250,10 @@ def test_grant_read_access_lets_a_role_read_views_and_future_tables(
         schema = migrated_store.schema
         reader_url = re.sub(r"//[^@]*@", f"//{role}:reader@", _with_userinfo(database_url))
         with psycopg.connect(reader_url, autocommit=True) as reader:
-            reader.execute(f"SET search_path TO {schema}")
+            # No SET search_path on purpose: Grafana cannot send one, so the
+            # database-level default the ingester sets must resolve the views.
+            path = reader.execute("SHOW search_path").fetchone()[0]
+            assert path.split(",")[0].strip().strip('"') == schema
             assert reader.execute("SELECT count(*) FROM minion_workflow_runs").fetchone()[0] == 0
             assert reader.execute("SELECT count(*) FROM workflow_jobs").fetchone()[0] == 0
             assert reader.execute("SELECT count(*) FROM later_migration").fetchone()[0] == 0
@@ -262,6 +265,8 @@ def test_grant_read_access_lets_a_role_read_views_and_future_tables(
         with psycopg.connect(database_url, autocommit=True) as admin:
             admin.execute(f"DROP OWNED BY {role}")
             admin.execute(f"DROP ROLE {role}")
+            db = admin.execute("SELECT current_database()").fetchone()[0]
+            admin.execute(f'ALTER DATABASE "{db}" RESET search_path')
 
 
 def test_grant_read_access_requires_an_existing_role(migrated_store: Store) -> None: