github-actions-ingester 0.2.1__tar.gz → 0.2.2__tar.gz

{github_actions_ingester-0.2.1 → github_actions_ingester-0.2.2}/CHANGELOG.md

@@ -4,6 +4,20 @@ All notable changes to this project are documented here. The format follows
 [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project
 adheres to [Semantic Versioning](https://semver.org/).
 
+## [0.2.2] — 2026-09-05
+
+### Fixed
+- The GitHub rate-limit gauges were refreshed once per repository, so during
+  a long first backfill of a busy repository they could lag by twenty
+  minutes while the real budget ran out. They now follow every response,
+  as the README already said they did.
+- `runs_upserted_total` and `jobs_upserted_total` move as rows are written
+  instead of once when the repository finishes.
+
+### Added
+- `jobs.progress` log line every 250 runs during the jobs phase, which was
+  silent for the whole duration of a large backfill.
+
 ## [0.2.1] — 2026-09-05
 
 ### Added

{github_actions_ingester-0.2.1 → github_actions_ingester-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: github-actions-ingester
-Version: 0.2.1
+Version: 0.2.2
 Summary: Standalone GitHub Actions ingester — pulls workflows, runs and jobs from the GitHub API into PostgreSQL for Grafana dashboards, FinOps and alerting. Prometheus /metrics for ingester health and scheduled-workflow liveness.
 Project-URL: Homepage, https://github.com/danielgines/github-actions-ingester
 Project-URL: Repository, https://github.com/danielgines/github-actions-ingester

{github_actions_ingester-0.2.1 → github_actions_ingester-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "github-actions-ingester"
-version = "0.2.1"
+version = "0.2.2"
 description = "Standalone GitHub Actions ingester — pulls workflows, runs and jobs from the GitHub API into PostgreSQL for Grafana dashboards, FinOps and alerting. Prometheus /metrics for ingester health and scheduled-workflow liveness."
 readme = "README.md"
 requires-python = ">=3.11"

{github_actions_ingester-0.2.1 → github_actions_ingester-0.2.2}/src/github_actions_ingester/__init__.py

@@ -6,4 +6,4 @@ its own, and exposes Prometheus metrics about the ingestion itself and
 about the liveness of scheduled workflows.
 """
 
-__version__ = "0.2.1"
+__version__ = "0.2.2"

{github_actions_ingester-0.2.1 → github_actions_ingester-0.2.2}/src/github_actions_ingester/__main__.py

@@ -26,7 +26,7 @@ from . import __version__
 from .app_manifest import convert_code, manifest_form_url, manifest_json, write_private_key
 from .collector import Collector
 from .config import Settings, load_settings
-from .github import AppAuth, GitHubAPIError, GitHubClient, TokenAuth
+from .github import AppAuth, GitHubAPIError, GitHubClient, RateLimitState, TokenAuth
 from .metrics import Metrics
 from .ratelimit import RateLimiter
 from .server import MetricsServer
@@ -77,6 +77,16 @@ def _build_client(settings: Settings, metrics: Metrics | None = None) -> GitHubC
         if metrics is not None:
             metrics.github_requests_total.labels(status=str(status)).inc()
 
+    # Refresh the budget gauges on every response, not once per repository:
+    # a busy repository keeps the collector inside one ingest_repository()
+    # call for many minutes, and an alert on the remaining budget must not
+    # wait for it to finish.
+    def on_rate_limit(rl: RateLimitState) -> None:
+        if metrics is not None:
+            metrics.github_rate_limit_remaining.set(rl.remaining)
+            metrics.github_rate_limit_limit.set(rl.limit)
+            metrics.github_rate_limit_reset_timestamp_seconds.set(rl.reset_at)
+
     return GitHubClient(
         auth=auth,
         base_url=settings.github_api_base,
@@ -85,6 +95,7 @@ def _build_client(settings: Settings, metrics: Metrics | None = None) -> GitHubC
         min_remaining=settings.api_min_remaining,
         max_retries=settings.api_max_retries,
         on_request=on_request,
+        on_rate_limit=on_rate_limit,
     )
 
 

{github_actions_ingester-0.2.1 → github_actions_ingester-0.2.2}/src/github_actions_ingester/collector.py

@@ -56,6 +56,7 @@ CONCLUSIONS = (
     "stale",
     "startup_failure",
 )
+_JOBS_PROGRESS_EVERY = 250
 _RUN_BATCH = 500
 
 
@@ -198,9 +199,10 @@ class Collector:
 
         written = 0
         for batch in _batched(self._client.list_runs(repo, since, now), _RUN_BATCH):
-            written += self._store.upsert_runs(batch)
-        if written:
-            self._metrics.runs_upserted_total.labels(repository=repo.full_name).inc(written)
+            n = self._store.upsert_runs(batch)
+            written += n
+            if n:
+                self._metrics.runs_upserted_total.labels(repository=repo.full_name).inc(n)
 
         # Open runs that fell out of the window: refresh them individually.
         stale = self._store.open_runs_before(repo.id, since, s.max_open_run_refresh)
@@ -213,8 +215,12 @@ class Collector:
                 raise
             self._store.upsert_runs([run])
 
+        # One request per run: on a first backfill of a busy repository this
+        # loop runs for many minutes, so it reports progress on the way and
+        # moves the counters as it goes instead of once at the end.
         jobs_written = 0
-        for run_id in self._store.runs_needing_jobs(repo.id):
+        pending = self._store.runs_needing_jobs(repo.id)
+        for done, run_id in enumerate(pending, start=1):
             try:
                 jobs = list(self._client.list_jobs(repo, run_id, s.jobs_filter))
             except GitHubAPIError as exc:
@@ -223,9 +229,18 @@ class Collector:
                     self._store.upsert_jobs(run_id, [])
                     continue
                 raise
-            jobs_written += self._store.upsert_jobs(run_id, jobs)
-        if jobs_written:
-            self._metrics.jobs_upserted_total.labels(repository=repo.full_name).inc(jobs_written)
+            n = self._store.upsert_jobs(run_id, jobs)
+            jobs_written += n
+            if n:
+                self._metrics.jobs_upserted_total.labels(repository=repo.full_name).inc(n)
+            if done % _JOBS_PROGRESS_EVERY == 0 and done < len(pending):
+                logger.info(
+                    "jobs.progress",
+                    repo=repo.full_name,
+                    runs_done=done,
+                    runs_total=len(pending),
+                    jobs=jobs_written,
+                )
 
         self._store.set_cursor(repo.id, now, written)
         logger.info(

{github_actions_ingester-0.2.1 → github_actions_ingester-0.2.2}/src/github_actions_ingester/github.py

@@ -338,6 +338,7 @@ class GitHubClient:
         min_remaining: int = 200,
         max_retries: int = 4,
         on_request: Callable[[int], None] | None = None,
+        on_rate_limit: Callable[[RateLimitState], None] | None = None,
         sleep: Callable[[float], None] = time.sleep,
         clock: Callable[[], float] = time.time,
     ) -> None:
@@ -355,6 +356,7 @@ class GitHubClient:
         self._min_remaining = min_remaining
         self._max_retries = max_retries
         self._on_request = on_request
+        self._on_rate_limit = on_rate_limit
         self._sleep = sleep
         self._clock = clock
         self.rate_limit = RateLimitState()
@@ -404,6 +406,8 @@ class GitHubClient:
                 self._sleep(min(2**attempt, 30))
                 continue
             self.rate_limit.update(resp.headers)
+            if self._on_rate_limit is not None:
+                self._on_rate_limit(self.rate_limit)
             if self._on_request is not None:
                 self._on_request(resp.status_code)
             if resp.status_code < 400:

{github_actions_ingester-0.2.1 → github_actions_ingester-0.2.2}/tests/test_collector.py

@@ -195,6 +195,37 @@ def test_first_cycle_backfills_everything(migrated_store: Store, clock: list[dat
     assert _gauge(m, "gha_scheduled_workflow_last_conclusion", **labels, conclusion="failure") == 0
 
 
+def test_jobs_phase_reports_progress_and_moves_counters(
+    migrated_store: Store, clock: list[datetime], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """A long jobs phase logs every N runs and increments the counter per run,
+    so a stalled backfill is visible before ingest_repository() returns."""
+    import github_actions_ingester.collector as mod
+
+    gh = FakeGitHub()
+    gh.runs[1] = [run(300 + i, 1, 10) for i in range(5)]
+    for i in range(5):
+        gh.jobs[300 + i] = [job(3000 + i, 300 + i, 1)]
+    monkeypatch.setattr(mod, "_JOBS_PROGRESS_EVERY", 2)
+    events: list[dict[str, Any]] = []
+    seen_counter: list[float | None] = []
+    c, m = make(migrated_store, gh, clock)
+
+    def spy(event: str, **kw: Any) -> None:
+        if event == "jobs.progress":
+            events.append(kw)
+            seen_counter.append(
+                _gauge(m, "gha_ingester_jobs_upserted_total", repository="acme/web")
+            )
+
+    monkeypatch.setattr(mod.logger, "info", spy)
+    c.run_cycle()
+    assert [(e["runs_done"], e["runs_total"]) for e in events] == [(2, 5), (4, 5)]
+    # The counter had already moved when the progress line was written.
+    assert seen_counter == [2.0, 4.0]
+    assert _gauge(m, "gha_ingester_jobs_upserted_total", repository="acme/web") == 5.0
+
+
 def test_second_cycle_is_incremental(migrated_store: Store, clock: list[datetime]) -> None:
     gh = FakeGitHub()
     c, _ = make(migrated_store, gh, clock)

{github_actions_ingester-0.2.1 → github_actions_ingester-0.2.2}/tests/test_github.py

@@ -330,6 +330,24 @@ def test_on_request_hook_receives_status(httpx_mock: HTTPXMock) -> None:
     assert seen == [404]
 
 
+def test_on_rate_limit_hook_fires_on_every_response(httpx_mock: HTTPXMock) -> None:
+    seen: list[int] = []
+    httpx_mock.add_response(
+        url=f"{API}/a",
+        json={},
+        headers={"X-RateLimit-Limit": "5000", "X-RateLimit-Remaining": "4000"},
+    )
+    httpx_mock.add_response(
+        url=f"{API}/b",
+        json={},
+        headers={"X-RateLimit-Limit": "5000", "X-RateLimit-Remaining": "3999"},
+    )
+    c = _client(on_rate_limit=lambda rl: seen.append(rl.remaining))
+    c.get("/a")
+    c.get("/b")
+    assert seen == [4000, 3999]
+
+
 # -- pagination / listings ---------------------------------------------------------