imbi-api 2.9.0__tar.gz → 2.9.2__tar.gz

{imbi_api-2.9.0 → imbi_api-2.9.2}/.pre-commit-config.yaml

@@ -21,6 +21,8 @@ repos:
     rev: v0.7.25
     hooks:
       - id: tombi-format
+        # uv owns uv.lock formatting; avoid tombi reflow churn
+        exclude: ^uv\.lock$
 
   - repo: local
     hooks:

{imbi_api-2.9.0 → imbi_api-2.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: imbi-api
-Version: 2.9.0
+Version: 2.9.2
 Summary: Imbi is a DevOps Service Management Platform designed to provide an efficient way to manage a large environment that contains many services and applications.
 Author-email: Alex Campbell <alexc@aweber.com>, Dave Shawley <daves@aweber.com>, "Gavin M. Roy" <gavinr@aweber.com>
 License: BSD-3-Clause
@@ -20,7 +20,7 @@ Requires-Dist: cyclonedx-python-lib<12,>=11.7
 Requires-Dist: fastapi
 Requires-Dist: filetype
 Requires-Dist: httpx<1,>=0.28.1
-Requires-Dist: imbi-common[databases,llm,sentry,server]==2.9.0
+Requires-Dist: imbi-common[databases,llm,sentry,server]==2.9.2
 Requires-Dist: jinja2
 Requires-Dist: jsonpatch>=1.33
 Requires-Dist: mcp>=1.26.0
@@ -31,7 +31,7 @@ Requires-Dist: pydantic[email]
 Requires-Dist: pyotp>=2.9.0
 Requires-Dist: python-multipart>=0.0.26
 Requires-Dist: qrcode>=8.0
-Requires-Dist: sentry-sdk>=2.48.0
+Requires-Dist: sentry-sdk<3,>=2.48.0
 Requires-Dist: slowapi==0.1.9
 Requires-Dist: typer
 Requires-Dist: uvicorn

{imbi_api-2.9.0 → imbi_api-2.9.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "imbi-api"
-version = "2.9.0"
+version = "2.9.2"
 description = "Imbi is a DevOps Service Management Platform designed to provide an efficient way to manage a large environment that contains many services and applications."
 readme = "README.md"
 requires-python = ">=3.14"
@@ -28,7 +28,7 @@ dependencies = [
   "fastapi",
   "filetype",
   "httpx>=0.28.1,<1",
-  "imbi-common[databases,llm,sentry,server]==2.9.0",
+  "imbi-common[databases,llm,sentry,server]==2.9.2",
   "jinja2",
   "jsonpatch>=1.33",
   "mcp>=1.26.0",
@@ -38,7 +38,7 @@ dependencies = [
   "pyotp>=2.9.0",
   "python-multipart>=0.0.26",
   "qrcode>=8.0",
-  "sentry-sdk>=2.48.0",
+  "sentry-sdk>=2.48.0,<3",  # 3.0 line abandoned upstream; stay on 2.x
   "slowapi==0.1.9",  # Pin: alpha quality, API may change
   "typer",
   "uvicorn",

{imbi_api-2.9.0 → imbi_api-2.9.2}/src/imbi_api/app.py

@@ -25,6 +25,7 @@ def create_app() -> fastapi.FastAPI:
             lifespans.anthropic_hook,
             valkey.valkey_lifespan,
             lifespans.score_worker_hook,
+            lifespans.commit_sync_worker_hook,
             lifespans.identity_refresh_hook,
         ),
         version=version,

{imbi_api-2.9.0 → imbi_api-2.9.2}/src/imbi_api/auth/permissions.py

@@ -13,7 +13,7 @@ import fastapi
 import jwt
 import pydantic
 from fastapi import security
-from imbi_common import graph
+from imbi_common import access_log, graph
 from imbi_common.auth import core
 
 from imbi_api import models, settings
@@ -658,7 +658,14 @@ async def _authenticate_token(
     treats the token as a JWT.
     """
     if token.startswith('ik_'):
-        return await authenticate_api_key(db, token, auth_settings)
+        ctx = await authenticate_api_key(db, token, auth_settings)
+        # Cache the key's owner so imbi-common's access log renders the
+        # person (email / service-account slug) instead of the opaque
+        # ``ik_<id>`` it parses from the Authorization header.
+        access_log.remember_api_key_principal(
+            ctx.session_id or '', ctx.principal_name
+        )
+        return ctx
     return await authenticate_jwt(db, token, auth_settings)
 
 

imbi_api-2.9.2/src/imbi_api/commit_sync/__init__.py

@@ -0,0 +1,28 @@
+"""On-demand commit/tag history sync for imbi-api.
+
+A project-scoped "Sync Commits & Tags" action (Project Doctor) enqueues a
+Valkey-stream job; a background worker resolves the project's
+``github-commit-sync`` webhook plugin, runs a full default-branch +
+all-tags backfill via the plugin's service credential, and records the
+last-sync status on the ``Project`` node for the UI to poll.
+"""
+
+from imbi_api.commit_sync.queue import (
+    consume_commit_sync,
+    enqueue_commit_sync,
+)
+from imbi_api.commit_sync.service import (
+    CommitSyncStatus,
+    CommitSyncUnavailable,
+    read_status,
+    run_sync,
+)
+
+__all__ = [
+    'CommitSyncStatus',
+    'CommitSyncUnavailable',
+    'consume_commit_sync',
+    'enqueue_commit_sync',
+    'read_status',
+    'run_sync',
+]

imbi_api-2.9.2/src/imbi_api/commit_sync/queue.py

@@ -0,0 +1,269 @@
+"""Commit/tag-sync queue (Valkey Streams).
+
+Mirrors :mod:`imbi_api.scoring.queue`: a single consumer group drains an
+``imbi:commit-sync`` stream, with a per-project debounce, stale-entry
+reclaim, and a dead-letter queue after repeated failures.  Each job runs
+a full commit/tag backfill via :func:`commit_sync.service.run_sync` and
+records the outcome on the ``Project`` node.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import socket
+import typing
+from collections import abc
+
+from imbi_common import graph
+from valkey import asyncio as valkey
+
+from imbi_api.commit_sync.service import (
+    CommitSyncUnavailable,
+    run_sync,
+    set_status,
+)
+
+STREAM = 'imbi:commit-sync'
+GROUP = 'commit-sync-workers'
+CONSUMER_PREFIX = 'worker'
+DLQ = 'imbi:commit-sync:dlq'
+DEBOUNCE_PREFIX = 'imbi:commit-sync:debounce'
+DEBOUNCE_SECONDS = 10
+MAX_DELIVERIES = 3
+CLAIM_IDLE_MS = 120_000
+
+LOGGER = logging.getLogger(__name__)
+
+
+async def enqueue_commit_sync(
+    client: valkey.Valkey | None,
+    org_slug: str,
+    project_id: str,
+    requested_by: str | None = None,
+) -> bool:
+    """Debounce-then-XADD a commit-sync job. Returns True if enqueued.
+
+    Tolerates *client* being ``None`` (returns False) so the endpoint can
+    surface "queueing unavailable" rather than 500 when Valkey is down.
+    """
+    if client is None:
+        return False
+    key = f'{DEBOUNCE_PREFIX}:{project_id}'
+    try:
+        acquired = await client.set(key, b'1', ex=DEBOUNCE_SECONDS, nx=True)
+        if not acquired:
+            return False
+        await client.xadd(
+            STREAM,
+            {
+                'org_slug': org_slug,
+                'project_id': project_id,
+                'requested_by': requested_by or 'system',
+            },
+        )
+    except Exception:
+        # Release the debounce key so a transient Valkey write failure
+        # doesn't block the user's immediate retry for DEBOUNCE_SECONDS.
+        try:
+            await client.delete(key)
+        except Exception:
+            LOGGER.exception('failed to clear debounce key for %s', project_id)
+        LOGGER.exception('enqueue_commit_sync failed for %s', project_id)
+        return False
+    return True
+
+
+async def ensure_group(client: valkey.Valkey) -> None:
+    try:
+        await client.xgroup_create(STREAM, GROUP, id='$', mkstream=True)
+    except Exception as err:
+        if 'BUSYGROUP' not in str(err):
+            LOGGER.exception('xgroup_create failed')
+            raise
+
+
+async def _process_message(db: graph.Graph, fields: dict[str, str]) -> None:
+    project_id = fields.get('project_id')
+    org_slug = fields.get('org_slug') or ''
+    requested_by = fields.get('requested_by') or 'system'
+    if not project_id:
+        return
+    await set_status(
+        db, project_id, status='running', requested_by=requested_by
+    )
+    try:
+        commits, tags = await run_sync(db, org_slug, project_id)
+    except CommitSyncUnavailable as exc:
+        # Misconfiguration, not transient: record and don't retry.
+        LOGGER.warning('commit-sync unavailable for %s: %s', project_id, exc)
+        await set_status(
+            db,
+            project_id,
+            status='failed',
+            requested_by=requested_by,
+            error=str(exc),
+        )
+        return
+    except Exception:
+        # Persist a generic, user-safe message; the polling endpoint
+        # exposes this status, so keep raw exception detail in logs only.
+        LOGGER.exception('commit-sync run failed for %s', project_id)
+        await set_status(
+            db,
+            project_id,
+            status='failed',
+            requested_by=requested_by,
+            error='Commit sync failed. See server logs for details.',
+        )
+        raise
+    LOGGER.info(
+        'commit-sync for %s recorded %d commits, %d tags',
+        project_id,
+        commits,
+        tags,
+    )
+    await set_status(
+        db,
+        project_id,
+        status='success',
+        requested_by=requested_by,
+        commits=commits,
+        tags=tags,
+    )
+
+
+def _decode_fields(
+    raw: abc.Mapping[bytes | str, bytes | str],
+) -> dict[str, str]:
+    return {
+        (k.decode() if isinstance(k, bytes) else k): (
+            v.decode() if isinstance(v, bytes) else v
+        )
+        for k, v in raw.items()
+    }
+
+
+async def _claim_stale(
+    client: valkey.Valkey,
+    consumer: str,
+) -> list[tuple[bytes, abc.Mapping[bytes | str, bytes | str]]]:
+    try:
+        result = await client.xautoclaim(
+            STREAM,
+            GROUP,
+            consumer,
+            min_idle_time=CLAIM_IDLE_MS,
+            start_id='0-0',
+            count=16,
+        )
+    except Exception as err:  # noqa: BLE001
+        LOGGER.debug('xautoclaim failed: %s', err)
+        return []
+    if isinstance(result, (list, tuple)) and len(result) >= 2:  # type: ignore[arg-type]
+        msgs: object = result[1]  # type: ignore[index]
+        if isinstance(msgs, list):
+            return msgs  # type: ignore[return-value]
+    return []
+
+
+async def _maybe_dead_letter(
+    client: valkey.Valkey,
+    msg_id: bytes,
+    fields: dict[str, str],
+) -> bool:
+    try:
+        info = await client.xpending_range(
+            STREAM, GROUP, min=msg_id, max=msg_id, count=1
+        )
+    except Exception:  # noqa: BLE001
+        return False
+    if not info:
+        return False
+    entry: object = info[0]  # type: ignore[index]
+    delivered: int | None = None
+    if isinstance(entry, dict):
+        raw_delivered = entry.get('times_delivered')  # type: ignore[union-attr]
+        if raw_delivered is not None:
+            delivered = int(raw_delivered)  # type: ignore[arg-type]
+    elif isinstance(entry, (list, tuple)) and len(entry) >= 4:  # type: ignore[arg-type]
+        raw_delivered = entry[3]  # type: ignore[index]
+        if raw_delivered is not None:
+            delivered = int(raw_delivered)  # type: ignore[arg-type]
+    if delivered is not None and delivered >= MAX_DELIVERIES:
+        await client.xadd(DLQ, fields)
+        await client.xack(STREAM, GROUP, msg_id)
+        LOGGER.warning(
+            'dead-lettered commit-sync msg %s after %s deliveries',
+            msg_id,
+            delivered,
+        )
+        return True
+    return False
+
+
+async def _handle_entries(
+    client: valkey.Valkey,
+    entries: list[tuple[bytes, abc.Mapping[bytes | str, bytes | str]]],
+    db: graph.Graph,
+    check_dlq: bool = False,
+) -> None:
+    for msg_id, raw_fields in entries:
+        fields = _decode_fields(raw_fields)
+        if check_dlq and await _maybe_dead_letter(client, msg_id, fields):
+            continue
+        try:
+            await _process_message(db, fields)
+        except Exception:
+            LOGGER.exception('commit-sync failed for %s', fields)
+            continue
+        await client.xack(STREAM, GROUP, msg_id)
+
+
+async def consume_commit_sync(
+    client: valkey.Valkey,
+    db: graph.Graph,
+    consumer: str | None = None,
+    stop: asyncio.Event | None = None,
+) -> None:
+    """Run the commit-sync consumer loop until *stop* is set."""
+    # Derive a per-process consumer name so concurrent workers don't share
+    # a Pending Entries List and stale-claim each other's in-flight jobs.
+    consumer = (
+        consumer or f'{CONSUMER_PREFIX}-{socket.gethostname()}-{os.getpid()}'
+    )
+    await ensure_group(client)
+    LOGGER.info('Commit-sync consumer loop running (consumer=%s)', consumer)
+    while stop is None or not stop.is_set():
+        stale = await _claim_stale(client, consumer)
+        if stale:
+            try:
+                await _handle_entries(client, stale, db, check_dlq=True)
+            except Exception:
+                LOGGER.exception('commit-sync stale-entry handling failed')
+                await asyncio.sleep(1)
+                continue
+        try:
+            response = await client.xreadgroup(
+                GROUP,
+                consumer,
+                {STREAM: '>'},
+                count=16,
+                block=2000,
+            )
+        except Exception:
+            LOGGER.exception('xreadgroup failed')
+            await asyncio.sleep(1)
+            continue
+        if not response:
+            continue
+        for _stream, entries in typing.cast(
+            'list[tuple[object, list[typing.Any]]]', response
+        ):
+            try:
+                await _handle_entries(client, entries, db)
+            except Exception:
+                LOGGER.exception('commit-sync entry handling failed')
+                await asyncio.sleep(1)
+                break