imbi-api 2.9.1__tar.gz → 2.9.2__tar.gz

{imbi_api-2.9.1 → imbi_api-2.9.2}/.pre-commit-config.yaml

@@ -21,6 +21,8 @@ repos:
     rev: v0.7.25
     hooks:
       - id: tombi-format
+        # uv owns uv.lock formatting; avoid tombi reflow churn
+        exclude: ^uv\.lock$
 
   - repo: local
     hooks:

{imbi_api-2.9.1 → imbi_api-2.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: imbi-api
-Version: 2.9.1
+Version: 2.9.2
 Summary: Imbi is a DevOps Service Management Platform designed to provide an efficient way to manage a large environment that contains many services and applications.
 Author-email: Alex Campbell <alexc@aweber.com>, Dave Shawley <daves@aweber.com>, "Gavin M. Roy" <gavinr@aweber.com>
 License: BSD-3-Clause
@@ -20,7 +20,7 @@ Requires-Dist: cyclonedx-python-lib<12,>=11.7
 Requires-Dist: fastapi
 Requires-Dist: filetype
 Requires-Dist: httpx<1,>=0.28.1
-Requires-Dist: imbi-common[databases,llm,sentry,server]==2.9.1
+Requires-Dist: imbi-common[databases,llm,sentry,server]==2.9.2
 Requires-Dist: jinja2
 Requires-Dist: jsonpatch>=1.33
 Requires-Dist: mcp>=1.26.0

{imbi_api-2.9.1 → imbi_api-2.9.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "imbi-api"
-version = "2.9.1"
+version = "2.9.2"
 description = "Imbi is a DevOps Service Management Platform designed to provide an efficient way to manage a large environment that contains many services and applications."
 readme = "README.md"
 requires-python = ">=3.14"
@@ -28,7 +28,7 @@ dependencies = [
   "fastapi",
   "filetype",
   "httpx>=0.28.1,<1",
-  "imbi-common[databases,llm,sentry,server]==2.9.1",
+  "imbi-common[databases,llm,sentry,server]==2.9.2",
   "jinja2",
   "jsonpatch>=1.33",
   "mcp>=1.26.0",

{imbi_api-2.9.1 → imbi_api-2.9.2}/src/imbi_api/app.py

@@ -25,6 +25,7 @@ def create_app() -> fastapi.FastAPI:
             lifespans.anthropic_hook,
             valkey.valkey_lifespan,
             lifespans.score_worker_hook,
+            lifespans.commit_sync_worker_hook,
             lifespans.identity_refresh_hook,
         ),
         version=version,

{imbi_api-2.9.1 → imbi_api-2.9.2}/src/imbi_api/auth/permissions.py

@@ -13,7 +13,7 @@ import fastapi
 import jwt
 import pydantic
 from fastapi import security
-from imbi_common import graph
+from imbi_common import access_log, graph
 from imbi_common.auth import core
 
 from imbi_api import models, settings
@@ -658,7 +658,14 @@ async def _authenticate_token(
     treats the token as a JWT.
     """
     if token.startswith('ik_'):
-        return await authenticate_api_key(db, token, auth_settings)
+        ctx = await authenticate_api_key(db, token, auth_settings)
+        # Cache the key's owner so imbi-common's access log renders the
+        # person (email / service-account slug) instead of the opaque
+        # ``ik_<id>`` it parses from the Authorization header.
+        access_log.remember_api_key_principal(
+            ctx.session_id or '', ctx.principal_name
+        )
+        return ctx
     return await authenticate_jwt(db, token, auth_settings)
 
 

imbi_api-2.9.2/src/imbi_api/commit_sync/__init__.py

@@ -0,0 +1,28 @@
+"""On-demand commit/tag history sync for imbi-api.
+
+A project-scoped "Sync Commits & Tags" action (Project Doctor) enqueues a
+Valkey-stream job; a background worker resolves the project's
+``github-commit-sync`` webhook plugin, runs a full default-branch +
+all-tags backfill via the plugin's service credential, and records the
+last-sync status on the ``Project`` node for the UI to poll.
+"""
+
+from imbi_api.commit_sync.queue import (
+    consume_commit_sync,
+    enqueue_commit_sync,
+)
+from imbi_api.commit_sync.service import (
+    CommitSyncStatus,
+    CommitSyncUnavailable,
+    read_status,
+    run_sync,
+)
+
+__all__ = [
+    'CommitSyncStatus',
+    'CommitSyncUnavailable',
+    'consume_commit_sync',
+    'enqueue_commit_sync',
+    'read_status',
+    'run_sync',
+]

imbi_api-2.9.2/src/imbi_api/commit_sync/queue.py

@@ -0,0 +1,269 @@
+"""Commit/tag-sync queue (Valkey Streams).
+
+Mirrors :mod:`imbi_api.scoring.queue`: a single consumer group drains an
+``imbi:commit-sync`` stream, with a per-project debounce, stale-entry
+reclaim, and a dead-letter queue after repeated failures.  Each job runs
+a full commit/tag backfill via :func:`commit_sync.service.run_sync` and
+records the outcome on the ``Project`` node.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import socket
+import typing
+from collections import abc
+
+from imbi_common import graph
+from valkey import asyncio as valkey
+
+from imbi_api.commit_sync.service import (
+    CommitSyncUnavailable,
+    run_sync,
+    set_status,
+)
+
+STREAM = 'imbi:commit-sync'
+GROUP = 'commit-sync-workers'
+CONSUMER_PREFIX = 'worker'
+DLQ = 'imbi:commit-sync:dlq'
+DEBOUNCE_PREFIX = 'imbi:commit-sync:debounce'
+DEBOUNCE_SECONDS = 10
+MAX_DELIVERIES = 3
+CLAIM_IDLE_MS = 120_000
+
+LOGGER = logging.getLogger(__name__)
+
+
+async def enqueue_commit_sync(
+    client: valkey.Valkey | None,
+    org_slug: str,
+    project_id: str,
+    requested_by: str | None = None,
+) -> bool:
+    """Debounce-then-XADD a commit-sync job. Returns True if enqueued.
+
+    Tolerates *client* being ``None`` (returns False) so the endpoint can
+    surface "queueing unavailable" rather than 500 when Valkey is down.
+    """
+    if client is None:
+        return False
+    key = f'{DEBOUNCE_PREFIX}:{project_id}'
+    try:
+        acquired = await client.set(key, b'1', ex=DEBOUNCE_SECONDS, nx=True)
+        if not acquired:
+            return False
+        await client.xadd(
+            STREAM,
+            {
+                'org_slug': org_slug,
+                'project_id': project_id,
+                'requested_by': requested_by or 'system',
+            },
+        )
+    except Exception:
+        # Release the debounce key so a transient Valkey write failure
+        # doesn't block the user's immediate retry for DEBOUNCE_SECONDS.
+        try:
+            await client.delete(key)
+        except Exception:
+            LOGGER.exception('failed to clear debounce key for %s', project_id)
+        LOGGER.exception('enqueue_commit_sync failed for %s', project_id)
+        return False
+    return True
+
+
+async def ensure_group(client: valkey.Valkey) -> None:
+    try:
+        await client.xgroup_create(STREAM, GROUP, id='$', mkstream=True)
+    except Exception as err:
+        if 'BUSYGROUP' not in str(err):
+            LOGGER.exception('xgroup_create failed')
+            raise
+
+
+async def _process_message(db: graph.Graph, fields: dict[str, str]) -> None:
+    project_id = fields.get('project_id')
+    org_slug = fields.get('org_slug') or ''
+    requested_by = fields.get('requested_by') or 'system'
+    if not project_id:
+        return
+    await set_status(
+        db, project_id, status='running', requested_by=requested_by
+    )
+    try:
+        commits, tags = await run_sync(db, org_slug, project_id)
+    except CommitSyncUnavailable as exc:
+        # Misconfiguration, not transient: record and don't retry.
+        LOGGER.warning('commit-sync unavailable for %s: %s', project_id, exc)
+        await set_status(
+            db,
+            project_id,
+            status='failed',
+            requested_by=requested_by,
+            error=str(exc),
+        )
+        return
+    except Exception:
+        # Persist a generic, user-safe message; the polling endpoint
+        # exposes this status, so keep raw exception detail in logs only.
+        LOGGER.exception('commit-sync run failed for %s', project_id)
+        await set_status(
+            db,
+            project_id,
+            status='failed',
+            requested_by=requested_by,
+            error='Commit sync failed. See server logs for details.',
+        )
+        raise
+    LOGGER.info(
+        'commit-sync for %s recorded %d commits, %d tags',
+        project_id,
+        commits,
+        tags,
+    )
+    await set_status(
+        db,
+        project_id,
+        status='success',
+        requested_by=requested_by,
+        commits=commits,
+        tags=tags,
+    )
+
+
+def _decode_fields(
+    raw: abc.Mapping[bytes | str, bytes | str],
+) -> dict[str, str]:
+    return {
+        (k.decode() if isinstance(k, bytes) else k): (
+            v.decode() if isinstance(v, bytes) else v
+        )
+        for k, v in raw.items()
+    }
+
+
+async def _claim_stale(
+    client: valkey.Valkey,
+    consumer: str,
+) -> list[tuple[bytes, abc.Mapping[bytes | str, bytes | str]]]:
+    try:
+        result = await client.xautoclaim(
+            STREAM,
+            GROUP,
+            consumer,
+            min_idle_time=CLAIM_IDLE_MS,
+            start_id='0-0',
+            count=16,
+        )
+    except Exception as err:  # noqa: BLE001
+        LOGGER.debug('xautoclaim failed: %s', err)
+        return []
+    if isinstance(result, (list, tuple)) and len(result) >= 2:  # type: ignore[arg-type]
+        msgs: object = result[1]  # type: ignore[index]
+        if isinstance(msgs, list):
+            return msgs  # type: ignore[return-value]
+    return []
+
+
+async def _maybe_dead_letter(
+    client: valkey.Valkey,
+    msg_id: bytes,
+    fields: dict[str, str],
+) -> bool:
+    try:
+        info = await client.xpending_range(
+            STREAM, GROUP, min=msg_id, max=msg_id, count=1
+        )
+    except Exception:  # noqa: BLE001
+        return False
+    if not info:
+        return False
+    entry: object = info[0]  # type: ignore[index]
+    delivered: int | None = None
+    if isinstance(entry, dict):
+        raw_delivered = entry.get('times_delivered')  # type: ignore[union-attr]
+        if raw_delivered is not None:
+            delivered = int(raw_delivered)  # type: ignore[arg-type]
+    elif isinstance(entry, (list, tuple)) and len(entry) >= 4:  # type: ignore[arg-type]
+        raw_delivered = entry[3]  # type: ignore[index]
+        if raw_delivered is not None:
+            delivered = int(raw_delivered)  # type: ignore[arg-type]
+    if delivered is not None and delivered >= MAX_DELIVERIES:
+        await client.xadd(DLQ, fields)
+        await client.xack(STREAM, GROUP, msg_id)
+        LOGGER.warning(
+            'dead-lettered commit-sync msg %s after %s deliveries',
+            msg_id,
+            delivered,
+        )
+        return True
+    return False
+
+
+async def _handle_entries(
+    client: valkey.Valkey,
+    entries: list[tuple[bytes, abc.Mapping[bytes | str, bytes | str]]],
+    db: graph.Graph,
+    check_dlq: bool = False,
+) -> None:
+    for msg_id, raw_fields in entries:
+        fields = _decode_fields(raw_fields)
+        if check_dlq and await _maybe_dead_letter(client, msg_id, fields):
+            continue
+        try:
+            await _process_message(db, fields)
+        except Exception:
+            LOGGER.exception('commit-sync failed for %s', fields)
+            continue
+        await client.xack(STREAM, GROUP, msg_id)
+
+
+async def consume_commit_sync(
+    client: valkey.Valkey,
+    db: graph.Graph,
+    consumer: str | None = None,
+    stop: asyncio.Event | None = None,
+) -> None:
+    """Run the commit-sync consumer loop until *stop* is set."""
+    # Derive a per-process consumer name so concurrent workers don't share
+    # a Pending Entries List and stale-claim each other's in-flight jobs.
+    consumer = (
+        consumer or f'{CONSUMER_PREFIX}-{socket.gethostname()}-{os.getpid()}'
+    )
+    await ensure_group(client)
+    LOGGER.info('Commit-sync consumer loop running (consumer=%s)', consumer)
+    while stop is None or not stop.is_set():
+        stale = await _claim_stale(client, consumer)
+        if stale:
+            try:
+                await _handle_entries(client, stale, db, check_dlq=True)
+            except Exception:
+                LOGGER.exception('commit-sync stale-entry handling failed')
+                await asyncio.sleep(1)
+                continue
+        try:
+            response = await client.xreadgroup(
+                GROUP,
+                consumer,
+                {STREAM: '>'},
+                count=16,
+                block=2000,
+            )
+        except Exception:
+            LOGGER.exception('xreadgroup failed')
+            await asyncio.sleep(1)
+            continue
+        if not response:
+            continue
+        for _stream, entries in typing.cast(
+            'list[tuple[object, list[typing.Any]]]', response
+        ):
+            try:
+                await _handle_entries(client, entries, db)
+            except Exception:
+                LOGGER.exception('commit-sync entry handling failed')
+                await asyncio.sleep(1)
+                break

imbi_api-2.9.2/src/imbi_api/commit_sync/service.py

@@ -0,0 +1,311 @@
+"""Resolve + invoke the commit-sync plugin and track its status.
+
+The on-demand sync acts with the ``github-commit-sync`` plugin's
+*service* credential (PAT or GitHub App), so there is no acting user: the
+worker resolves the plugin attached to a ``ThirdPartyService`` the project
+``EXISTS_IN``, builds the :class:`PluginContext` it needs (project links +
+the connected ``service_plugins`` for host resolution), decrypts the
+credential, and awaits the plugin's ``sync_all_history`` method.
+
+Last-sync state is persisted as a handful of properties on the ``Project``
+node so the UI can poll it without a dedicated status store.
+"""
+
+from __future__ import annotations
+
+import datetime
+import logging
+import typing
+from collections import abc
+
+import pydantic
+from imbi_common import graph
+from imbi_common.plugins.base import PluginContext, ServicePlugin
+from imbi_common.plugins.errors import PluginNotFoundError
+from imbi_common.plugins.registry import RegistryEntry, get_plugin
+
+from imbi_api.plugins import parse_options
+from imbi_api.plugins.credentials import get_plugin_credentials
+
+LOGGER = logging.getLogger(__name__)
+
+_COMMIT_SYNC_SLUG = 'github-commit-sync'
+# Persisted error strings are truncated so a noisy upstream message can't
+# bloat the Project node.
+_MAX_ERROR_LEN = 500
+
+SyncState = typing.Literal['idle', 'queued', 'running', 'success', 'failed']
+
+
+class CommitSyncUnavailable(Exception):
+    """No ``github-commit-sync`` plugin is reachable for the project."""
+
+
+class CommitSyncStatus(pydantic.BaseModel):
+    """Last-sync state for a project's commit/tag history."""
+
+    status: SyncState = 'idle'
+    last_synced_at: datetime.datetime | None = None
+    commits_synced: int | None = None
+    tags_synced: int | None = None
+    error: str | None = None
+    requested_by: str | None = None
+
+
+class _ResolvedCommitSync(typing.NamedTuple):
+    plugin_id: str
+    entry: RegistryEntry
+    tps_slug: str
+    service_endpoint: str | None
+    service_plugins: list[ServicePlugin]
+
+
+async def _resolve_plugin(
+    db: graph.Graph, project_id: str
+) -> _ResolvedCommitSync:
+    """Find the ``github-commit-sync`` plugin attached to a service the
+    project ``EXISTS_IN`` and gather the sibling plugins on that service.
+
+    Raises :class:`CommitSyncUnavailable` when no such plugin is
+    configured (or its registry entry is missing).
+    """
+    query: typing.LiteralString = """
+    MATCH (proj:Project {{id: {project_id}}})
+      -[:EXISTS_IN]->(tps:ThirdPartyService)
+      -[:HAS_PLUGIN]->(csp:Plugin {{plugin_slug: {slug}}})
+    OPTIONAL MATCH (tps)-[:HAS_PLUGIN]->(sib:Plugin)
+    WITH tps, csp,
+      collect(DISTINCT {{slug: sib.plugin_slug, options: sib.options}})
+        AS siblings
+    RETURN csp.id AS plugin_id,
+           tps.slug AS tps_slug,
+           tps.api_endpoint AS api_endpoint,
+           siblings AS siblings
+    LIMIT 1
+    """
+    records = await db.execute(
+        query,
+        {'project_id': project_id, 'slug': _COMMIT_SYNC_SLUG},
+        ['plugin_id', 'tps_slug', 'api_endpoint', 'siblings'],
+    )
+    if not records:
+        raise CommitSyncUnavailable(
+            'No github-commit-sync plugin is connected to a service this '
+            'project belongs to; configure it on the GitHub service.'
+        )
+    record = records[0]
+    plugin_id = graph.parse_agtype(record.get('plugin_id'))
+    if not plugin_id:
+        raise CommitSyncUnavailable(
+            'github-commit-sync plugin row is missing an id'
+        )
+    try:
+        entry = get_plugin(_COMMIT_SYNC_SLUG)
+    except PluginNotFoundError as exc:
+        raise CommitSyncUnavailable(
+            'github-commit-sync plugin is not loaded in the registry'
+        ) from exc
+    tps_slug = graph.parse_agtype(record.get('tps_slug'))
+    api_endpoint = graph.parse_agtype(record.get('api_endpoint'))
+    siblings = typing.cast(
+        'list[dict[str, typing.Any]]',
+        graph.parse_agtype(record.get('siblings')) or [],
+    )
+    service_plugins: list[ServicePlugin] = []
+    for sib in siblings:
+        slug = sib.get('slug')
+        if not slug:
+            continue
+        service_plugins.append(
+            ServicePlugin(
+                slug=str(slug), options=parse_options(sib.get('options'))
+            )
+        )
+    return _ResolvedCommitSync(
+        plugin_id=str(plugin_id),
+        entry=entry,
+        tps_slug=str(tps_slug) if tps_slug else '',
+        service_endpoint=str(api_endpoint) if api_endpoint else None,
+        service_plugins=service_plugins,
+    )
+
+
+async def _build_context(
+    db: graph.Graph,
+    org_slug: str,
+    project_id: str,
+    resolved: _ResolvedCommitSync,
+) -> PluginContext:
+    """Assemble the :class:`PluginContext` the plugin needs (no actor)."""
+    # Imported here (not at module load) so the worker/service module
+    # never pulls the endpoints package at import time.
+    from imbi_api.endpoints import _helpers
+
+    project_slug, team_slug = await _helpers.lookup_project_slugs(
+        db, project_id
+    )
+    project_links = await _helpers.lookup_project_links(db, project_id)
+    project_type_slugs = await _helpers.lookup_project_type_slugs(
+        db, project_id
+    )
+    service_connections = await _helpers.lookup_project_exists_in(
+        db, project_id
+    )
+    assignment_options: dict[str, typing.Any] = {
+        'service_slug': resolved.tps_slug,
+    }
+    if resolved.service_endpoint:
+        assignment_options['service_endpoint'] = resolved.service_endpoint
+    return PluginContext(
+        project_id=project_id,
+        project_slug=project_slug,
+        org_slug=org_slug,
+        team_slug=team_slug,
+        assignment_options=assignment_options,
+        service_plugins=resolved.service_plugins,
+        project_links=project_links,
+        project_type_slugs=project_type_slugs,
+        third_party_service_slug=resolved.tps_slug or None,
+        service_connections=service_connections,
+    )
+
+
+async def check_available(db: graph.Graph, project_id: str) -> None:
+    """Raise :class:`CommitSyncUnavailable` if the project can't be synced.
+
+    Used by the enqueue endpoint to fail fast (400) rather than queueing a
+    job that the worker can only mark failed.
+    """
+    await _resolve_plugin(db, project_id)
+
+
+async def run_sync(
+    db: graph.Graph, org_slug: str, project_id: str
+) -> tuple[int, int]:
+    """Resolve the commit-sync plugin and run a full history backfill.
+
+    Returns ``(commits_recorded, tags_recorded)``.  Raises
+    :class:`CommitSyncUnavailable` when no plugin/credential is
+    configured; other failures propagate so the caller can record them.
+    """
+    resolved = await _resolve_plugin(db, project_id)
+    ctx = await _build_context(db, org_slug, project_id, resolved)
+    credentials = await get_plugin_credentials(
+        db, resolved.plugin_id, resolved.entry
+    )
+    handler = resolved.entry.handler_cls()
+    sync = getattr(handler, 'sync_all_history', None)
+    if sync is None:
+        raise CommitSyncUnavailable(
+            'github-commit-sync plugin does not implement sync_all_history; '
+            'upgrade imbi-plugin-github'
+        )
+    sync_fn = typing.cast(
+        'abc.Callable[..., abc.Awaitable[tuple[int, int]]]', sync
+    )
+    return await sync_fn(ctx=ctx, credentials=credentials)
+
+
+def _now_iso() -> str:
+    return datetime.datetime.now(datetime.UTC).isoformat()
+
+
+async def set_status(
+    db: graph.Graph,
+    project_id: str,
+    *,
+    status: SyncState,
+    requested_by: str = '',
+    commits: int = 0,
+    tags: int = 0,
+    error: str = '',
+) -> None:
+    """Persist last-sync state on the ``Project`` node (best-effort)."""
+    query: typing.LiteralString = """
+    MATCH (p:Project {{id: {project_id}}})
+    SET p.commit_sync_status = {status},
+        p.commit_sync_at = {at},
+        p.commit_sync_by = {by},
+        p.commit_sync_commits = {commits},
+        p.commit_sync_tags = {tags},
+        p.commit_sync_error = {error}
+    RETURN p.id AS id
+    """
+    try:
+        await db.execute(
+            query,
+            {
+                'project_id': project_id,
+                'status': status,
+                'at': _now_iso(),
+                'by': requested_by,
+                'commits': commits,
+                'tags': tags,
+                'error': error[:_MAX_ERROR_LEN],
+            },
+            ['id'],
+        )
+    except Exception:  # noqa: BLE001
+        LOGGER.warning(
+            'Failed to persist commit-sync status for project %s',
+            project_id,
+            exc_info=True,
+        )
+
+
+def _opt_str(value: object) -> str | None:
+    text = str(value) if value is not None else ''
+    return text or None
+
+
+def _opt_int(value: object) -> int | None:
+    if isinstance(value, bool):
+        return int(value)
+    if isinstance(value, int):
+        return value
+    if isinstance(value, str):
+        try:
+            return int(value)
+        except ValueError:
+            return None
+    return None
+
+
+async def read_status(db: graph.Graph, project_id: str) -> CommitSyncStatus:
+    """Read last-sync state from the ``Project`` node (``idle`` default)."""
+    query: typing.LiteralString = """
+    MATCH (p:Project {{id: {project_id}}})
+    RETURN p.commit_sync_status AS status,
+           p.commit_sync_at AS at,
+           p.commit_sync_by AS by,
+           p.commit_sync_commits AS commits,
+           p.commit_sync_tags AS tags,
+           p.commit_sync_error AS error
+    """
+    records = await db.execute(
+        query,
+        {'project_id': project_id},
+        ['status', 'at', 'by', 'commits', 'tags', 'error'],
+    )
+    if not records:
+        return CommitSyncStatus()
+    row = records[0]
+    status_raw = graph.parse_agtype(row.get('status'))
+    status: SyncState = 'idle'
+    if status_raw in ('queued', 'running', 'success', 'failed', 'idle'):
+        status = status_raw
+    at_raw = _opt_str(graph.parse_agtype(row.get('at')))
+    last_synced_at: datetime.datetime | None = None
+    if at_raw:
+        try:
+            last_synced_at = datetime.datetime.fromisoformat(at_raw)
+        except ValueError:
+            last_synced_at = None
+    return CommitSyncStatus(
+        status=status,
+        last_synced_at=last_synced_at,
+        commits_synced=_opt_int(graph.parse_agtype(row.get('commits'))),
+        tags_synced=_opt_int(graph.parse_agtype(row.get('tags'))),
+        error=_opt_str(graph.parse_agtype(row.get('error'))),
+        requested_by=_opt_str(graph.parse_agtype(row.get('by'))),
+    )