imbi-api 2.34.0__tar.gz → 2.35.0__tar.gz

{imbi_api-2.34.0 → imbi_api-2.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: imbi-api
-Version: 2.34.0
+Version: 2.35.0
 Summary: Imbi is a DevOps Service Management Platform designed to provide an efficient way to manage a large environment that contains many services and applications.
 Author-email: Alex Campbell <alexc@aweber.com>, Dave Shawley <daves@aweber.com>, "Gavin M. Roy" <gavinr@aweber.com>
 License-Expression: BSD-3-Clause
@@ -19,7 +19,7 @@ Requires-Dist: cyclonedx-python-lib<12,>=11.7
 Requires-Dist: fastapi
 Requires-Dist: filetype
 Requires-Dist: httpx<1,>=0.28.1
-Requires-Dist: imbi-common[databases,llm,sentry,server]==2.34.0
+Requires-Dist: imbi-common[databases,llm,sentry,server]==2.35.0
 Requires-Dist: jinja2
 Requires-Dist: jsonpatch>=1.33
 Requires-Dist: mcp>=1.26.0

{imbi_api-2.34.0 → imbi_api-2.35.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "imbi-api"
-version = "2.34.0"
+version = "2.35.0"
 description = "Imbi is a DevOps Service Management Platform designed to provide an efficient way to manage a large environment that contains many services and applications."
 readme = "README.md"
 requires-python = ">=3.14"
@@ -28,7 +28,7 @@ dependencies = [
   "fastapi",
   "filetype",
   "httpx>=0.28.1,<1",
-  "imbi-common[databases,llm,sentry,server]==2.34.0",
+  "imbi-common[databases,llm,sentry,server]==2.35.0",
   "jinja2",
   "jsonpatch>=1.33",
   "mcp>=1.26.0",

{imbi_api-2.34.0 → imbi_api-2.35.0}/src/imbi/api/commit_pushed_at_repair.py

@@ -80,7 +80,7 @@ import typing
 
 import pydantic
 
-from imbi.common import clickhouse, models
+from imbi.common import clickhouse, iggy, models
 
 LOGGER = logging.getLogger(__name__)
 
@@ -235,7 +235,7 @@ async def repair_project(
         largest = max(largest, shift)
         repaired.append(_repaired_record(row, delivered))
     if repaired and not dry_run:
-        await clickhouse.insert('commits', repaired)
+        await iggy.publish('commits', 'maintenance', repaired)
         LOGGER.info(
             'restored pushed_at on %d commit(s) for project %s',
             len(repaired),

{imbi_api-2.34.0 → imbi_api-2.35.0}/src/imbi/api/component_facts.py

@@ -332,10 +332,14 @@ async def release_components(
     ``ComponentRelease`` node and were never copied here, so the
     listing endpoint fetches them from the graph by id -- a bounded
     ``IN`` over one release's versions, not a traversal.
+
+    ``DISTINCT`` because the stream delivers at least once: a
+    redelivered fact row lands inside the winning batch, where the
+    batch ranking cannot remove it.
     """
     rows = await clickhouse.query(
         _by_release(
-            'SELECT c.component_id AS component_id,'
+            'SELECT DISTINCT c.component_id AS component_id,'
             ' c.component_release_id AS component_release_id,'
             ' c.purl_name AS purl_name,'
             ' c.ecosystem AS ecosystem,'

{imbi_api-2.34.0 → imbi_api-2.35.0}/src/imbi/api/drift.py

@@ -40,8 +40,7 @@ import typing
 import fastapi
 
 from imbi.api.auth import principals
-from imbi.common import clickhouse, graph
-from imbi.common.clickhouse import client as ch_client
+from imbi.common import clickhouse, graph, iggy
 from imbi.common.plugins import errors as plugin_errors
 
 LOGGER = logging.getLogger(__name__)
@@ -54,14 +53,6 @@ NAMESPACE = 'imbi-drift'
 #: commit, which is what OR-ing a range needs.
 VERDICT_TABLE = 'commit_drift'
 
-_VERDICT_COLUMNS = [
-    'project_id',
-    'sha',
-    'drift_detected',
-    'paths',
-    'recorded_at',
-]
-
 REQUESTED_BY = principals.DRIFT_SYNC
 
 #: How many unanswered Releases one sweep asks the remote about.  Every
@@ -172,7 +163,7 @@ def parse_note(body: str | None) -> bool | None:
 async def record_verdicts(
     project_id: str, verdicts: dict[str, NoteVerdict]
 ) -> int | None:
-    """Persist per-commit verdicts, one insert for the whole batch.
+    """Persist per-commit verdicts, one publish for the whole batch.
 
     Keyed by the annotated commit's full SHA to join ``imbi.commits``,
     so only callers holding full SHAs write here -- the notes tree is
@@ -191,16 +182,20 @@ async def record_verdicts(
     """
     now = datetime.datetime.now(datetime.UTC)
     rows = [
-        [project_id, sha.lower(), verdict.drift_detected, verdict.paths, now]
+        {
+            'project_id': project_id,
+            'sha': sha.lower(),
+            'drift_detected': verdict.drift_detected,
+            'paths': verdict.paths,
+            'recorded_at': now,
+        }
         for sha, verdict in verdicts.items()
         if verdict.drift_detected is not None
     ]
     if not rows:
         return 0
     try:
-        await ch_client.Clickhouse.get_instance().insert(
-            VERDICT_TABLE, rows, _VERDICT_COLUMNS
-        )
+        await iggy.publish_rows(VERDICT_TABLE, 'drift', rows)
     except Exception:
         LOGGER.exception(
             'could not record %d drift verdicts for project %s',

{imbi_api-2.34.0 → imbi_api-2.35.0}/src/imbi/api/email/__init__.py

@@ -13,7 +13,7 @@ rendered using Jinja2 templates with both HTML and plain text versions.
 import logging
 from urllib import parse
 
-from imbi.common import clickhouse, graph
+from imbi.common import graph, iggy
 
 from .client import EmailClient
 from .dependencies import InjectEmailClient, InjectTemplateManager
@@ -154,18 +154,18 @@ async def send_password_reset(
 
 
 async def _save_audit(audit: models.EmailAudit) -> None:
-    """Save email audit record to ClickHouse.
+    """Publish the email audit record to the ``email_audit`` stream.
 
     Args:
         audit: Email audit record to save
 
     """
     try:
-        await clickhouse.insert('email_audit', [audit])
-        LOGGER.debug('Email audit saved to ClickHouse: %s', audit.to_email)
-    except clickhouse.client.DatabaseError as err:
+        await iggy.publish('email_audit', 'email', [audit])
+        LOGGER.debug('Email audit published: %s', audit.to_email)
+    except iggy.PublishError as err:
         # Log error but don't fail the email send
         LOGGER.warning(
-            'Failed to save email audit to ClickHouse: %s',
+            'Failed to publish email audit: %s',
             err,
         )

{imbi_api-2.34.0 → imbi_api-2.35.0}/src/imbi/api/endpoints/_document_events.py

@@ -10,7 +10,7 @@ than surfaced.
 import datetime
 import logging
 
-from imbi.common import clickhouse, models
+from imbi.common import iggy, models
 
 LOGGER = logging.getLogger(__name__)
 
@@ -34,8 +34,9 @@ async def emit_like_event(
     activity feed, which keys on ``attributed_to`` rather than project.
     """
     try:
-        await clickhouse.insert(
+        await iggy.publish(
             'events',
+            'documents',
             [
                 models.Event(
                     project_id=project_id,

{imbi_api-2.34.0 → imbi_api-2.35.0}/src/imbi/api/endpoints/_document_history.py

@@ -18,7 +18,7 @@ import typing
 
 import pydantic
 
-from imbi.common import clickhouse
+from imbi.common import clickhouse, iggy
 
 LOGGER = logging.getLogger(__name__)
 
@@ -79,8 +79,9 @@ async def record_created(
 ) -> None:
     """Record version 1 for a newly created document. Best-effort."""
     try:
-        await clickhouse.insert(
+        await iggy.publish(
             'document_versions',
+            'documents',
             [
                 DocumentVersionRow(
                     document_id=document_id,
@@ -162,7 +163,7 @@ async def record_updated(
                 recorded_at=now,
             )
         )
-        await clickhouse.insert('document_versions', rows)
+        await iggy.publish('document_versions', 'documents', rows)
     except Exception:
         LOGGER.exception(
             'failed to record version %s for document %s',

{imbi_api-2.34.0 → imbi_api-2.35.0}/src/imbi/api/endpoints/_document_reads.py

@@ -27,7 +27,7 @@ import pydantic
 from valkey import asyncio as valkey_asyncio
 
 from imbi.api.endpoints import _document_scope
-from imbi.common import clickhouse, graph
+from imbi.common import clickhouse, graph, iggy
 
 LOGGER = logging.getLogger(__name__)
 
@@ -238,8 +238,9 @@ async def record_events(rows: list[DocumentReadEventRow]) -> None:
     if not rows:
         return
     try:
-        await clickhouse.insert(
+        await iggy.publish(
             'document_read_events',
+            'documents',
             typing.cast('list[pydantic.BaseModel]', rows),
         )
     except Exception:
@@ -352,7 +353,7 @@ async def finalize_sessions(session_ids: list[str]) -> int:
     if not rows:
         return 0
     try:
-        await clickhouse.insert('document_read_sessions', rows)
+        await iggy.publish('document_read_sessions', 'documents', rows)
     except Exception:
         LOGGER.exception('failed to finalize %d session(s)', len(rows))
         return 0
@@ -366,8 +367,20 @@ async def finalize_sessions(session_ids: list[str]) -> int:
 #: never having been finalized.
 SWEEP_LOOKBACK_HOURS = 24
 
-# Sessions whose last heartbeat has aged past the idle timeout and that
-# have no finalized row yet.
+# Sessions whose last heartbeat has aged past the idle timeout and whose
+# finalized row does not yet cover that heartbeat.
+#
+# The condition is "not finalized *through* the last beat", not "not
+# finalized at all". Heartbeats reach ClickHouse through Iggy, so a
+# session finalized inline by its own final flush aggregates only the
+# beats the sink had already drained -- the last one or two are
+# routinely missing, and `engaged_ms` and `max_scroll_pct` are short by
+# that much. Skipping every session that has any row would make that
+# shortfall permanent. Comparing `ended_at` against the last beat makes
+# this sweep a repair pass instead: it re-finalizes the session once the
+# sink has caught up, `finalize_sessions` is idempotent, and the
+# rewritten row then covers the last beat, so the session stops
+# matching.
 #
 # Both sides are pinned to the same recent window so neither scans more
 # than the other needs: without the `recorded_at` bound the outer scan
@@ -386,11 +399,12 @@ FROM (
     HAVING last_beat < now() - INTERVAL {idle_seconds:UInt32} SECOND
 ) AS e
 LEFT JOIN (
-    SELECT DISTINCT session_id
+    SELECT session_id, max(ended_at) AS finalized_through
     FROM imbi.document_read_sessions
     WHERE started_at > now() - INTERVAL {lookback_hours:UInt32} HOUR
+    GROUP BY session_id
 ) AS s USING (session_id)
-WHERE s.session_id = ''
+WHERE s.session_id = '' OR s.finalized_through < e.last_beat
 LIMIT {batch:UInt32}
 """
 

{imbi_api-2.34.0 → imbi_api-2.35.0}/src/imbi/api/endpoints/comments.py

@@ -25,9 +25,8 @@ import pydantic
 
 from imbi.api.auth import permissions
 from imbi.api.endpoints import _search_index
-from imbi.common import graph, models
+from imbi.common import graph, iggy, models
 from imbi.common import patch as json_patch
-from imbi.common.clickhouse import client as ch_client
 
 LOGGER = logging.getLogger(__name__)
 
@@ -55,18 +54,19 @@ async def _emit_comment_event(
     failed analytics write must not fail the comment request.
     """
     try:
-        await ch_client.Clickhouse.get_instance().insert(
+        await iggy.publish_rows(
             'events',
+            'comments',
             [
-                [
-                    nanoid.generate(),
-                    project_id or '',
-                    datetime.datetime.now(datetime.UTC),
-                    _COMMENT_EVENT_TYPE,
-                    'internal',
-                    principal,
-                    {},
-                    {
+                {
+                    'id': nanoid.generate(),
+                    'project_id': project_id or '',
+                    'recorded_at': datetime.datetime.now(datetime.UTC),
+                    'type': _COMMENT_EVENT_TYPE,
+                    'integration': 'internal',
+                    'attributed_to': principal,
+                    'metadata': {},
+                    'payload': {
                         'document_id': document_id,
                         'thread_id': thread_id,
                         'comment_id': comment_id,
@@ -74,17 +74,7 @@ async def _emit_comment_event(
                         'action': action,
                         'excerpt': body[:140],
                     },
-                ]
-            ],
-            [
-                'id',
-                'project_id',
-                'recorded_at',
-                'type',
-                'integration',
-                'attributed_to',
-                'metadata',
-                'payload',
+                }
             ],
         )
     except Exception:

{imbi_api-2.34.0 → imbi_api-2.35.0}/src/imbi/api/endpoints/dashboard.py

@@ -1,6 +1,7 @@
 """Admin dashboard system-health status endpoint.
 
-Probes the datastores in-process (PostgreSQL/AGE, ClickHouse, Valkey)
+Probes the datastores in-process (PostgreSQL/AGE, ClickHouse, Valkey,
+Iggy)
 and the sibling HTTP services via their ``/status`` endpoints. Every
 check runs concurrently with a short timeout so the endpoint never
 blocks on a degraded dependency.
@@ -18,7 +19,7 @@ import pydantic
 
 from imbi.api import settings, version
 from imbi.api.auth import permissions
-from imbi.common import clickhouse, graph, valkey
+from imbi.common import clickhouse, graph, iggy, valkey
 
 LOGGER = logging.getLogger(__name__)
 
@@ -207,6 +208,35 @@ async def _check_valkey() -> DatastoreStatus:
     )
 
 
+async def _check_iggy() -> DatastoreStatus:
+    """Ping Iggy and total the bytes held by the topics Imbi publishes."""
+    start = time.perf_counter()
+    try:
+        await asyncio.wait_for(iggy.ping(), _CHECK_TIMEOUT)
+    except Exception as err:  # noqa: BLE001
+        LOGGER.warning('Iggy health check failed: %s', err)
+        return DatastoreStatus(
+            name='Iggy',
+            role='Event streaming',
+            status='error',
+            detail=str(err),
+        )
+    latency = _ms(start)
+    # Size is best-effort; it must not fail the (already-ok) check.
+    try:
+        size = await asyncio.wait_for(iggy.stored_bytes(), _CHECK_TIMEOUT)
+    except Exception as err:  # noqa: BLE001
+        LOGGER.warning('Iggy size probe failed: %s', err)
+        size = None
+    return DatastoreStatus(
+        name='Iggy',
+        role='Event streaming',
+        status='ok',
+        latency_ms=latency,
+        size_bytes=size,
+    )
+
+
 async def _check_service(
     client: httpx.AsyncClient, name: str, base_url: str
 ) -> ServiceStatus:
@@ -251,6 +281,7 @@ async def get_dashboard_status(
                 _check_postgres(db),
                 _check_clickhouse(),
                 _check_valkey(),
+                _check_iggy(),
             ),
             asyncio.gather(
                 _check_service(

{imbi_api-2.34.0 → imbi_api-2.35.0}/src/imbi/api/endpoints/operations_log.py

@@ -12,6 +12,7 @@ are hidden.
 
 from __future__ import annotations
 
+import asyncio
 import datetime
 import logging
 import typing
@@ -29,7 +30,7 @@ from imbi.api.endpoints._pagination import (
     encode_cursor,
     parse_iso,
 )
-from imbi.common import clickhouse, models
+from imbi.common import clickhouse, iggy, models
 from imbi.common import patch as json_patch
 from imbi.common.plugins import OpsLogTemplate
 from imbi.common.plugins.registry import list_plugins
@@ -147,18 +148,19 @@ def _model_to_row(entry: models.OperationLog) -> dict[str, typing.Any]:
     return dumped
 
 
-async def _insert_row(row: dict[str, typing.Any]) -> None:
-    """Insert a single row into operations_log.
+#: One poll interval of the ClickHouse sink. A row published moments
+#: ago is not readable until the sink has drained it, so a read that
+#: expects it waits this long once before concluding it is absent.
+SINK_POLL_INTERVAL_SECONDS: typing.Final[float] = 0.25
 
-    Calls the class method directly with explicit column names/values
-    because the module-level ``clickhouse.insert`` wrapper only accepts
-    pydantic models and loses the alias during serialization.
+
+async def _insert_row(row: dict[str, typing.Any], topic: str = 'api') -> None:
+    """Publish a single ``operations_log`` row to the Iggy stream.
+
+    The row is a column-name mapping, so it goes through
+    ``publish_rows``; the ``_row_version`` alias is already rendered.
     """
-    await clickhouse.client.Clickhouse.get_instance().insert(
-        'operations_log',
-        [list(row.values())],
-        list(row.keys()),
-    )
+    await iggy.publish_rows('operations_log', topic, [row])
 
 
 _BY_EXTERNAL_RUN_SQL: typing.Final[str] = (
@@ -176,14 +178,24 @@ async def complete_opslog_entry(
     """Mark the opslog entry for ``external_run_id`` as completed.
 
     Looks up the latest in-progress row by ``external_run_id``, then
-    re-inserts it with ``completed_at`` set and a bumped ``_row_version``
+    re-publishes it with ``completed_at`` set and a bumped ``_row_version``
     so ReplacingMergeTree converges to the completed state.  Returns
     ``True`` when an entry was found and written, ``False`` otherwise.
     Silently no-ops when the entry is already completed or missing.
+
+    The open row reaches ClickHouse through the sink, so when the lookup
+    finds nothing it is retried once after one poll interval before the
+    entry is treated as missing. The start and the completion are
+    normally seconds to minutes apart, so the retry is a safety net.
     """
     rows = await clickhouse.client.Clickhouse.get_instance().query(
         _BY_EXTERNAL_RUN_SQL, {'run_id': external_run_id}
     )
+    if not rows:
+        await asyncio.sleep(SINK_POLL_INTERVAL_SECONDS)
+        rows = await clickhouse.client.Clickhouse.get_instance().query(
+            _BY_EXTERNAL_RUN_SQL, {'run_id': external_run_id}
+        )
     if not rows:
         LOGGER.debug(
             'complete_opslog_entry: no open entry for external_run_id=%s',
@@ -193,7 +205,7 @@ async def complete_opslog_entry(
     row = dict(rows[0])
     row['completed_at'] = completed_at
     row['_row_version'] = next_row_version(int(row['_row_version']))
-    await _insert_row(row)
+    await _insert_row(row, 'deployments')
     return True
 
 

{imbi_api-2.34.0 → imbi_api-2.35.0}/src/imbi/api/endpoints/project_configuration.py

@@ -18,7 +18,7 @@ from imbi.api.endpoints._helpers import (
 from imbi.api.identity.host_integration import call_with_identity_retry
 from imbi.api.plugins import call_with_timeout
 from imbi.api.plugins.resolution import ResolvedCapability, resolve_capability
-from imbi.common import clickhouse, graph, valkey
+from imbi.common import graph, iggy, valkey
 from imbi.common import models as common_models
 from imbi.common.plugins import decrypt_integration_credentials
 from imbi.common.plugins.base import (
@@ -412,8 +412,4 @@ async def _write_audit(
     )
     row = entry.model_dump(by_alias=True, mode='python')
     row['is_deleted'] = 1 if entry.is_deleted else 0
-    await clickhouse.client.Clickhouse.get_instance().insert(
-        'operations_log',
-        [list(row.values())],
-        list(row.keys()),
-    )
+    await iggy.publish_rows('operations_log', 'configuration', [row])