synaptiq 2.0.2__tar.gz → 2.0.3__tar.gz

{synaptiq-2.0.2 → synaptiq-2.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synaptiq
-Version: 2.0.2
+Version: 2.0.3
 Summary: Graph-powered code intelligence engine — indexes codebases into a knowledge graph, exposed via MCP tools for AI agents and a CLI for developers.
 Keywords: code-intelligence,knowledge-graph,mcp,tree-sitter,static-analysis,dead-code,claude-code
 Author: Stevica Canadi

{synaptiq-2.0.2 → synaptiq-2.0.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "synaptiq"
-version = "2.0.2"
+version = "2.0.3"
 description = "Graph-powered code intelligence engine — indexes codebases into a knowledge graph, exposed via MCP tools for AI agents and a CLI for developers."
 readme = "README.md"
 license = "MIT"

{synaptiq-2.0.2 → synaptiq-2.0.3}/src/synaptiq/__init__.py

@@ -1,3 +1,3 @@
 """Synaptiq — Graph-powered code intelligence engine."""
 
-__version__ = "2.0.2"
+__version__ = "2.0.3"

{synaptiq-2.0.2 → synaptiq-2.0.3}/src/synaptiq/core/embeddings/lazy_worker.py

@@ -342,6 +342,7 @@ def _encode_and_store(data_dir: Path, db_path: Path, started_at: str) -> None:
     from synaptiq.core.embeddings.embedder import (
         embed_graph,
         embeddable_node_count,
+        partition_embeddings,
         tier_from_meta,
     )
 
@@ -369,16 +370,53 @@ def _encode_and_store(data_dir: Path, db_path: Path, started_at: str) -> None:
             return
         graph, previous = loaded
 
-        total = embeddable_node_count(graph)
-        if total == 0:
+        if embeddable_node_count(graph) == 0:
             _update_meta_embeddings(data_dir, 0, expect_anchor=anchor)
             write_state(data_dir, "complete", done=0, total=0, started_at=started_at)
             return
 
-        write_state(data_dir, "encoding", done=0, total=total, started_at=started_at)
+        # Only the PENDING delta is actually encoded — embed_graph reuses every
+        # unchanged vector and runs the model on just the new/changed symbols. So
+        # report progress against the pending count, not the full embeddable-node
+        # count: a one-file change must read "encoding 3/12", never
+        # "encoding 0/26,909". partition_embeddings runs the same split
+        # embed_graph does but never loads the model (generate-text cost), so this
+        # is cheap next to the encode it precedes.
+        reused_vecs, pending = partition_embeddings(graph, previous, tier=tier.name)
+        reused = len(reused_vecs)
+        if pending == 0:
+            # Nothing new to encode (everything reused). Still store the reused
+            # set so meta reflects the true vector count, but there is no work to
+            # show progress for.
+            embeddings = reused_vecs
+            if _read_meta_timestamp(data_dir) != anchor:
+                continue
+            if _store_with_retry(db_path, embeddings):
+                _update_meta_embeddings(data_dir, len(embeddings), expect_anchor=anchor)
+                write_state(data_dir, "complete", done=0, total=0, started_at=started_at)
+            else:
+                write_state(
+                    data_dir,
+                    "deferred",
+                    done=0,
+                    total=0,
+                    started_at=started_at,
+                    detail="index locked; re-run `synaptiq analyze` to encode",
+                )
+            return
 
-        def _on_progress(done: int, tot: int) -> None:
-            write_state(data_dir, "encoding", done=done, total=tot, started_at=started_at)
+        write_state(data_dir, "encoding", done=0, total=pending, started_at=started_at)
+
+        def _on_progress(done: int, _full: int) -> None:
+            # embed_graph reports done = reused + encoded-so-far against the full
+            # embeddable count; re-base onto the pending delta for honest status.
+            write_state(
+                data_dir,
+                "encoding",
+                done=max(0, min(done - reused, pending)),
+                total=pending,
+                started_at=started_at,
+            )
 
         embeddings = embed_graph(
             graph, tier=tier.name, previous=previous, progress_callback=_on_progress
@@ -393,16 +431,18 @@ def _encode_and_store(data_dir: Path, db_path: Path, started_at: str) -> None:
             )
             continue
 
-        # (f) Store under the single-writer lock (retry → deferred).
+        # (f) Store under the single-writer lock (retry → deferred). Progress is
+        # reported against the pending delta (see above), so a completed encode
+        # reads "pending/pending".
         if _store_with_retry(db_path, embeddings):
             _update_meta_embeddings(data_dir, len(embeddings), expect_anchor=anchor)
-            write_state(data_dir, "complete", done=total, total=total, started_at=started_at)
+            write_state(data_dir, "complete", done=pending, total=pending, started_at=started_at)
         else:
             write_state(
                 data_dir,
                 "deferred",
-                done=total,
-                total=total,
+                done=pending,
+                total=pending,
                 started_at=started_at,
                 detail="index locked; re-run `synaptiq analyze` to encode",
             )

{synaptiq-2.0.2 → synaptiq-2.0.3}/src/synaptiq/core/ingestion/manifest.py

@@ -29,6 +29,7 @@ from __future__ import annotations
 
 import hashlib
 import json
+import logging
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from typing import Any
@@ -36,6 +37,8 @@ from typing import Any
 from synaptiq.core.graph.graph import KnowledgeGraph
 from synaptiq.core.graph.model import GraphNode, NodeLabel, RelType
 
+logger = logging.getLogger(__name__)
+
 # ---------------------------------------------------------------------------
 # Versioning
 # ---------------------------------------------------------------------------
@@ -494,9 +497,25 @@ def _parse_file_manifest_row(row: list[Any]) -> FileManifest | None:
         path = row[0]
         content = row[1] or ""
         language = row[2] or ""
-        symbol_ids = json.loads(row[3]) if row[3] else []
-        symbol_sigs = json.loads(row[4]) if row[4] else {}
-        raw_edges = json.loads(row[5]) if row[5] else []
+        # ``symbol_ids`` / ``symbol_sigs`` / ``out_edges`` are ALWAYS serialized
+        # as non-empty JSON (``"[]"`` / ``"{}"``; see serialize_file_manifest), so
+        # a NULL/empty value in any of them is not "no symbols" — it means the
+        # storage layer silently DROPPED the field (the ladybug bug where large
+        # string columns are corrupted when written into a node table that
+        # previously had rows DETACH DELETEd, which 2.0.3 fixes at the source).
+        # Treating that as empty provenance is the worst outcome: invisible
+        # incremental blindness. So fail LOUD — a dropped field poisons the row
+        # (caller -> None -> full rebuild), never silent partial provenance.
+        if not row[3] or not row[4] or not row[5]:
+            logger.warning(
+                "manifest row for %r has empty symbol provenance (dropped by "
+                "storage); treating the whole manifest as corrupt -> full rebuild",
+                path,
+            )
+            return None
+        symbol_ids = json.loads(row[3])
+        symbol_sigs = json.loads(row[4])
+        raw_edges = json.loads(row[5])
         # ``unresolved_imports`` was added after the first manifest schema; a row
         # written before it (or a NULL from CSV COPY) has fewer than 7 cols —
         # tolerate that with an empty list rather than treating it as corrupt.
@@ -600,6 +619,12 @@ def load_manifest_from_rows(
     for row in file_rows:
         fm = _parse_file_manifest_row(row)
         if fm is None:
+            path = row[0] if row else "<unknown>"
+            logger.warning(
+                "corrupt manifest file row (%r); manifest is untrustworthy -> "
+                "forcing a full rebuild",
+                path,
+            )
             return None
         files[fm.path] = fm
     return Manifest(index=index, files=files)

{synaptiq-2.0.2 → synaptiq-2.0.3}/src/synaptiq/core/storage/ladybug_backend.py

@@ -48,6 +48,7 @@ except ImportError:  # pragma: no cover - exercised in CSV-only environments
 from synaptiq.core.graph.graph import KnowledgeGraph
 from synaptiq.core.graph.model import GraphNode, GraphRelationship, NodeLabel
 from synaptiq.core.ingestion.manifest import (
+    FILE_MANIFEST_COLUMNS,
     Manifest,
     build_manifest,
     load_manifest_from_rows,
@@ -2290,21 +2291,87 @@ class LadybugBackend:
         except Exception:
             logger.debug("manifest table creation skipped", exc_info=True)
 
+    def _reset_manifest_file_table(self) -> None:
+        """DROP + re-CREATE the FileManifest table for FRESH string storage.
+
+        The crux of the 2.0.3 manifest-persistence fix. LadybugDB silently drops
+        the values of large STRING columns when rows are written into a node
+        table that previously had rows removed via ``DETACH DELETE`` — verified on
+        a real 3,563-file repo where the SECOND manifest write lost 533 rows /
+        5,397 symbol ids, size-correlated (the files with the most symbols first),
+        *identically* through CSV COPY, in-memory Arrow COPY, and parameterized
+        MERGE. The corruption lives in the table's reused string storage, not in
+        any one writer; only never-deleted-from storage round-trips losslessly.
+        ``bulk_load`` stamps the first manifest into a pristine ``.rebuild`` table
+        (always fine); the losses were in the SECOND write onward — the
+        ``stamp_full_manifest`` after coupling and every incremental
+        consolidation reused the populated live table. Dropping and recreating
+        gives every write a pristine table, which round-trips losslessly.
+        """
+        assert self._conn is not None
+        try:
+            self._conn.execute(f"DROP TABLE IF EXISTS {_MANIFEST_FILE_TABLE}")
+        except Exception:
+            logger.debug("manifest file-table drop skipped", exc_info=True)
+        self._conn.execute(_FILE_MANIFEST_DDL)
+
+    def _reset_manifest_index_table(self) -> None:
+        """DROP + re-CREATE the IndexManifest table (drops the prior singleton)."""
+        assert self._conn is not None
+        try:
+            self._conn.execute(f"DROP TABLE IF EXISTS {_MANIFEST_INDEX_TABLE}")
+        except Exception:
+            logger.debug("manifest index-table drop skipped", exc_info=True)
+        self._conn.execute(_INDEX_MANIFEST_DDL)
+
+    def _refresh_write_conn(self) -> None:
+        """Recreate the write connection to drop LadybugDB's stale caches after DDL.
+
+        LadybugDB caches prepared statements (keyed by query string) and table
+        catalog entries per connection, and does NOT invalidate them when a table
+        is ``DROP``ped and re-``CREATE``d underneath. Re-running a parameterized
+        statement (the index-singleton ``CREATE``, the ``CREATE`` row fallback) or
+        a ``MATCH``-based query against a just-reset table on the SAME connection
+        then crashes with ``unordered_map::at: key not found`` /
+        ``Cannot find table catalog entry``. Manifest writes reuse one write
+        connection across incremental consolidations and the daemon's lifetime, so
+        after resetting the manifest tables we swap in a fresh connection whose
+        caches are empty — the light, targeted analogue of what the ``bulk_load``
+        swap gets for free by calling :meth:`initialize`. The DB handle is
+        untouched (cheap); pending manifest DDL already auto-committed, so nothing
+        is lost. ``self._prepared`` (this backend's own per-connection cache) is
+        cleared for the same reason it is in :meth:`initialize`.
+        """
+        assert self._db is not None
+        old = self._conn
+        self._conn = ladybug.Connection(self._db)
+        self._prepared = {}
+        if old is not None:
+            try:
+                old.close()
+            except Exception:
+                logger.debug("closing prior write connection failed", exc_info=True)
+
     def write_manifest(self, manifest: Manifest) -> None:
         """Persist *manifest* into the current DB, replacing any prior manifest.
 
-        Full replace: clears both manifest tables, bulk-writes the per-file rows,
-        then writes the index singleton **last** as a completion marker (a
-        partial/crashed write leaves no singleton, so :meth:`read_manifest`
-        reports "no manifest" and the caller does a full rebuild). Not wrapped in
-        an explicit transaction — it mirrors the COPY-based bulk path and is
-        crash-safe by riding :meth:`bulk_load`'s ``.rebuild`` swap. Uses the
+        Full replace via DROP + re-CREATE of both manifest tables (see
+        :meth:`_reset_manifest_file_table` for why fresh storage is mandatory),
+        then bulk-writes the per-file rows, then writes the index singleton
+        **last** as a completion marker: resetting the index table removes the
+        prior singleton, so throughout the rewrite there is no completion marker —
+        a partial/crashed write leaves none, :meth:`read_manifest` reports "no
+        manifest", and the caller does a full rebuild. Not wrapped in an explicit
+        transaction (DDL auto-commits) — crash-safety rides the singleton-last
+        marker and, for the ``bulk_load`` write, the ``.rebuild`` swap. Uses the
         write connection, so callers serialize via the external lock.
         """
         assert self._conn is not None
-        self._create_manifest_tables()
-        self._conn.execute(f"MATCH (m:{_MANIFEST_FILE_TABLE}) DETACH DELETE m")
-        self._conn.execute(f"MATCH (im:{_MANIFEST_INDEX_TABLE}) DETACH DELETE im")
+        self._reset_manifest_file_table()
+        self._reset_manifest_index_table()
+        # Drop LadybugDB's per-connection caches staled by the resets above,
+        # before any cached/parameterized DML runs on this connection.
+        self._refresh_write_conn()
         rows = [serialize_file_manifest(fm) for fm in manifest.files.values()]
         if rows:
             self._write_manifest_file_rows(rows)
@@ -2315,22 +2382,60 @@ class LadybugBackend:
         )
 
     def _write_manifest_file_rows(self, rows: list[list[Any]]) -> None:
-        """Insert FileManifest rows via CSV COPY, falling back to row MERGE.
-
-        Mirrors the node bulk path: COPY is fast at repo scale, and the MERGE
-        fallback keeps the write working in COPY-unavailable environments.
+        """Bulk-write FileManifest rows into the freshly-reset (empty) table.
+
+        The table is pristine (``write_manifest`` reset it), so this only ever
+        INSERTs. Prefers the in-memory Arrow COPY (pyarrow is a core dep; native
+        arbitrary-size strings, no temp file), falls back to CSV COPY, then to
+        row-by-row ``CREATE`` for COPY-unavailable environments. ``CREATE`` — not
+        ``MERGE`` — because LadybugDB raises a stale-catalog error on a
+        MATCH-based ``MERGE`` executed against a just-DROP+CREATEd table, and a
+        fresh empty table never needs upsert semantics. Each fallback first
+        re-resets the file table so a COPY that failed after partial progress
+        cannot leave duplicate/torn rows behind.
         """
         assert self._conn is not None
+        if _HAS_PYARROW:
+            try:
+                self._write_manifest_file_rows_arrow(rows)
+                return
+            except Exception:
+                logger.debug("manifest Arrow COPY failed; trying CSV COPY", exc_info=True)
+                self._reset_manifest_file_table()
+                self._refresh_write_conn()
         try:
             self._csv_copy(_MANIFEST_FILE_TABLE, rows)
             return
         except Exception:
-            logger.debug("manifest CSV COPY failed; falling back to row inserts", exc_info=True)
+            logger.debug("manifest CSV COPY failed; falling back to row CREATE", exc_info=True)
+            self._reset_manifest_file_table()
+            self._refresh_write_conn()
+        self._write_manifest_file_rows_create(rows)
+
+    def _write_manifest_file_rows_arrow(self, rows: list[list[Any]]) -> None:
+        """COPY FileManifest rows from an in-memory pyarrow table (native strings).
+
+        Columns are built in :data:`FILE_MANIFEST_COLUMNS` order — contractual
+        with the DDL and the read query. Values are stored verbatim (no
+        empty→NULL coercion): the JSON columns are never empty, and preserving
+        them exactly keeps the reader's loud empty-provenance gate meaningful.
+        """
+        assert self._conn is not None
+        arrow_tbl = pa.table(
+            {
+                name: pa.array([row[i] for row in rows], type=pa.string())
+                for i, name in enumerate(FILE_MANIFEST_COLUMNS)
+            }
+        )
+        self._arrow_copy(_MANIFEST_FILE_TABLE, arrow_tbl)
+
+    def _write_manifest_file_rows_create(self, rows: list[list[Any]]) -> None:
+        """Row-by-row ``CREATE`` fallback for COPY-unavailable environments."""
+        assert self._conn is not None
         stmt = (
-            f"MERGE (m:{_MANIFEST_FILE_TABLE} {{path: $path}}) "
-            "SET m.content_sha = $content_sha, m.language = $language, "
-            "m.symbol_ids = $symbol_ids, m.symbol_sigs = $symbol_sigs, "
-            "m.out_edges = $out_edges, m.unresolved_imports = $unresolved_imports"
+            f"CREATE (m:{_MANIFEST_FILE_TABLE} {{path: $path, content_sha: $content_sha, "
+            "language: $language, symbol_ids: $symbol_ids, symbol_sigs: $symbol_sigs, "
+            "out_edges: $out_edges, unresolved_imports: $unresolved_imports}})"
         )
         for row in rows:
             self._conn.execute(