iris-vector-graph 2.3.0__tar.gz → 2.4.1__tar.gz

{iris_vector_graph-2.3.0 → iris_vector_graph-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iris-vector-graph
-Version: 2.3.0
+Version: 2.4.1
 Summary: Transactional Graph + Vector retrieval system for InterSystems IRIS with hybrid search, openCypher, and GraphQL APIs
 Project-URL: Homepage, https://github.com/intersystems-community/iris-vector-graph
 Project-URL: Documentation, https://github.com/intersystems-community/iris-vector-graph/tree/main/docs
@@ -25,6 +25,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Requires-Python: >=3.10
 Requires-Dist: intersystems-iris>=1.0.0
+Requires-Dist: iris-embedded-python-wrapper>=0.5.20
 Provides-Extra: biodata
 Requires-Dist: biopython>=1.81; extra == 'biodata'
 Requires-Dist: bioservices>=1.11.0; extra == 'biodata'

{iris_vector_graph-2.3.0 → iris_vector_graph-2.4.1}/iris_src/src/Graph/KG/EdgeScan.cls

@@ -1,3 +1,27 @@
+/// Graph.KG.EdgeScan — adjacency-global maintenance + scan procedures.
+///
+/// ============================================================================
+/// ^KG ADJACENCY LAYOUT OF RECORD (canonical — do not deviate)
+/// ============================================================================
+/// The production adjacency layout is SHARD-PREFIXED:
+///
+///     ^KG("out", 0, s, p, o) = weight     outbound edges
+///     ^KG("in",  0, o, p, s) = weight     inbound edges
+///     ^KG("deg", s)          = out-degree
+///     ^KG("degp", s, p)      = per-predicate out-degree
+///
+/// Every live reader scans this layout: MatchEdges (here), Graph.KG.Algorithms,
+/// Graph.KG.Centrality, Graph.KG.Subgraph, and the NKG builders. WriteAdjacency
+/// (below), TraversalBuild.BuildKG, TemporalIndex.InsertEdge, and the (currently
+/// undeployed) Graph.KG.GraphIndex functional index all WRITE this exact form.
+///
+/// There is a LEGACY no-shard layout — ^KG("out", s, p, o) — still written by
+/// the bench harness (Graph.KG.BenchSeeder / BenchFormat) and the standalone CSV
+/// Graph.KG.Loader, and read by dead Else-fallback branches in NKGAccelTraversal
+/// / NKGAccelAdjacency. Nothing in the production write/read path uses it.
+/// DO NOT add new readers or writers of the no-shard form. If you need adjacency,
+/// use ^KG("out", 0, ...). See docs/index-maintenance-architecture-review.md.
+/// ============================================================================
 Class Graph.KG.EdgeScan Extends %RegisteredObject
 {
 

iris_vector_graph-2.4.1/iris_src/src/Graph/KG/EmbedQueue.cls

@@ -0,0 +1,210 @@
+Class Graph.KG.EmbedQueue Extends %RegisteredObject
+{
+
+/// Enhanced embedding queue (spec 199). Entries are subscripted:
+///   ^EmbedQueue(reqId, "status")    = "PENDING" | "DONE" | "ERROR"
+///   ^EmbedQueue(reqId, "text")      = text to embed
+///   ^EmbedQueue(reqId, "node_id")   = node id (node-keyed entries) or ""
+///   ^EmbedQueue(reqId, "result")    = "<f1>,<f2>,...,<fN>" comma-joined vector (on DONE)
+///   ^EmbedQueue(reqId, "error")     = message (on ERROR)
+///   ^EmbedQueue(reqId, "queued_at") = $ZDateTime timestamp
+///
+/// reqId = node_id (node-keyed enqueue; re-enqueue overwrites — one entry per node)
+///       or "txt:<n>" (free-text enqueue; always-new, never deduplicated).
+///
+/// Single-processor: ClaimPendingBatch is a plain ordered scan (no atomic claim state).
+/// Statuses are written UPPERCASE; reads tolerate legacy lowercase pending/done/error.
+/// The actual batched embedding is performed Python-side (process_embed_queue), which
+/// claims a batch, encodes all texts in one model call, and writes results via SetResult.
+
+/// Normalize a stored status value to UPPERCASE canonical form (back-compat).
+ClassMethod NormStatus(s As %String) As %String [ Private ]
+{
+    Quit $ZConvert(s, "U")
+}
+
+/// Node-keyed enqueue (one entry per node id; overwrite). The text to embed defaults
+/// to the node id; pass an explicit ``text`` to embed something else for the node.
+ClassMethod Enqueue(nodeId As %String, embeddingConfig As %String = "", text As %String = "") As %Status
+{
+    If text = "" { Set text = nodeId }
+    Set ^EmbedQueue(nodeId, "status") = "PENDING"
+    Set ^EmbedQueue(nodeId, "text") = text
+    Set ^EmbedQueue(nodeId, "node_id") = nodeId
+    Set ^EmbedQueue(nodeId, "result") = ""
+    Set ^EmbedQueue(nodeId, "error") = ""
+    Set ^EmbedQueue(nodeId, "config") = embeddingConfig
+    Set ^EmbedQueue(nodeId, "queued_at") = $ZDateTime($Horolog, 3)
+    Quit $$$OK
+}
+
+/// nodeIds may be a JSON array string (as sent by the Python client) or a
+/// %DynamicArray. Returns count enqueued.
+ClassMethod BulkEnqueue(nodeIds As %String, embeddingConfig As %String = "") As %Integer
+{
+    Set arr = nodeIds
+    If '$IsObject(arr) { Set arr = ##class(%DynamicArray).%FromJSON(nodeIds) }
+    Set count = 0
+    Set iter = arr.%GetIterator()
+    While iter.%GetNext(.idx, .nodeId) {
+        Set sc = ..Enqueue(nodeId, embeddingConfig)
+        If $$$ISOK(sc) { Set count = count + 1 }
+    }
+    Quit count
+}
+
+/// Free-text enqueue (always-new entry; reqId = "txt:<seq>"). texts may be a JSON
+/// array string (Python client) or a %DynamicArray. Returns count enqueued.
+ClassMethod BulkEnqueueText(texts As %String, embeddingConfig As %String = "") As %Integer
+{
+    Set arr = texts
+    If '$IsObject(arr) { Set arr = ##class(%DynamicArray).%FromJSON(texts) }
+    Set count = 0
+    Set iter = arr.%GetIterator()
+    While iter.%GetNext(.idx, .text) {
+        Continue:text=""
+        Set seq = $Increment(^EmbedQueue("$seq"))
+        Set reqId = "txt:" _ seq
+        Set ^EmbedQueue(reqId, "status") = "PENDING"
+        Set ^EmbedQueue(reqId, "text") = text
+        Set ^EmbedQueue(reqId, "node_id") = ""
+        Set ^EmbedQueue(reqId, "result") = ""
+        Set ^EmbedQueue(reqId, "error") = ""
+        Set ^EmbedQueue(reqId, "config") = embeddingConfig
+        Set ^EmbedQueue(reqId, "queued_at") = $ZDateTime($Horolog, 3)
+        Set count = count + 1
+    }
+    Quit count
+}
+
+/// Return up to batchSize PENDING entries as JSON [{reqId,text,node_id}, ...].
+/// Plain ordered scan (single-processor; no claim state).
+ClassMethod ClaimPendingBatch(batchSize As %Integer = 100) As %String
+{
+    Set arr = []
+    Set reqId = ""
+    Set n = 0
+    For {
+        Set reqId = $Order(^EmbedQueue(reqId))
+        Quit:reqId=""
+        Continue:reqId="$seq"
+        Set st = ..NormStatus($Get(^EmbedQueue(reqId, "status")))
+        Continue:st'="PENDING"
+        Set obj = {}
+        Set obj.reqId = reqId
+        Set obj.text = $Get(^EmbedQueue(reqId, "text"))
+        Set obj."node_id" = $Get(^EmbedQueue(reqId, "node_id"))
+        Do arr.%Push(obj)
+        Set n = n + 1
+        Quit:n>=batchSize
+    }
+    Quit arr.%ToJSON()
+}
+
+/// Set an entry's result. status="DONE" → payload is the result vector string;
+/// status="ERROR" → payload is the error message.
+ClassMethod SetResult(reqId As %String, status As %String, payload As %String = "") As %Status
+{
+    If '$Data(^EmbedQueue(reqId, "status")) { Quit $$$ERROR($$$GeneralError, "unknown reqId") }
+    Set up = $ZConvert(status, "U")
+    Set ^EmbedQueue(reqId, "status") = up
+    If up = "DONE" {
+        Set ^EmbedQueue(reqId, "result") = payload
+        Set ^EmbedQueue(reqId, "error") = ""
+    } ElseIf up = "ERROR" {
+        Set ^EmbedQueue(reqId, "error") = payload
+    }
+    Quit $$$OK
+}
+
+ClassMethod PendingCount() As %Integer
+{
+    Set count = 0
+    Set reqId = ""
+    For {
+        Set reqId = $Order(^EmbedQueue(reqId))
+        Quit:reqId=""
+        Continue:reqId="$seq"
+        If ..NormStatus($Get(^EmbedQueue(reqId, "status"))) = "PENDING" { Set count = count + 1 }
+    }
+    Quit count
+}
+
+/// Remove DONE entries; leave PENDING and ERROR intact. Returns number cleared.
+ClassMethod ClearDone() As %Integer
+{
+    Set cleared = 0
+    Set reqId = ""
+    For {
+        Set reqId = $Order(^EmbedQueue(reqId))
+        Quit:reqId=""
+        Continue:reqId="$seq"
+        If ..NormStatus($Get(^EmbedQueue(reqId, "status"))) = "DONE" {
+            Kill ^EmbedQueue(reqId)
+            Set cleared = cleared + 1
+        }
+    }
+    Quit cleared
+}
+
+/// Legacy native-EMBEDDING() per-row processing path, retained for callers that do not
+/// use the Python batched path. Reads the subscripted schema.
+ClassMethod ProcessBatch(batchSize As %Integer = 100, timeout As %Integer = 30) As %String
+{
+    Set processed = 0
+    Set errors = 0
+    Set reqId = ""
+    For {
+        Set reqId = $Order(^EmbedQueue(reqId))
+        Quit:reqId=""
+        Continue:reqId="$seq"
+        Quit:processed>=batchSize
+        Continue:..NormStatus($Get(^EmbedQueue(reqId, "status")))'="PENDING"
+
+        Set nodeId = $Get(^EmbedQueue(reqId, "node_id"))
+        Set embConfig = $Get(^EmbedQueue(reqId, "config"))
+        Set text = $Get(^EmbedQueue(reqId, "text"))
+        Try {
+            Set sql = "SELECT EMBEDDING(?, ?) FROM (SELECT 1)"
+            Set stmt = ##class(%SQL.Statement).%New()
+            Set sc = stmt.%Prepare(sql)
+            If $$$ISOK(sc) {
+                Set rs = stmt.%Execute(text, embConfig)
+                If rs.%SQLCODE = 0 && rs.%Next() {
+                    Set embVal = rs.%GetData(1)
+                    If (embVal '= "") && (nodeId '= "") {
+                        Set insertSql = "INSERT OR REPLACE INTO Graph_KG.kg_NodeEmbeddings (id, emb) VALUES (?, TO_VECTOR(?, DOUBLE))"
+                        Set stmt2 = ##class(%SQL.Statement).%New()
+                        Do stmt2.%Prepare(insertSql)
+                        Do stmt2.%Execute(nodeId, embVal)
+                    }
+                    If embVal '= "" { Set ^EmbedQueue(reqId, "result") = embVal }
+                }
+            }
+            Set ^EmbedQueue(reqId, "status") = "DONE"
+            Set processed = processed + 1
+        } Catch ex {
+            Set ^EmbedQueue(reqId, "status") = "ERROR"
+            Set ^EmbedQueue(reqId, "error") = ex.DisplayString()
+            Set errors = errors + 1
+        }
+    }
+    Set result = {"processed": (processed), "errors": (errors)}
+    Quit result.%ToJSON()
+}
+
+/// Start a background IRIS Task to process the queue.
+ClassMethod StartBackgroundTask(batchSize As %Integer = 100) As %String
+{
+    Set task = ##class(%SYSTEM.Task).%New()
+    Set task.Name = "IVG EmbedQueue Worker"
+    Set task.NameSpace = $ZNamespace
+    Set task.TaskClass = "Graph.KG.EmbedQueueTask"
+    Set sc = task.%Save()
+    If $$$ISERR(sc) { Quit "error: "_$System.Status.GetErrorText(sc) }
+    Set taskId = task.%Id()
+    Do ##class(%SYSTEM.Task).RunNow(taskId)
+    Quit taskId
+}
+
+}

{iris_vector_graph-2.3.0 → iris_vector_graph-2.4.1}/iris_src/src/Graph/KG/GraphIndex.cls

@@ -62,8 +62,12 @@ ClassMethod InsertIndex(pID As %String, s As %Binary, p As %Binary, o As %Binary
 {
 	Set weight = ##class(Graph.KG.GraphIndex).ExtractWeight(qualifiers)
 
-	Set ^KG("out", s, p, o) = weight
-	Set ^KG("in", o, p, s) = weight
+	// Shard-0 layout — MUST match EdgeScan.WriteAdjacency / TraversalBuild.BuildKG.
+	// Every live reader (MatchEdges, Algorithms, Centrality, NKG BFS) scans
+	// ^KG("out", 0, ...). Writing the legacy no-shard layout here would populate
+	// globals nothing reads. See Graph.KG.EdgeScan:WriteAdjacency for the canonical form.
+	Set ^KG("out", 0, s, p, o) = weight
+	Set ^KG("in", 0, o, p, s) = weight
 	Set tmp = $Increment(^KG("deg", s))
 	Set tmp = $Increment(^KG("degp", s, p))
 
@@ -147,8 +151,8 @@ ClassMethod UpdateIndex(pID As %String, s As %Binary, p As %Binary, o As %Binary
 
 ClassMethod DeleteIndex(pID As %String, s As %Binary, p As %Binary, o As %Binary, qualifiers As %Binary = "") As %Status [ ServerOnly = 1 ]
 {
-	Kill ^KG("out", s, p, o)
-	Kill ^KG("in", o, p, s)
+	Kill ^KG("out", 0, s, p, o)
+	Kill ^KG("in", 0, o, p, s)
 	Set tmp = $Increment(^KG("deg", s), -1)
 	Set tmp = $Increment(^KG("degp", s, p), -1)
 	If $Get(^KG("deg", s)) <= 0 Kill ^KG("deg", s)

{iris_vector_graph-2.3.0 → iris_vector_graph-2.4.1}/iris_src/src/Graph/KG/Loader.cls

@@ -1,4 +1,11 @@
-/// Fast CSV loader for Graph data
+/// Fast CSV loader for Graph data.
+///
+/// WARNING — LEGACY NO-SHARD LAYOUT. This loader writes ^KG("out", s, p, o)
+/// (no shard prefix), which is NOT the production layout. Live readers scan
+/// ^KG("out", 0, s, p, o) (see Graph.KG.EdgeScan for the layout of record), so
+/// data loaded here is invisible to BFS / centrality / Cypher. It has no callers
+/// in the engine. Prefer engine.bulk_load_session() / bulk_create_edges(). If you
+/// must use this, run engine.sync() afterward to rebuild the shard-0 globals.
 Class Graph.KG.Loader Extends %RegisteredObject
 {
 

{iris_vector_graph-2.3.0 → iris_vector_graph-2.4.1}/iris_src/src/Graph/KG/NKGAccelAdjacency.cls

@@ -1,3 +1,10 @@
+/// NKG-accelerated adjacency lookups over the interned ^NKG index.
+///
+/// NOTE: the ^KG fallback branches read the LEGACY no-shard
+/// ^KG("out", node, ...) layout, NOT the production shard-0 layout
+/// (^KG("out", 0, ...) — see Graph.KG.EdgeScan for the layout of record).
+/// They are dead in production (UseNKG() is true after any sync()). Fix them to
+/// the shard-0 layout before relying on them.
 Class Graph.KG.NKGAccelAdjacency Extends %RegisteredObject
 {
 

{iris_vector_graph-2.3.0 → iris_vector_graph-2.4.1}/iris_src/src/Graph/KG/NKGAccelTraversal.cls

@@ -1,3 +1,11 @@
+/// NKG-accelerated traversal (K-hop, BFS) over the interned ^NKG index.
+///
+/// NOTE: the `If useNKG { ^NKG... } Else { ^KG... }` fallback branches read the
+/// LEGACY no-shard ^KG("out", node, pred, dst) layout, NOT the production
+/// shard-0 layout (^KG("out", 0, ...) — see Graph.KG.EdgeScan for the layout of
+/// record). These Else branches are effectively dead in production: UseNKG()
+/// returns true whenever ^NKG("$meta") exists, which is true after any sync().
+/// If you ever make them reachable, fix them to scan the shard-0 layout first.
 Class Graph.KG.NKGAccelTraversal Extends %RegisteredObject
 {
 

{iris_vector_graph-2.3.0 → iris_vector_graph-2.4.1}/iris_vector_graph/_engine/admin.py

@@ -305,6 +305,99 @@ class AdminMixin:
                        "^NKG_populated": adjacency.nkg_populated} if internals else None,
         )
 
+    def verify_sync(self, heal: bool = False) -> "SyncReport":
+        """Check whether the ^KG/^NKG adjacency indexes agree with the SQL tables.
+
+        The authoritative graph state lives in ``Graph_KG.rdf_edges``. The ^KG/^NKG
+        globals are a derived acceleration index, maintained inline by
+        ``create_edge`` / ``bulk_*`` / ``WriteAdjacency`` but NOT by every write
+        path. BYPASS paths — ``drop_graph``, ``delete_node``, raw SQL inserts, the
+        ``map_sql_table`` bridge, or an interrupted bulk load — mutate the table
+        without touching the globals, leaving BFS / centrality / var-length Cypher
+        silently stale. This compares the SQL edge count against the ^NKG edge
+        count and reports the divergence.
+
+        Args:
+            heal: When True, run ``sync()`` to rebuild the globals if drift is found.
+
+        Returns:
+            SyncReport. ``report.in_sync`` is False when counts diverge OR the
+            in-process ``_nkg_dirty`` flag is set. ``bool(report)`` mirrors
+            ``in_sync`` for ``if not engine.verify_sync(): engine.sync()`` usage.
+        """
+        from iris_vector_graph.status import SyncReport
+
+        sql_edges = 0
+        global_edges = 0
+        global_nodes = 0
+        detail = None
+        indeterminate = False
+
+        cursor = self.conn.cursor()
+        try:
+            cursor.execute(f"SELECT COUNT(*) FROM {_table('rdf_edges')}")
+            row = cursor.fetchone()
+            sql_edges = int(row[0]) if row else 0
+        except Exception as e:
+            indeterminate = True
+            detail = f"SQL edge count failed: {str(e)[:120]}"
+
+        try:
+            iris_obj = self._iris_obj()
+            global_edges = int(iris_obj.classMethodValue("Graph.KG.Traversal", "NKGEdgeCount"))
+            global_nodes = int(iris_obj.classMethodValue("Graph.KG.Traversal", "NKGNodeCount"))
+        except Exception as e:
+            indeterminate = True
+            detail = f"NKG count failed (globals likely unbuilt): {str(e)[:120]}"
+
+        pending = bool(getattr(self, "_nkg_dirty", False))
+
+        # Drift oracle. ``_nkg_dirty`` is the authoritative in-process signal: a
+        # write path that bypassed adjacency maintenance set it. The count
+        # comparison is a SECONDARY heuristic and is deliberately one-directional:
+        # we only flag drift when SQL edges *exceed* indexed edges (the
+        # unambiguous "rows exist in the table but not in the index" case).
+        #
+        # We do NOT flag the reverse (globals > SQL), because ^NKG's meta
+        # edgeCount is known to over-count: InternNode/InternLabel are
+        # append-only (never decremented on delete) and NKG interning ignores
+        # graph_id while the SQL UNIQUE is on (s,p,o,graph_id). Treating
+        # globals>SQL as drift would produce false positives on any DB that has
+        # seen deletes. A full rebuild (sync()) is the only way to reconcile
+        # the meta counter, and verify_sync(heal=True) does exactly that.
+        sql_exceeds = sql_edges > global_edges
+        in_sync = (not sql_exceeds) and (not pending) and (not indeterminate)
+
+        if not in_sync and detail is None:
+            if pending and not sql_exceeds:
+                detail = "pending_sync flag set — writes occurred without sync()"
+            else:
+                detail = (
+                    f"drift: {sql_edges} SQL edges vs {global_edges} indexed edges. "
+                    "A write path bypassed adjacency maintenance — call engine.sync()."
+                )
+
+        report = SyncReport(
+            in_sync=in_sync,
+            sql_edges=sql_edges,
+            global_edges=global_edges,
+            global_nodes=global_nodes,
+            pending_sync=pending,
+            healed=False,
+            detail=detail,
+        )
+
+        if heal and not in_sync:
+            try:
+                self.sync()
+                report.healed = True
+                report.in_sync = True
+                report.detail = (detail or "") + " | healed via sync()"
+            except Exception as e:
+                report.detail = (detail or "") + f" | heal failed: {str(e)[:120]}"
+
+        return report
+
     def list_active_queries(self, limit: int = 50) -> List[Dict[str, Any]]:
         """Return active IRIS SQL queries.
 

{iris_vector_graph-2.3.0 → iris_vector_graph-2.4.1}/iris_vector_graph/_engine/embeddings.py

@@ -52,6 +52,10 @@ class EmbeddingsMixin:
                     _logging.getLogger("safetensors").setLevel(_logging.ERROR)
                 except Exception:
                     pass
+                # Local import avoids a circular import (engine.py imports this
+                # mixin at module load); function-local matches the pattern used
+                # elsewhere in this file (embed_selector, get_schema_prefix).
+                from iris_vector_graph.engine import _load_sentence_transformer, _is_sentence_transformer
                 self.embedder = _load_sentence_transformer("all-MiniLM-L6-v2")
                 logger.info("Auto-initialized SentenceTransformer('all-MiniLM-L6-v2')")
             except ImportError:
@@ -272,6 +276,7 @@ class EmbeddingsMixin:
     ) -> dict:
         from iris_vector_graph.embed_selector import EmbedSelector, build_node_where
         from iris_vector_graph.cypher import get_schema_prefix
+        from iris_vector_graph.engine import _load_sentence_transformer, _is_sentence_transformer
 
         sel = EmbedSelector(
             label=label,
@@ -436,6 +441,7 @@ class EmbeddingsMixin:
     ) -> dict:
         from iris_vector_graph.embed_selector import EmbedSelector, build_edge_where
         from iris_vector_graph.cypher import get_schema_prefix
+        from iris_vector_graph.engine import _load_sentence_transformer, _is_sentence_transformer
 
         sel = EmbedSelector(
             predicate=predicate,
@@ -655,45 +661,171 @@ class EmbeddingsMixin:
 
     def enqueue_for_embedding(
         self,
-        node_ids: List[str],
+        node_ids: Optional[List[str]] = None,
         embedding_config: str = "",
+        texts: Optional[List[str]] = None,
     ) -> int:
-        try:
-            import json as _json
-            ids_json = _json.dumps(node_ids)
-            result = self._call_classmethod(
-                "Graph.KG.EmbedQueue", "BulkEnqueue",
-                ids_json, embedding_config,
-            )
-            return int(str(result))
-        except Exception as e:
-            logger.warning("enqueue_for_embedding failed: %s", e)
-            return 0
+        """Enqueue embedding work onto the async queue. Returns the number enqueued.
+
+        Two modes (spec 199):
+          - ``node_ids``: node-keyed entries (re-enqueue overwrites — one entry per
+            node; backward compatible with the prior signature).
+          - ``texts``: free-text entries (always-new; never deduplicated).
+        Both may be supplied; an empty/None pair returns 0. Degrades gracefully
+        (returns 0 + warns) when the queue backend is unavailable.
+        """
+        from iris_vector_graph.schema import _call_classmethod
+        import json as _json
+        total = 0
+        if node_ids:
+            try:
+                ids_json = _json.dumps(node_ids)
+                result = _call_classmethod(
+                    self.conn, "Graph.KG.EmbedQueue", "BulkEnqueue",
+                    ids_json, embedding_config,
+                )
+                total += int(str(result))
+            except Exception as e:
+                logger.warning("enqueue_for_embedding (node_ids) failed: %s", e)
+        if texts:
+            try:
+                texts_json = _json.dumps(texts)
+                result = _call_classmethod(
+                    self.conn, "Graph.KG.EmbedQueue", "BulkEnqueueText",
+                    texts_json, embedding_config,
+                )
+                total += int(str(result))
+            except Exception as e:
+                logger.warning("enqueue_for_embedding (texts) failed: %s", e)
+        return total
 
 
     def process_embed_queue(self, batch_size: int = 100) -> dict:
+        """Process up to ``batch_size`` PENDING queue entries (spec 199).
+
+        Claims a batch of pending entries, embeds ALL of their texts in a SINGLE
+        embedder call (the batched performance win), writes each result back to the
+        queue entry, and — for node-keyed entries — upserts the vector into
+        ``kg_NodeEmbeddings`` so semantic search finds it. A per-entry embedding
+        failure marks only that entry ERROR; the rest of the batch still completes.
+        Returns ``{"processed": int, "errors": int}``. Degrades gracefully when the
+        backend is unavailable.
+        """
+        import json as _json
+        from iris_vector_graph.schema import _call_classmethod
         try:
-            import json as _json
-            result_json = str(self._call_classmethod(
-                "Graph.KG.EmbedQueue", "ProcessBatch",
-                batch_size, 30,
+            claim_json = str(_call_classmethod(
+                self.conn, "Graph.KG.EmbedQueue", "ClaimPendingBatch", batch_size,
             ))
-            return _json.loads(result_json)
+            entries = _json.loads(claim_json) if claim_json else []
         except Exception as e:
-            logger.warning("process_embed_queue failed: %s", e)
+            logger.warning("process_embed_queue claim failed: %s", e)
             return {"processed": 0, "errors": 0}
 
+        if not entries:
+            return {"processed": 0, "errors": 0}
+
+        texts = [e.get("text", "") for e in entries]
+        # Single batched embedder call (SC-002). If the whole call fails, fall back to
+        # per-entry so one poison text does not fail the entire batch (FR-007).
+        vectors = None
+        try:
+            vectors = self._encode_batch(texts)
+        except Exception as e:
+            logger.warning("batch encode failed, falling back per-entry: %s", e)
+
+        processed = 0
+        errors = 0
+        for idx, entry in enumerate(entries):
+            req_id = entry.get("reqId", "")
+            node_id = entry.get("node_id", "") or ""
+            try:
+                if vectors is not None:
+                    vec = vectors[idx]
+                else:
+                    vec = self._encode_batch([entry.get("text", "")])[0]
+                vec_str = ",".join(str(float(x)) for x in vec)
+                _call_classmethod(
+                    self.conn, "Graph.KG.EmbedQueue", "SetResult",
+                    req_id, "DONE", vec_str,
+                )
+                if node_id:
+                    self._upsert_node_embedding(node_id, vec)
+                processed += 1
+            except Exception as e:
+                errors += 1
+                try:
+                    _call_classmethod(
+                        self.conn, "Graph.KG.EmbedQueue", "SetResult",
+                        req_id, "ERROR", str(e)[:500],
+                    )
+                except Exception as se:
+                    logger.warning("SetResult ERROR failed for %s: %s", req_id, se)
+        return {"processed": processed, "errors": errors}
+
+    def _encode_batch(self, texts: List[str]) -> list:
+        """Embed a list of texts in one embedder call, resolving the engine embedder
+        (same auto-init path used by embed_text). Returns a list of vectors."""
+        if not self.embedder:
+            # Reuse embed_text's auto-init by embedding the first item, then encode all.
+            # embed_text resolves/sets self.embedder (or raises a clear RuntimeError).
+            if texts:
+                self.embed_text(texts[0])
+        embedder = self.embedder
+        if hasattr(embedder, "encode"):
+            result = embedder.encode(texts)
+            return [r.tolist() if hasattr(r, "tolist") else list(r) for r in result]
+        if hasattr(embedder, "embed"):
+            return [embedder.embed(t) for t in texts]
+        if callable(embedder):
+            return [embedder(t) for t in texts]
+        raise TypeError(f"Embedder {type(embedder)} has no encode/embed and is not callable")
+
+    def _upsert_node_embedding(self, node_id: str, vec) -> None:
+        """Write a node's embedding into kg_NodeEmbeddings so vector search finds it."""
+        emb_str = ",".join(str(float(x)) for x in vec)
+        cursor = self.conn.cursor()
+        try:
+            cursor.execute(
+                f"DELETE FROM {_table('kg_NodeEmbeddings')} WHERE id=?", [node_id]
+            )
+            cursor.execute(
+                f"INSERT INTO {_table('kg_NodeEmbeddings')} (id, emb) "
+                f"VALUES (?, TO_VECTOR('{emb_str}', {self.vector_dtype}))",
+                [node_id],
+            )
+            self.conn.commit()
+        finally:
+            try:
+                cursor.close()
+            except Exception:
+                pass
+
+    def clear_done(self) -> int:
+        """Remove completed (DONE) queue entries; leave PENDING/ERROR intact (spec 199,
+        FR-009). Returns the number cleared; 0 when the backend is unavailable."""
+        from iris_vector_graph.schema import _call_classmethod
+        try:
+            return int(str(_call_classmethod(
+                self.conn, "Graph.KG.EmbedQueue", "ClearDone",
+            )))
+        except Exception as e:
+            logger.warning("clear_done failed: %s", e)
+            return 0
+
 
     def embed_queue_pending(self) -> int:
         try:
-            return int(str(self._call_classmethod("Graph.KG.EmbedQueue", "PendingCount")))
+            from iris_vector_graph.schema import _call_classmethod
+            return int(str(_call_classmethod(self.conn, "Graph.KG.EmbedQueue", "PendingCount")))
         except Exception:
             return 0
 
 
     def start_background_embedding(self, batch_size: int = 100) -> str:
         try:
-            return str(self._call_classmethod("Graph.KG.EmbedQueue", "StartBackgroundTask", batch_size))
+            from iris_vector_graph.schema import _call_classmethod
+            return str(_call_classmethod(self.conn, "Graph.KG.EmbedQueue", "StartBackgroundTask", batch_size))
         except Exception as e:
             logger.warning("start_background_embedding failed: %s", e)
             return ""

{iris_vector_graph-2.3.0 → iris_vector_graph-2.4.1}/iris_vector_graph/_engine/fhir.py

@@ -74,6 +74,21 @@ class FhirMixin:
     def map_sql_table(
         self, table: str, id_column: str, label: str, property_columns=None
     ) -> dict:
+        """Map an existing SQL table to a graph label as a read-only projection.
+
+        Cypher queries against ``label`` are translated to JOINs on ``table`` at
+        query time — rows are never copied into rdf_edges/nodes.
+
+        ADJACENCY CONTRACT: because the rows are not materialized into
+        ``Graph_KG.rdf_edges``, edges reached through this mapping (or via
+        ``map_sql_relationship``) are NOT present in the ^KG/^NKG adjacency
+        globals. Property/pattern Cypher sees them; the global-backed algorithms
+        do NOT — BFS, centrality, var-length paths, and the NKG-accelerated
+        traversals will not traverse projected edges. There is no automatic
+        propagation and ``verify_sync()`` cannot see this gap (nothing was
+        written to rdf_edges). If you need these edges in graph algorithms,
+        materialize them (create_edge / bulk_create_edges) rather than mapping.
+        """
         from iris_vector_graph.security import sanitize_identifier
 
         sanitize_identifier(table)