iris-vector-graph 3.1.0__tar.gz → 3.2.0__tar.gz

{iris_vector_graph-3.1.0 → iris_vector_graph-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: iris-vector-graph
-Version: 3.1.0
+Version: 3.2.0
 Summary: Native graph database layer for InterSystems IRIS — openCypher, hybrid vector search, temporal property graph, RDF export, SHACL validation
 Project-URL: Homepage, https://github.com/intersystems-community/iris-vector-graph
 Project-URL: Documentation, https://github.com/intersystems-community/iris-vector-graph/tree/main/docs
@@ -28,6 +28,7 @@ Requires-Dist: intersystems-iris>=1.0.0
 Requires-Dist: iris-embedded-python-wrapper>=0.5.20
 Requires-Dist: numpy>=1.24.0
 Requires-Dist: pydantic>=2.0.0
+Requires-Dist: requests>=2.28.0
 Provides-Extra: ai
 Requires-Dist: iris-agentic-dev>=1.0; extra == 'ai'
 Provides-Extra: biodata

{iris_vector_graph-3.1.0 → iris_vector_graph-3.2.0}/iris_src/src/Graph/KG/EmbedQueue.cls

@@ -7,6 +7,8 @@ Class Graph.KG.EmbedQueue Extends %RegisteredObject
 ///   ^EmbedQueue(reqId, "node_id")   = node id (node-keyed entries) or ""
 ///   ^EmbedQueue(reqId, "result")    = "<f1>,<f2>,...,<fN>" comma-joined vector (on DONE)
 ///   ^EmbedQueue(reqId, "error")     = message (on ERROR)
+///   ^EmbedQueue(reqId, "config")    = embedding config the entry asked for ("" = the
+///                                     worker's own model, whatever that is)
 ///   ^EmbedQueue(reqId, "queued_at") = $ZDateTime timestamp
 ///
 /// reqId = node_id (node-keyed enqueue; re-enqueue overwrites — one entry per node)
@@ -77,8 +79,12 @@ ClassMethod BulkEnqueueText(texts As %String, embeddingConfig As %String = "") A
     Quit count
 }
 
-/// Return up to batchSize PENDING entries as JSON [{reqId,text,node_id}, ...].
+/// Return up to batchSize PENDING entries as JSON [{reqId,text,node_id,config}, ...].
 /// Plain ordered scan (single-processor; no claim state).
+///
+/// ``config`` is what the entry was enqueued with (spec 226). Enqueue has always stored
+/// it; without it on the wire a worker cannot tell which model an entry asked for, so the
+/// queue would be the one write path that escapes the embedding identity registry.
 ClassMethod ClaimPendingBatch(batchSize As %Integer = 100) As %String
 {
     Set arr = []
@@ -94,6 +100,7 @@ ClassMethod ClaimPendingBatch(batchSize As %Integer = 100) As %String
         Set obj.reqId = reqId
         Set obj.text = $Get(^EmbedQueue(reqId, "text"))
         Set obj."node_id" = $Get(^EmbedQueue(reqId, "node_id"))
+        Set obj.config = $Get(^EmbedQueue(reqId, "config"))
         Do arr.%Push(obj)
         Set n = n + 1
         Quit:n>=batchSize
@@ -117,6 +124,21 @@ ClassMethod SetResult(reqId As %String, status As %String, payload As %String =
     Quit $$$OK
 }
 
+/// One entry's own state as JSON {reqId,status,error,config,node_id}. A claim only ever
+/// returns PENDING entries, so before this there was nothing that could read back why a
+/// single entry failed — the queue's only Python-visible state was aggregate counts
+/// (spec 226). Returns a record with an empty status when reqId is unknown.
+ClassMethod GetEntry(reqId As %String) As %String
+{
+    Set obj = {}
+    Set obj.reqId = reqId
+    Set obj.status = ..NormStatus($Get(^EmbedQueue(reqId, "status")))
+    Set obj.error = $Get(^EmbedQueue(reqId, "error"))
+    Set obj.config = $Get(^EmbedQueue(reqId, "config"))
+    Set obj."node_id" = $Get(^EmbedQueue(reqId, "node_id"))
+    Quit obj.%ToJSON()
+}
+
 ClassMethod PendingCount() As %Integer
 {
     Set count = 0

{iris_vector_graph-3.1.0 → iris_vector_graph-3.2.0}/iris_vector_graph/__init__.py

@@ -41,7 +41,17 @@ from .errors import (
     NodeNotFoundError,
     PrerequisiteError,
 )
-from .exceptions import NamespaceConsistencyError, NamespaceMismatchWarning
+from .embedding_identity import (
+    MECHANISMS,
+    EmbeddingIdentity,
+    identity_from_config,
+    normalize_model_key,
+)
+from .exceptions import (
+    EmbeddingIdentityConflict,
+    NamespaceConsistencyError,
+    NamespaceMismatchWarning,
+)
 from .fhir_bridge import (
     FHIRSearchTool,
     GetPatientKGNeighborhoodTool,
@@ -83,6 +93,11 @@ except Exception:
 
 __all__ = [
     "IRISGraphEngine",
+    "EmbeddingIdentity",
+    "EmbeddingIdentityConflict",
+    "MECHANISMS",
+    "identity_from_config",
+    "normalize_model_key",
     "GraphSchema",
     "IRISCapabilities",
     "VectorOptimizer",

{iris_vector_graph-3.1.0 → iris_vector_graph-3.2.0}/iris_vector_graph/_engine/admin.py

@@ -82,6 +82,24 @@ class AdminMixin:
             return self._show_constraints()
         return IVGResult(columns=["value"], rows=[])
 
+    def _hnsw_indexes(self, table_name: str) -> List[tuple]:
+        """The HNSW indexes IRIS actually holds on ``table_name``, as ``(name, properties)``.
+
+        Delegates to :meth:`GraphSchema.hnsw_indexes`, which owns the dictionary read. The
+        name is the index's ``SqlName`` — what the operator typed in ``CREATE INDEX`` and
+        what they would type to drop it.
+        """
+        from iris_vector_graph.schema import GraphSchema
+
+        cursor = self.conn.cursor()
+        try:
+            return GraphSchema.hnsw_indexes(cursor, table_name)
+        finally:
+            try:
+                cursor.close()
+            except Exception:
+                pass
+
     def _show_indexes(self) -> "IVGResult":
         cols = ["name", "type", "entityType", "labelsOrTypes", "properties", "state"]
         rows = []
@@ -94,24 +112,26 @@ class AdminMixin:
             except Exception:
                 return default or []
 
-        # HNSW vector index on kg_NodeEmbeddings
-        hnsw_count = 0
-        try:
-            cursor.execute(f"SELECT COUNT(*) FROM {self._t('kg_NodeEmbeddings_optimized')}")
-            r = cursor.fetchone()
-            hnsw_count = int(r[0]) if r else 0
-        except Exception:
-            pass
-        rows.append(
-            [
-                "hnsw_node_embeddings",
-                "VECTOR(HNSW)",
-                "NODE",
-                ["*"],
-                ["emb"],
-                "ONLINE" if hnsw_count > 0 else "BUILDING",
-            ]
-        )
+        # HNSW vector indexes, read from the class dictionary. Nothing is emitted when no
+        # index exists (spec 226, FR-018): this used to append a `hnsw_node_embeddings` row
+        # unconditionally and derive its state from the row count of
+        # `kg_NodeEmbeddings_optimized` — `ONLINE` with rows, `BUILDING` without. A row
+        # count is not an index, and `BUILDING` told operators to wait for a build that was
+        # never running. On the shipped schema it could not run: both embedding tables key
+        # on a VARCHAR `id`, and IRIS refuses ANN indices there (ERROR #7222).
+        for table_name in ("kg_NodeEmbeddings", "kg_NodeEmbeddings_optimized"):
+            table = self._t(table_name)
+            for index_name, properties in self._hnsw_indexes(table):
+                rows.append(
+                    [
+                        index_name,
+                        "VECTOR(HNSW)",
+                        "NODE",
+                        [table],
+                        [p for p in str(properties).split(",") if p] or ["emb"],
+                        "ONLINE",
+                    ]
+                )
 
         for (name,) in _try(f"SELECT DISTINCT name FROM {self._t('kg_IVFMeta')}"):
             rows.append([name, "VECTOR(IVF)", "NODE", ["*"], ["emb"], "ONLINE"])

{iris_vector_graph-3.1.0 → iris_vector_graph-3.2.0}/iris_vector_graph/_engine/embeddings.py

@@ -2,6 +2,7 @@ from typing import List, Dict, Any, Optional
 import json
 import logging
 
+from iris_vector_graph.exceptions import EmbeddingIdentityConflict
 from iris_vector_graph.schema import GraphSchema
 
 logger = logging.getLogger(__name__)
@@ -179,6 +180,14 @@ class EmbeddingsMixin:
         _dtype = (dtype or self.vector_dtype).upper()
         self._assert_node_exists(node_id)
 
+        # spec 226: before anything is written, and before the fallback below can infer a
+        # width from the input and mutate self.embedding_dimension to it. The width offered
+        # is the vector's own length — that is what IRIS would reject at INSERT, and what a
+        # writer declaring no model is still held to.
+        self.enforce_embedding_identity(
+            "kg_NodeEmbeddings", dimension=len(embedding), dtype=_dtype
+        )
+
         try:
             dim = self._get_embedding_dimension()
         except ValueError:
@@ -217,6 +226,14 @@ class EmbeddingsMixin:
         if not items:
             return True
 
+        # spec 226: one registry read for the whole batch, before the first INSERT and
+        # before the inference fallback below. A refused batch writes nothing — not a
+        # partial prefix — because the check runs outside the transaction entirely.
+        for offered_width in sorted({len(item["embedding"]) for item in items}):
+            self.enforce_embedding_identity(
+                "kg_NodeEmbeddings", dimension=offered_width, dtype=_dtype
+            )
+
         try:
             dim = self._get_embedding_dimension()
         except ValueError:
@@ -673,9 +690,25 @@ class EmbeddingsMixin:
           - ``texts``: free-text entries (always-new; never deduplicated).
         Both may be supplied; an empty/None pair returns 0. Degrades gracefully
         (returns 0 + warns) when the queue backend is unavailable.
+
+        Raises:
+            EmbeddingIdentityConflict: ``embedding_config`` (or, when empty, this engine's
+                own model) disagrees with what ``kg_NodeEmbeddings`` records. Queueing work
+                that cannot be honoured only moves the failure onto a worker, away from
+                whoever asked for it — so this is refused here, before anything is queued
+                (spec 226, FR-011). Unlike the backend-unavailable case below, a conflict is
+                never downgraded to a warning.
         """
         from iris_vector_graph.schema import _call_classmethod
         import json as _json
+
+        # Deliberately outside the try/except blocks below, which exist to tolerate a
+        # missing queue backend. A conflict is not a degraded backend.
+        if node_ids or texts:
+            self.enforce_embedding_identity(
+                "kg_NodeEmbeddings", config=embedding_config
+            )
+
         total = 0
         if node_ids:
             try:
@@ -725,6 +758,28 @@ class EmbeddingsMixin:
         if not entries:
             return {"processed": 0, "errors": 0}
 
+        # spec 226 / FR-011: fail conflicting entries before the model runs. Each entry
+        # carries the model it asked for (`config`); an empty config is this worker's own
+        # identity, not an unknown. Filtering here rather than inside the loop below keeps
+        # the batched encode aligned with the entries it is for, spends no model time on
+        # work that could never be stored, and — the point of the batch — lets one bad
+        # entry fail without stalling the rest.
+        honoured = []
+        refused = 0
+        for entry in entries:
+            try:
+                self.enforce_embedding_identity(
+                    "kg_NodeEmbeddings", config=entry.get("config") or ""
+                )
+            except EmbeddingIdentityConflict as e:
+                refused += 1
+                self._fail_queue_entry(entry.get("reqId", ""), str(e))
+                continue
+            honoured.append(entry)
+        entries = honoured
+        if not entries:
+            return {"processed": 0, "errors": refused}
+
         texts = [e.get("text", "") for e in entries]
         # Single batched embedder call (SC-002). If the whole call fails, fall back to
         # per-entry so one poison text does not fail the entire batch (FR-007).
@@ -735,7 +790,7 @@ class EmbeddingsMixin:
             logger.warning("batch encode failed, falling back per-entry: %s", e)
 
         processed = 0
-        errors = 0
+        errors = refused
         for idx, entry in enumerate(entries):
             req_id = entry.get("reqId", "")
             node_id = entry.get("node_id", "") or ""
@@ -754,15 +809,24 @@ class EmbeddingsMixin:
                 processed += 1
             except Exception as e:
                 errors += 1
-                try:
-                    _call_classmethod(
-                        self.conn, "Graph.KG.EmbedQueue", "SetResult",
-                        req_id, "ERROR", str(e)[:500],
-                    )
-                except Exception as se:
-                    logger.warning("SetResult ERROR failed for %s: %s", req_id, se)
+                self._fail_queue_entry(req_id, str(e))
         return {"processed": processed, "errors": errors}
 
+    def _fail_queue_entry(self, req_id: str, message: str) -> None:
+        """Mark one queue entry ERROR with the reason, leaving the rest of the batch alone.
+
+        The reason is readable back via ``Graph.KG.EmbedQueue.GetEntry``, which is the only
+        way a per-entry failure is visible at all: a claim returns PENDING entries only.
+        """
+        from iris_vector_graph.schema import _call_classmethod
+        try:
+            _call_classmethod(
+                self.conn, "Graph.KG.EmbedQueue", "SetResult",
+                req_id, "ERROR", message[:500],
+            )
+        except Exception as se:
+            logger.warning("SetResult ERROR failed for %s: %s", req_id, se)
+
     def _encode_batch(self, texts: List[str]) -> list:
         """Embed a list of texts in one embedder call, resolving the engine embedder
         (same auto-init path used by embed_text). Returns a list of vectors."""
@@ -782,7 +846,14 @@ class EmbeddingsMixin:
         raise TypeError(f"Embedder {type(embedder)} has no encode/embed and is not callable")
 
     def _upsert_node_embedding(self, node_id: str, vec) -> None:
-        """Write a node's embedding into kg_NodeEmbeddings so vector search finds it."""
+        """Write a node's embedding into kg_NodeEmbeddings so vector search finds it.
+
+        Subject to the same identity contract as ``store_embedding`` (spec 226, FR-011):
+        this is the seam the embed queue writes through, so skipping it here would leave
+        the queue as the way past the registry.
+        """
+        vec = list(vec)
+        self.enforce_embedding_identity("kg_NodeEmbeddings", dimension=len(vec))
         emb_str = ",".join(str(float(x)) for x in vec)
         cursor = self.conn.cursor()
         try: