iris-vector-graph 2.3.1__tar.gz → 2.4.2__tar.gz

{iris_vector_graph-2.3.1 → iris_vector_graph-2.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iris-vector-graph
-Version: 2.3.1
+Version: 2.4.2
 Summary: Transactional Graph + Vector retrieval system for InterSystems IRIS with hybrid search, openCypher, and GraphQL APIs
 Project-URL: Homepage, https://github.com/intersystems-community/iris-vector-graph
 Project-URL: Documentation, https://github.com/intersystems-community/iris-vector-graph/tree/main/docs
@@ -25,6 +25,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Requires-Python: >=3.10
 Requires-Dist: intersystems-iris>=1.0.0
+Requires-Dist: iris-embedded-python-wrapper>=0.5.20
 Provides-Extra: biodata
 Requires-Dist: biopython>=1.81; extra == 'biodata'
 Requires-Dist: bioservices>=1.11.0; extra == 'biodata'

iris_vector_graph-2.4.2/iris_src/src/Graph/KG/EmbedQueue.cls

@@ -0,0 +1,210 @@
+Class Graph.KG.EmbedQueue Extends %RegisteredObject
+{
+
+/// Enhanced embedding queue (spec 199). Entries are subscripted:
+///   ^EmbedQueue(reqId, "status")    = "PENDING" | "DONE" | "ERROR"
+///   ^EmbedQueue(reqId, "text")      = text to embed
+///   ^EmbedQueue(reqId, "node_id")   = node id (node-keyed entries) or ""
+///   ^EmbedQueue(reqId, "result")    = "<f1>,<f2>,...,<fN>" comma-joined vector (on DONE)
+///   ^EmbedQueue(reqId, "error")     = message (on ERROR)
+///   ^EmbedQueue(reqId, "queued_at") = $ZDateTime timestamp
+///
+/// reqId = node_id (node-keyed enqueue; re-enqueue overwrites — one entry per node)
+///       or "txt:<n>" (free-text enqueue; always-new, never deduplicated).
+///
+/// Single-processor: ClaimPendingBatch is a plain ordered scan (no atomic claim state).
+/// Statuses are written UPPERCASE; reads tolerate legacy lowercase pending/done/error.
+/// The actual batched embedding is performed Python-side (process_embed_queue), which
+/// claims a batch, encodes all texts in one model call, and writes results via SetResult.
+
+/// Normalize a stored status value to UPPERCASE canonical form (back-compat).
+ClassMethod NormStatus(s As %String) As %String [ Private ]
+{
+    Quit $ZConvert(s, "U")
+}
+
+/// Node-keyed enqueue (one entry per node id; overwrite). The text to embed defaults
+/// to the node id; pass an explicit ``text`` to embed something else for the node.
+ClassMethod Enqueue(nodeId As %String, embeddingConfig As %String = "", text As %String = "") As %Status
+{
+    If text = "" { Set text = nodeId }
+    Set ^EmbedQueue(nodeId, "status") = "PENDING"
+    Set ^EmbedQueue(nodeId, "text") = text
+    Set ^EmbedQueue(nodeId, "node_id") = nodeId
+    Set ^EmbedQueue(nodeId, "result") = ""
+    Set ^EmbedQueue(nodeId, "error") = ""
+    Set ^EmbedQueue(nodeId, "config") = embeddingConfig
+    Set ^EmbedQueue(nodeId, "queued_at") = $ZDateTime($Horolog, 3)
+    Quit $$$OK
+}
+
+/// nodeIds may be a JSON array string (as sent by the Python client) or a
+/// %DynamicArray. Returns count enqueued.
+ClassMethod BulkEnqueue(nodeIds As %String, embeddingConfig As %String = "") As %Integer
+{
+    Set arr = nodeIds
+    If '$IsObject(arr) { Set arr = ##class(%DynamicArray).%FromJSON(nodeIds) }
+    Set count = 0
+    Set iter = arr.%GetIterator()
+    While iter.%GetNext(.idx, .nodeId) {
+        Set sc = ..Enqueue(nodeId, embeddingConfig)
+        If $$$ISOK(sc) { Set count = count + 1 }
+    }
+    Quit count
+}
+
+/// Free-text enqueue (always-new entry; reqId = "txt:<seq>"). texts may be a JSON
+/// array string (Python client) or a %DynamicArray. Returns count enqueued.
+ClassMethod BulkEnqueueText(texts As %String, embeddingConfig As %String = "") As %Integer
+{
+    Set arr = texts
+    If '$IsObject(arr) { Set arr = ##class(%DynamicArray).%FromJSON(texts) }
+    Set count = 0
+    Set iter = arr.%GetIterator()
+    While iter.%GetNext(.idx, .text) {
+        Continue:text=""
+        Set seq = $Increment(^EmbedQueue("$seq"))
+        Set reqId = "txt:" _ seq
+        Set ^EmbedQueue(reqId, "status") = "PENDING"
+        Set ^EmbedQueue(reqId, "text") = text
+        Set ^EmbedQueue(reqId, "node_id") = ""
+        Set ^EmbedQueue(reqId, "result") = ""
+        Set ^EmbedQueue(reqId, "error") = ""
+        Set ^EmbedQueue(reqId, "config") = embeddingConfig
+        Set ^EmbedQueue(reqId, "queued_at") = $ZDateTime($Horolog, 3)
+        Set count = count + 1
+    }
+    Quit count
+}
+
+/// Return up to batchSize PENDING entries as JSON [{reqId,text,node_id}, ...].
+/// Plain ordered scan (single-processor; no claim state).
+ClassMethod ClaimPendingBatch(batchSize As %Integer = 100) As %String
+{
+    Set arr = []
+    Set reqId = ""
+    Set n = 0
+    For {
+        Set reqId = $Order(^EmbedQueue(reqId))
+        Quit:reqId=""
+        Continue:reqId="$seq"
+        Set st = ..NormStatus($Get(^EmbedQueue(reqId, "status")))
+        Continue:st'="PENDING"
+        Set obj = {}
+        Set obj.reqId = reqId
+        Set obj.text = $Get(^EmbedQueue(reqId, "text"))
+        Set obj."node_id" = $Get(^EmbedQueue(reqId, "node_id"))
+        Do arr.%Push(obj)
+        Set n = n + 1
+        Quit:n>=batchSize
+    }
+    Quit arr.%ToJSON()
+}
+
+/// Set an entry's result. status="DONE" → payload is the result vector string;
+/// status="ERROR" → payload is the error message.
+ClassMethod SetResult(reqId As %String, status As %String, payload As %String = "") As %Status
+{
+    If '$Data(^EmbedQueue(reqId, "status")) { Quit $$$ERROR($$$GeneralError, "unknown reqId") }
+    Set up = $ZConvert(status, "U")
+    Set ^EmbedQueue(reqId, "status") = up
+    If up = "DONE" {
+        Set ^EmbedQueue(reqId, "result") = payload
+        Set ^EmbedQueue(reqId, "error") = ""
+    } ElseIf up = "ERROR" {
+        Set ^EmbedQueue(reqId, "error") = payload
+    }
+    Quit $$$OK
+}
+
+ClassMethod PendingCount() As %Integer
+{
+    Set count = 0
+    Set reqId = ""
+    For {
+        Set reqId = $Order(^EmbedQueue(reqId))
+        Quit:reqId=""
+        Continue:reqId="$seq"
+        If ..NormStatus($Get(^EmbedQueue(reqId, "status"))) = "PENDING" { Set count = count + 1 }
+    }
+    Quit count
+}
+
+/// Remove DONE entries; leave PENDING and ERROR intact. Returns number cleared.
+ClassMethod ClearDone() As %Integer
+{
+    Set cleared = 0
+    Set reqId = ""
+    For {
+        Set reqId = $Order(^EmbedQueue(reqId))
+        Quit:reqId=""
+        Continue:reqId="$seq"
+        If ..NormStatus($Get(^EmbedQueue(reqId, "status"))) = "DONE" {
+            Kill ^EmbedQueue(reqId)
+            Set cleared = cleared + 1
+        }
+    }
+    Quit cleared
+}
+
+/// Legacy native-EMBEDDING() per-row processing path, retained for callers that do not
+/// use the Python batched path. Reads the subscripted schema.
+ClassMethod ProcessBatch(batchSize As %Integer = 100, timeout As %Integer = 30) As %String
+{
+    Set processed = 0
+    Set errors = 0
+    Set reqId = ""
+    For {
+        Set reqId = $Order(^EmbedQueue(reqId))
+        Quit:reqId=""
+        Continue:reqId="$seq"
+        Quit:processed>=batchSize
+        Continue:..NormStatus($Get(^EmbedQueue(reqId, "status")))'="PENDING"
+
+        Set nodeId = $Get(^EmbedQueue(reqId, "node_id"))
+        Set embConfig = $Get(^EmbedQueue(reqId, "config"))
+        Set text = $Get(^EmbedQueue(reqId, "text"))
+        Try {
+            Set sql = "SELECT EMBEDDING(?, ?) FROM (SELECT 1)"
+            Set stmt = ##class(%SQL.Statement).%New()
+            Set sc = stmt.%Prepare(sql)
+            If $$$ISOK(sc) {
+                Set rs = stmt.%Execute(text, embConfig)
+                If rs.%SQLCODE = 0 && rs.%Next() {
+                    Set embVal = rs.%GetData(1)
+                    If (embVal '= "") && (nodeId '= "") {
+                        Set insertSql = "INSERT OR REPLACE INTO Graph_KG.kg_NodeEmbeddings (id, emb) VALUES (?, TO_VECTOR(?, DOUBLE))"
+                        Set stmt2 = ##class(%SQL.Statement).%New()
+                        Do stmt2.%Prepare(insertSql)
+                        Do stmt2.%Execute(nodeId, embVal)
+                    }
+                    If embVal '= "" { Set ^EmbedQueue(reqId, "result") = embVal }
+                }
+            }
+            Set ^EmbedQueue(reqId, "status") = "DONE"
+            Set processed = processed + 1
+        } Catch ex {
+            Set ^EmbedQueue(reqId, "status") = "ERROR"
+            Set ^EmbedQueue(reqId, "error") = ex.DisplayString()
+            Set errors = errors + 1
+        }
+    }
+    Set result = {"processed": (processed), "errors": (errors)}
+    Quit result.%ToJSON()
+}
+
+/// Start a background IRIS Task to process the queue.
+ClassMethod StartBackgroundTask(batchSize As %Integer = 100) As %String
+{
+    Set task = ##class(%SYSTEM.Task).%New()
+    Set task.Name = "IVG EmbedQueue Worker"
+    Set task.NameSpace = $ZNamespace
+    Set task.TaskClass = "Graph.KG.EmbedQueueTask"
+    Set sc = task.%Save()
+    If $$$ISERR(sc) { Quit "error: "_$System.Status.GetErrorText(sc) }
+    Set taskId = task.%Id()
+    Do ##class(%SYSTEM.Task).RunNow(taskId)
+    Quit taskId
+}
+
+}

{iris_vector_graph-2.3.1 → iris_vector_graph-2.4.2}/iris_vector_graph/_engine/embeddings.py

@@ -52,6 +52,10 @@ class EmbeddingsMixin:
                     _logging.getLogger("safetensors").setLevel(_logging.ERROR)
                 except Exception:
                     pass
+                # Local import avoids a circular import (engine.py imports this
+                # mixin at module load); function-local matches the pattern used
+                # elsewhere in this file (embed_selector, get_schema_prefix).
+                from iris_vector_graph.engine import _load_sentence_transformer, _is_sentence_transformer
                 self.embedder = _load_sentence_transformer("all-MiniLM-L6-v2")
                 logger.info("Auto-initialized SentenceTransformer('all-MiniLM-L6-v2')")
             except ImportError:
@@ -272,6 +276,7 @@ class EmbeddingsMixin:
     ) -> dict:
         from iris_vector_graph.embed_selector import EmbedSelector, build_node_where
         from iris_vector_graph.cypher import get_schema_prefix
+        from iris_vector_graph.engine import _load_sentence_transformer, _is_sentence_transformer
 
         sel = EmbedSelector(
             label=label,
@@ -436,6 +441,7 @@ class EmbeddingsMixin:
     ) -> dict:
         from iris_vector_graph.embed_selector import EmbedSelector, build_edge_where
         from iris_vector_graph.cypher import get_schema_prefix
+        from iris_vector_graph.engine import _load_sentence_transformer, _is_sentence_transformer
 
         sel = EmbedSelector(
             predicate=predicate,
@@ -655,45 +661,171 @@ class EmbeddingsMixin:
 
     def enqueue_for_embedding(
         self,
-        node_ids: List[str],
+        node_ids: Optional[List[str]] = None,
         embedding_config: str = "",
+        texts: Optional[List[str]] = None,
     ) -> int:
-        try:
-            import json as _json
-            ids_json = _json.dumps(node_ids)
-            result = self._call_classmethod(
-                "Graph.KG.EmbedQueue", "BulkEnqueue",
-                ids_json, embedding_config,
-            )
-            return int(str(result))
-        except Exception as e:
-            logger.warning("enqueue_for_embedding failed: %s", e)
-            return 0
+        """Enqueue embedding work onto the async queue. Returns the number enqueued.
+
+        Two modes (spec 199):
+          - ``node_ids``: node-keyed entries (re-enqueue overwrites — one entry per
+            node; backward compatible with the prior signature).
+          - ``texts``: free-text entries (always-new; never deduplicated).
+        Both may be supplied; an empty/None pair returns 0. Degrades gracefully
+        (returns 0 + warns) when the queue backend is unavailable.
+        """
+        from iris_vector_graph.schema import _call_classmethod
+        import json as _json
+        total = 0
+        if node_ids:
+            try:
+                ids_json = _json.dumps(node_ids)
+                result = _call_classmethod(
+                    self.conn, "Graph.KG.EmbedQueue", "BulkEnqueue",
+                    ids_json, embedding_config,
+                )
+                total += int(str(result))
+            except Exception as e:
+                logger.warning("enqueue_for_embedding (node_ids) failed: %s", e)
+        if texts:
+            try:
+                texts_json = _json.dumps(texts)
+                result = _call_classmethod(
+                    self.conn, "Graph.KG.EmbedQueue", "BulkEnqueueText",
+                    texts_json, embedding_config,
+                )
+                total += int(str(result))
+            except Exception as e:
+                logger.warning("enqueue_for_embedding (texts) failed: %s", e)
+        return total
 
 
     def process_embed_queue(self, batch_size: int = 100) -> dict:
+        """Process up to ``batch_size`` PENDING queue entries (spec 199).
+
+        Claims a batch of pending entries, embeds ALL of their texts in a SINGLE
+        embedder call (the batched performance win), writes each result back to the
+        queue entry, and — for node-keyed entries — upserts the vector into
+        ``kg_NodeEmbeddings`` so semantic search finds it. A per-entry embedding
+        failure marks only that entry ERROR; the rest of the batch still completes.
+        Returns ``{"processed": int, "errors": int}``. Degrades gracefully when the
+        backend is unavailable.
+        """
+        import json as _json
+        from iris_vector_graph.schema import _call_classmethod
         try:
-            import json as _json
-            result_json = str(self._call_classmethod(
-                "Graph.KG.EmbedQueue", "ProcessBatch",
-                batch_size, 30,
+            claim_json = str(_call_classmethod(
+                self.conn, "Graph.KG.EmbedQueue", "ClaimPendingBatch", batch_size,
             ))
-            return _json.loads(result_json)
+            entries = _json.loads(claim_json) if claim_json else []
         except Exception as e:
-            logger.warning("process_embed_queue failed: %s", e)
+            logger.warning("process_embed_queue claim failed: %s", e)
             return {"processed": 0, "errors": 0}
 
+        if not entries:
+            return {"processed": 0, "errors": 0}
+
+        texts = [e.get("text", "") for e in entries]
+        # Single batched embedder call (SC-002). If the whole call fails, fall back to
+        # per-entry so one poison text does not fail the entire batch (FR-007).
+        vectors = None
+        try:
+            vectors = self._encode_batch(texts)
+        except Exception as e:
+            logger.warning("batch encode failed, falling back per-entry: %s", e)
+
+        processed = 0
+        errors = 0
+        for idx, entry in enumerate(entries):
+            req_id = entry.get("reqId", "")
+            node_id = entry.get("node_id", "") or ""
+            try:
+                if vectors is not None:
+                    vec = vectors[idx]
+                else:
+                    vec = self._encode_batch([entry.get("text", "")])[0]
+                vec_str = ",".join(str(float(x)) for x in vec)
+                _call_classmethod(
+                    self.conn, "Graph.KG.EmbedQueue", "SetResult",
+                    req_id, "DONE", vec_str,
+                )
+                if node_id:
+                    self._upsert_node_embedding(node_id, vec)
+                processed += 1
+            except Exception as e:
+                errors += 1
+                try:
+                    _call_classmethod(
+                        self.conn, "Graph.KG.EmbedQueue", "SetResult",
+                        req_id, "ERROR", str(e)[:500],
+                    )
+                except Exception as se:
+                    logger.warning("SetResult ERROR failed for %s: %s", req_id, se)
+        return {"processed": processed, "errors": errors}
+
+    def _encode_batch(self, texts: List[str]) -> list:
+        """Embed a list of texts in one embedder call, resolving the engine embedder
+        (same auto-init path used by embed_text). Returns a list of vectors."""
+        if not self.embedder:
+            # Reuse embed_text's auto-init by embedding the first item, then encode all.
+            # embed_text resolves/sets self.embedder (or raises a clear RuntimeError).
+            if texts:
+                self.embed_text(texts[0])
+        embedder = self.embedder
+        if hasattr(embedder, "encode"):
+            result = embedder.encode(texts)
+            return [r.tolist() if hasattr(r, "tolist") else list(r) for r in result]
+        if hasattr(embedder, "embed"):
+            return [embedder.embed(t) for t in texts]
+        if callable(embedder):
+            return [embedder(t) for t in texts]
+        raise TypeError(f"Embedder {type(embedder)} has no encode/embed and is not callable")
+
+    def _upsert_node_embedding(self, node_id: str, vec) -> None:
+        """Write a node's embedding into kg_NodeEmbeddings so vector search finds it."""
+        emb_str = ",".join(str(float(x)) for x in vec)
+        cursor = self.conn.cursor()
+        try:
+            cursor.execute(
+                f"DELETE FROM {_table('kg_NodeEmbeddings')} WHERE id=?", [node_id]
+            )
+            cursor.execute(
+                f"INSERT INTO {_table('kg_NodeEmbeddings')} (id, emb) "
+                f"VALUES (?, TO_VECTOR('{emb_str}', {self.vector_dtype}))",
+                [node_id],
+            )
+            self.conn.commit()
+        finally:
+            try:
+                cursor.close()
+            except Exception:
+                pass
+
+    def clear_done(self) -> int:
+        """Remove completed (DONE) queue entries; leave PENDING/ERROR intact (spec 199,
+        FR-009). Returns the number cleared; 0 when the backend is unavailable."""
+        from iris_vector_graph.schema import _call_classmethod
+        try:
+            return int(str(_call_classmethod(
+                self.conn, "Graph.KG.EmbedQueue", "ClearDone",
+            )))
+        except Exception as e:
+            logger.warning("clear_done failed: %s", e)
+            return 0
+
 
     def embed_queue_pending(self) -> int:
         try:
-            return int(str(self._call_classmethod("Graph.KG.EmbedQueue", "PendingCount")))
+            from iris_vector_graph.schema import _call_classmethod
+            return int(str(_call_classmethod(self.conn, "Graph.KG.EmbedQueue", "PendingCount")))
         except Exception:
             return 0
 
 
     def start_background_embedding(self, batch_size: int = 100) -> str:
         try:
-            return str(self._call_classmethod("Graph.KG.EmbedQueue", "StartBackgroundTask", batch_size))
+            from iris_vector_graph.schema import _call_classmethod
+            return str(_call_classmethod(self.conn, "Graph.KG.EmbedQueue", "StartBackgroundTask", batch_size))
         except Exception as e:
             logger.warning("start_background_embedding failed: %s", e)
             return ""

{iris_vector_graph-2.3.1 → iris_vector_graph-2.4.2}/iris_vector_graph/_engine/nodes_edges.py

@@ -459,7 +459,11 @@ class NodesEdgesMixin:
             graph: Optional named graph identifier.
 
         Returns:
-            True if the edge was created or already existed, False on error.
+            True if a NEW edge was created. False if the edge already existed
+            (UNIQUE violation, swallowed safely) OR on error. The duplicate case is
+            safe and never raises — but it is NOT distinguishable from an error by the
+            return value, so callers that re-create edges defensively should call
+            create_edge unconditionally and not gate success on the return value.
         """
         EdgeInput(source_id=source_id, predicate=predicate, target_id=target_id)
         cursor = self.conn.cursor()

{iris_vector_graph-2.3.1 → iris_vector_graph-2.4.2}/iris_vector_graph/cypher_api.py

@@ -138,14 +138,17 @@ def _make_engine() -> IRISGraphEngine:
         namespace = os.environ.get("IRIS_NAMESPACE", "USER")
         username = os.environ.get("IRIS_USERNAME", "_SYSTEM")
         password = os.environ.get("IRIS_PASSWORD", "SYS")
+        # Standard seam: the iris wrapper's dbapi.connect (base dependency). The
+        # fallback also routes through dbapi.connect since the wrapper's top-level
+        # `iris` always provides it.
         if _iris_wrapper and hasattr(_iris_wrapper, "dbapi"):
             conn = _iris_wrapper.dbapi.connect(
                 hostname=host, port=port, namespace=namespace,
                 username=username, password=password,
             )
         else:
-            conn = iris.connect(hostname=host, port=port, namespace=namespace,
-                                username=username, password=password)
+            conn = iris.dbapi.connect(hostname=host, port=port, namespace=namespace,
+                                      username=username, password=password)
         return IRISGraphEngine(conn)
 
     if _state.startswith("embedded") and _iris_wrapper and hasattr(_iris_wrapper, "dbapi"):

{iris_vector_graph-2.3.1 → iris_vector_graph-2.4.2}/iris_vector_graph/engine.py

@@ -170,7 +170,9 @@ class IRISGraphEngine(RdfExportMixin, ShaclMixin, ProvMixin, TemporalMixin, Snap
     ) -> "IRISGraphEngine":
         import iris as _iris
         conn_params = dict(hostname=hostname, port=port, namespace=namespace, username=username, password=password)
-        conn = _iris.connect(**conn_params)
+        # Standard connection seam: iris-embedded-python-wrapper's dbapi.connect
+        # (drop-in for iris.connect with DB-API exception semantics).
+        conn = _iris.dbapi.connect(**conn_params)
         engine = cls(conn, embedding_dimension=embedding_dimension, **kwargs)
         engine._connection_params = conn_params
         return engine
@@ -216,7 +218,7 @@ class IRISGraphEngine(RdfExportMixin, ShaclMixin, ProvMixin, TemporalMixin, Snap
             if any(x in err_str for x in ("epipe", "broken pipe", "connection reset", "closed")):
                 if self._connection_params:
                     import iris as _iris
-                    self.conn = _iris.connect(**self._connection_params)
+                    self.conn = _iris.dbapi.connect(**self._connection_params)
                     logger.info("IRIS connection re-established after EPIPE")
                 else:
                     raise RuntimeError(

{iris_vector_graph-2.3.1 → iris_vector_graph-2.4.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "iris-vector-graph"
-version = "2.3.1"
+version = "2.4.2"
 description = "Transactional Graph + Vector retrieval system for InterSystems IRIS with hybrid search, openCypher, and GraphQL APIs"
 readme = "README.md"
 license = "MIT"
@@ -40,6 +40,12 @@ requires-python = ">=3.10"
 # Core dependencies
 dependencies = [
     "intersystems-iris>=1.0.0",
+    # iris-embedded-python-wrapper provides a unified top-level `iris` module that
+    # works both as an external SDK client (iris.dbapi.connect / iris.connect) and
+    # inside IRIS embedded Python (iris.sql / iris.gref / runtime.state). It is the
+    # standard connection seam — base dependency so `import iris` always resolves to
+    # the superset. See iris_vector_graph/embedded.py for the embedded context.
+    "iris-embedded-python-wrapper>=0.5.20",
 ]
 
 [project.optional-dependencies]
@@ -216,7 +222,8 @@ markers = [
     "perf: marks tests as opt-in performance benchmarks (run with '-m perf')",
     "requires_iris: marks tests that require IRIS database connection",
     "requires_database: marks tests that require database connection (alias for requires_iris)",
-    "e2e: marks tests as end-to-end system tests (all query engines)"
+    "e2e: marks tests as end-to-end system tests (all query engines)",
+    "requires_clean_isolation: marks tests needing an unpolluted database (run in isolation)"
 ]
 
 [tool.coverage.run]

{iris_vector_graph-2.3.1 → iris_vector_graph-2.4.2}/tests/TESTING.md

@@ -20,22 +20,22 @@ pytest -m performance
 
 The `iris_test_container` fixture (session scope) manages the container lifecycle:
 
-1. **Attach or Start**: If a container named `iris-vector-graph-main` is already running, attaches to it via `IRISContainer.attach()`. Otherwise starts a fresh container.
-2. **Ready**: Waits for the IRIS SuperServer to be ready (180s timeout).
-3. **Initialize**: Automatically creates the `Graph_KG` schema, tables, views, and loads ObjectScript classes.
-4. **Security**: Creates a dedicated **`test`** user with password **`test`** and ensures password expiry is disabled.
-5. **Verify**: Blocks until a real `test`/`test` connection succeeds — no test module ever races on a not-yet-ready container.
-6. **Execute**: Runs the requested tests.
-7. **Tear down**: Stops containers that were started by the fixture; leaves pre-existing containers running.
+1.  **Attach or Start**: If a container named `iris-vector-graph-main` is already running, attaches to it via `IRISContainer.attach()`. Otherwise starts a fresh container.
+2.  **Ready**: Waits for the IRIS SuperServer to be ready (180s timeout).
+3.  **Initialize**: Automatically creates the `Graph_KG` schema, tables, views, and loads ObjectScript classes.
+4.  **Security**: Creates a dedicated **`test`** user with password **`test`** and ensures password expiry is disabled.
+5.  **Verify**: Blocks until a real `test`/`test` connection succeeds — no test module ever races on a not-yet-ready container.
+6.  **Execute**: Runs the requested tests.
+7.  **Tear down**: Stops containers that were started by the fixture; leaves pre-existing containers running.
 
 ## Key Fixtures
 
-| Fixture               | Scope    | Description                                                                                    |
-| --------------------- | -------- | ---------------------------------------------------------------------------------------------- |
-| `iris_test_container` | Session  | Managed IRIS container for the entire test session.                                            |
-| `iris_connection`     | Module   | Reusable DBAPI connection from the managed container.                                          |
-| `iris_cursor`         | Function | Fresh cursor with automatic rollback isolation.                                                |
-| `clean_test_data`     | Function | Provides a unique prefix (e.g., `TEST_abc123:`) and auto-deletes matching data after the test. |
+| Fixture | Scope | Description |
+|---------|-------|-------------|
+| `iris_test_container` | Session | Managed IRIS container for the entire test session. |
+| `iris_connection` | Module | Reusable DBAPI connection from the managed container. |
+| `iris_cursor` | Function | Fresh cursor with automatic rollback isolation. |
+| `clean_test_data` | Function | Provides a unique prefix (e.g., `TEST_abc123:`) and auto-deletes matching data after the test. |
 
 ## Writing New Tests
 
@@ -49,64 +49,55 @@ def test_my_feature(iris_connection, clean_test_data):
     """Test with managed container and auto-cleanup."""
     prefix = clean_test_data
     cursor = iris_connection.cursor()
-
+    
     # Create test data using the unique prefix
     cursor.execute(
         "INSERT INTO nodes (node_id) VALUES (?)",
         (f"{prefix}node1",)
     )
     iris_connection.commit()
-
+    
     # Assertions
     cursor.execute("SELECT COUNT(*) FROM nodes WHERE node_id LIKE ?", (f"{prefix}%",))
     assert cursor.fetchone()[0] == 1
-
+    
     # No need to manual cleanup - clean_test_data fixture handles it!
 ```
 
 ## Configuration
 
-| Environment Variable | Default | Description                 |
-| -------------------- | ------- | --------------------------- |
-| `IRIS_TEST_TIMEOUT`  | `180`   | Startup timeout in seconds. |
+| Environment Variable | Default | Description |
+|----------------------|---------|-------------|
+| `IRIS_TEST_TIMEOUT` | `180` | Startup timeout in seconds. |
 
 ## Troubleshooting
 
 ### "Password change required" or "Access Denied"
-
 IRIS Community Edition often enforces a password change on first login. The test infrastructure automatically handles this by:
-
 1. Creating a dedicated `test`/`test` user with `%ALL` roles.
 2. Forcefully clearing `ChangePassword` and `PasswordNeverExpires` flags via ObjectScript.
 
 If you encounter this when connecting manually:
-
 - Use the **`test`** user instead of `_SYSTEM`.
 - Or reset the `_SYSTEM` password: `Set usr = ##class(Security.Users).%OpenId("_SYSTEM"), usr.ChangePassword=0, usr.PasswordNeverExpires=1 Do usr.%Save()`
 
 ### Schema (`Graph_KG`)
-
 All tables are created in the **`Graph_KG`** schema with views mirrored into `SQLUser` for unqualified access. Use fully-qualified names in tests (`Graph_KG.nodes`) or the unqualified view aliases (`nodes`, `rdf_edges`, etc.).
 
 ### "AttributeError: module 'iris' has no attribute 'connect'"
-
 This used to happen because of a naming conflict with the `iris/` directory. That directory has been renamed to `iris_src/`, so `import iris` should now work correctly.
 
 The `iris_connection` fixture handles connection management for you.
 
 ### Resetting the Environment
-
 If the test container gets into a bad state:
-
 ```bash
 docker rm -f iris-vector-graph-main
 pytest
 ```
 
 ### Production-Scale Benchmarks
-
 The `test_nodepk_production_scale` tests create 100K nodes / 500K edges and are **skipped by default** to avoid multi-minute runtimes and segfaults under memory pressure. To run them:
-
 ```bash
 RUN_PRODUCTION_SCALE=1 pytest tests/integration/test_nodepk_production_scale.py
 ```
@@ -114,7 +105,6 @@ RUN_PRODUCTION_SCALE=1 pytest tests/integration/test_nodepk_production_scale.py
 ## Constitutional Compliance
 
 This infrastructure enforces **Principle II: Test-First with Live Database**:
-
 - ✅ All `@pytest.mark.requires_database` tests hit real IRIS.
 - ✅ No mocked database for integration tests.
 - ✅ Schema automatically initialized.