iris-vector-graph 4.2.1__tar.gz → 4.2.3__tar.gz

{iris_vector_graph-4.2.1 → iris_vector_graph-4.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: iris-vector-graph
-Version: 4.2.1
+Version: 4.2.3
 Summary: Native graph database layer for InterSystems IRIS — openCypher, hybrid vector search, temporal property graph, RDF export, SHACL validation
 Project-URL: Homepage, https://github.com/intersystems-community/iris-vector-graph
 Project-URL: Documentation, https://github.com/intersystems-community/iris-vector-graph/tree/main/docs

iris_vector_graph-4.2.3/iris_vector_graph/_dictionary.py

@@ -0,0 +1,36 @@
+"""Whether a compiled class or method exists, asked in SQL.
+
+The Native API calls these used (``%Dictionary.CompiledClass.%ExistsId``) cannot be reached the same way from
+every host: under irispython a ``%`` method is spelled ``_ExistsId``, and through the wgproto Callin worker the
+call, once spelled that way, segfaults the worker (signal 11, measured on Build 154). A SELECT works from all of
+them.
+"""
+
+from __future__ import annotations
+
+
+def _count(conn, sql: str, params) -> int:
+    cursor = conn.cursor()
+    try:
+        cursor.execute(sql, params)
+        row = cursor.fetchone()
+        return int(row[0]) if row and row[0] is not None else 0
+    finally:
+        try:
+            cursor.close()
+        except Exception:
+            pass
+
+
+def compiled_class_exists(conn, name: str) -> bool:
+    return bool(_count(conn, "SELECT COUNT(*) FROM %Dictionary.CompiledClass WHERE Name = ?", [name]))
+
+
+def compiled_method_exists(conn, class_name: str, method: str) -> bool:
+    return bool(
+        _count(
+            conn,
+            "SELECT COUNT(*) FROM %Dictionary.CompiledMethod WHERE Parent = ? AND Name = ?",
+            [class_name, method],
+        )
+    )

{iris_vector_graph-4.2.1 → iris_vector_graph-4.2.3}/iris_vector_graph/_engine/class_deploy.py

@@ -202,7 +202,13 @@ def upload_and_compile(
             if _ok(iris_obj, status):
                 break
         else:
-            result.errors.append(_error_text(iris_obj, status))
+            # Build 154 fails the batch on Graph.KG.MCPToolSet ("MCPTools has not had inheritance
+            # resolved") although every class compiles alone, so compile them one at a time before
+            # calling the deploy failed.
+            for name in batch:
+                one = iris_obj.classMethodValue("%SYSTEM.OBJ", "CompileList", f"{name}.cls", "ck-d")
+                if not _ok(iris_obj, one):
+                    result.errors.append(f"{name}: {_error_text(iris_obj, one)}")
 
     result.deployed = True
     if not result.errors:
@@ -225,7 +231,7 @@ def deploy_packaged_classes(conn, force: Optional[bool] = None) -> Optional[Depl
 
     if isinstance(conn, EmbeddedConnection):
         # A process inside IRIS has no ``iris.createIRIS``.
-        iris_obj = EmbeddedIRIS()
+        iris_obj = EmbeddedIRIS(py_names=True)  # %New is _New under irispython
     else:
         import iris as _iris_pkg
 

{iris_vector_graph-4.2.1 → iris_vector_graph-4.2.3}/iris_vector_graph/cypher_api.py

@@ -198,6 +198,11 @@ def _run_cypher(query: str, parameters: dict | None = None, limit: int = 1000) -
                     _reset_engine()
                     continue
                 raise
+    error = getattr(result, "error", None)
+    if isinstance(error, str) and error:
+        # execute_sql folds a failed statement into IVGResult.error; answering "OK, no rows" for it hid
+        # a parameter error from the caller as an empty graph.
+        raise RuntimeError(error)
     columns = result.columns
     rows = result.rows
     if len(rows) > limit:

{iris_vector_graph-4.2.1 → iris_vector_graph-4.2.3}/iris_vector_graph/embedded.py

@@ -116,6 +116,11 @@ def _require_iris_sql():
         ) from exc
 
 
+#: More bound parameters than this and ``stmt.execute(*params)`` has overflowed the Python-to-IRIS call
+#: stack (measured with 400 under irispython); such a statement is sent with its values written in.
+MAX_BOUND_PARAMS = 100
+
+
 def _is_no_data(exc: Exception) -> bool:
     """SQLCODE 100: an UPDATE or DELETE that matched no row. A client cursor returns an empty
     result for it; ``iris.sql`` raises ``SQLError`` with an empty message."""
@@ -237,39 +242,6 @@ class EmbeddedCursor:
             return self._iris_sql
         return _require_iris_sql()
 
-    def _run(self, iris_sql, sql, params):
-        """Run ``sql`` by the first embedded path that accepts it; return the result set."""
-        try:
-            if params and any(isinstance(p, str) and p == "" for p in params):
-                # An empty string bound as a parameter reaches the engine as SQL NULL, so a
-                # NOT NULL column holding '' (graph_id of the default graph) rejected every
-                # default-graph write. Inline the literals for these statements.
-                return _exec_inline(iris_sql, sql, params)
-            else:
-                stmt = iris_sql.prepare(sql)
-                if params:
-                    return stmt.execute(*params)
-                else:
-                    return stmt.execute()
-        except Exception as exc1:
-            if not _is_ddtab_error(exc1):
-                raise
-            try:
-                return iris_sql.exec(_inline_params(sql, params or []))
-            except Exception as exc2:
-                if not _is_ddtab_error(exc2):
-                    raise
-                try:
-                    return _sql_statement_execute(sql, params)
-                except Exception as exc3:
-                    raise RuntimeError(
-                        f"All three embedded SQL paths failed.\n"
-                        f"  prepare: {exc1}\n"
-                        f"  exec: {exc2}\n"
-                        f"  %%SQL.Statement: {exc3}"
-                    ) from exc3
-
-
     def execute(self, sql, params=None):
         iris_sql = self._get_iris_sql()
         lowered = sql.strip().upper()
@@ -281,9 +253,43 @@ class EmbeddedCursor:
             self.rowcount = -1
             return
 
+        # The statement runs in this frame, as it did before SQLCODE 100 was handled: a result set
+        # handed back out of a helper's frame made the wgproto Callin worker segfault (4.2.1).
         no_data = False
         try:
-            self._rs = self._run(iris_sql, sql, params)
+            try:
+                if params and (
+                    len(params) > MAX_BOUND_PARAMS
+                    or any(isinstance(p, str) and p == "" for p in params)
+                ):
+                    # An empty string bound as a parameter reaches the engine as SQL NULL, so a
+                    # NOT NULL column holding '' (graph_id of the default graph) rejected every
+                    # default-graph write; and `stmt.execute(*params)` with hundreds of values dies
+                    # with <STACK>. Inline the literals for these statements.
+                    self._rs = _exec_inline(iris_sql, sql, params)
+                else:
+                    stmt = iris_sql.prepare(sql)
+                    if params:
+                        self._rs = stmt.execute(*params)
+                    else:
+                        self._rs = stmt.execute()
+            except Exception as exc1:
+                if not _is_ddtab_error(exc1):
+                    raise
+                try:
+                    self._rs = iris_sql.exec(_inline_params(sql, params or []))
+                except Exception as exc2:
+                    if not _is_ddtab_error(exc2):
+                        raise
+                    try:
+                        self._rs = _sql_statement_execute(sql, params)
+                    except Exception as exc3:
+                        raise RuntimeError(
+                            f"All three embedded SQL paths failed.\n"
+                            f"  prepare: {exc1}\n"
+                            f"  exec: {exc2}\n"
+                            f"  %%SQL.Statement: {exc3}"
+                        ) from exc3
         except Exception as exc:
             if not _is_no_data(exc):
                 raise
@@ -394,8 +400,16 @@ class EmbeddedIRIS:
     that reaches ObjectScript (``rebuild_kg``, ``sync``, the BM25 and ledger clients).
     """
 
-    def __init__(self, iris_module=None):
+    def __init__(self, iris_module=None, py_names=False):
         self._iris = iris_module
+        # py_names spells a leading % as _ (``%New`` is ``_New``), which irispython needs for system
+        # methods. Off by default: through the wgproto Callin worker a %-method that now really runs
+        # segfaults (signal 11, measured on Build 154 by Mindwalk with the 4.2.1/4.2.2 mapping), where it
+        # used to raise AttributeError and let the caller fall back. Only the class deploy asks for it.
+        self._py_names = py_names
+
+    def _name(self, method):
+        return _py_name(method) if self._py_names else method
 
     def _cls(self, name):
         iris = self._iris
@@ -404,13 +418,13 @@ class EmbeddedIRIS:
         return iris.cls(name)
 
     def classMethodValue(self, cls, method, *args):
-        return getattr(self._cls(cls), _py_name(method))(*args)
+        return getattr(self._cls(cls), self._name(method))(*args)
 
     def classMethodVoid(self, cls, method, *args):
-        getattr(self._cls(cls), _py_name(method))(*args)
+        getattr(self._cls(cls), self._name(method))(*args)
 
     def classMethodObject(self, cls, method, *args):
-        return getattr(self._cls(cls), _py_name(method))(*args)
+        return getattr(self._cls(cls), self._name(method))(*args)
 
     def get(self, name, *subscripts):
         """A global node, ``None`` when undefined: ``get("^IVG.Deploy", "fingerprint")``."""

{iris_vector_graph-4.2.1 → iris_vector_graph-4.2.3}/iris_vector_graph/ledger/guard.py

@@ -9,6 +9,8 @@ engines built against mocks or pre-deployment containers behave as before.
 from __future__ import annotations
 
 import json
+
+from iris_vector_graph._dictionary import compiled_class_exists
 import time
 from typing import Any, Dict, Optional
 
@@ -41,18 +43,9 @@ class LedgerGuard:
         try:
             iris_obj = self._engine._iris_obj()
             if self._class_present is None:
-                # %SYSTEM.OBJ.Exists is not reachable through the Native API on all builds;
-                # %Dictionary.CompiledClass.%ExistsId is.
-                self._class_present = bool(
-                    int(
-                        str(
-                            iris_obj.classMethodValue(
-                                "%Dictionary.CompiledClass", "%ExistsId", LEDGER_CLASS
-                            )
-                        )
-                        or 0
-                    )
-                )
+                # Asked in SQL (iris_vector_graph/_dictionary.py): %SYSTEM.OBJ.Exists is not reachable through
+                # the Native API on all builds, and a %-method is not safe from every host.
+                self._class_present = compiled_class_exists(self._engine.conn, LEDGER_CLASS)
             if self._class_present:
                 raw = str(iris_obj.classMethodValue(LEDGER_CLASS, "Meta"))
                 parsed = json.loads(raw) if raw else {}

{iris_vector_graph-4.2.1 → iris_vector_graph-4.2.3}/iris_vector_graph/migrations/docs_and_edge_vectors.py

@@ -48,6 +48,8 @@ import logging
 from dataclasses import dataclass, field
 from typing import Callable, Dict, List, Optional, Sequence, Tuple
 
+from iris_vector_graph.schema import GraphSchema
+
 logger = logging.getLogger(__name__)
 
 #: The schema every statement here qualifies. Overridable for a non-default install.
@@ -264,10 +266,37 @@ def finish_docs(conn, *, schema: str = DEFAULT_SCHEMA) -> None:
 def prepare_edge_vectors(
     conn, dimension: int, *, dtype: str = "DOUBLE", schema: str = DEFAULT_SCHEMA
 ) -> None:
-    """Run :func:`edge_vectors_prepare_sql`, tolerating a re-run."""
+    """Run :func:`edge_vectors_prepare_sql`, tolerating a re-run.
+
+    A staging or quarantine table a failed pass left at a different width is rebuilt first: a re-run that
+    reused it refused every row again (-104), and the operator had to find and drop both by hand.
+    """
+    _reset_wrong_width_tables(conn, dimension, schema)
     _run_tolerantly(conn, edge_vectors_prepare_sql(dimension, dtype=dtype, schema=schema))
 
 
+def _reset_wrong_width_tables(conn, dimension: int, schema: str) -> None:
+    cursor = conn.cursor()
+    try:
+        for table in (f"{schema}.kg_EdgeEmbeddings{STAGING_SUFFIX}", f"{schema}.edge_vector_quarantine"):
+            declared = GraphSchema.get_embedding_dimension(cursor, table)
+            if declared is None or declared == int(dimension):
+                continue
+            rows = _read(cursor, f"SELECT COUNT(*) FROM {table}")
+            held = int(rows[0][0]) if rows and rows[0] and rows[0][0] else 0
+            if held:
+                raise RuntimeError(
+                    f"{table} holds {held} row(s) at width {declared}, but the edge vectors are being placed "
+                    f"at width {dimension}. Those rows cannot be moved to the new width; resolve them, then "
+                    "drop the table and re-run."
+                )
+            logger.warning("Rebuilding empty %s: it was left at width %s, the run needs %s", table, declared, dimension)
+            cursor.execute(f"DROP TABLE {table}")
+        _commit(conn)
+    finally:
+        _close(cursor)
+
+
 def finish_edge_vectors(conn, *, schema: str = DEFAULT_SCHEMA) -> None:
     """Run :func:`edge_vectors_finish_sql`, tolerating a re-run."""
     _run_tolerantly(conn, edge_vectors_finish_sql(schema))
@@ -330,15 +359,18 @@ def place_documents(
                 "first, or run upgrade_to_4_0_0(), which does."
             )
         rows = _read(cursor, unplaced)
+        graphs_of = _graph_map(
+            cursor,
+            f"SELECT DISTINCT n.node_id, n.graph_id FROM {schema}.nodes n "
+            f"JOIN {schema}.docs d ON d.id = n.node_id",
+            1,
+        )
         report = _Tally()
+        placed: Dict[str, List[str]] = {}
+        moved: Dict[str, List[str]] = {}
         for row in rows:
             doc_id = str(row[0])
-            text = row[1] if len(row) > 1 else None
-            graphs = _graphs_of(
-                cursor,
-                f"SELECT DISTINCT graph_id FROM {schema}.nodes WHERE node_id = ?",
-                [doc_id],
-            )
+            graphs = graphs_of.get((doc_id,), [])
             graph, reason = _decide(
                 graphs,
                 resolver,
@@ -346,34 +378,44 @@ def place_documents(
                 missing_reason="no_node",
             )
             if graph is not None:
-                if not dry_run:
+                placed.setdefault(graph, []).append(doc_id)
+                report.place(graph)
+            else:
+                moved.setdefault(reason, []).append(doc_id)
+                report.quarantine(reason, doc_id)
+        if not dry_run:
+            for graph, ids in placed.items():
+                for chunk in _chunks(ids):
                     _write(
                         cursor,
                         f"UPDATE {schema}.docs SET graph_id = ? "
-                        "WHERE id = ? AND graph_id IS NULL",
-                        [graph, doc_id],
+                        f"WHERE graph_id IS NULL AND id IN ({_marks(len(chunk))})",
+                        [graph, *chunk],
                     )
-                report.place(graph)
-                continue
-            if not dry_run:
-                # The row is *moved*: the copy carries the text, and only then is the
-                # original removed. A pass killed between the two statements leaves the
-                # row in both tables, which the duplicate tolerance below resolves on
-                # the next run — the order that cannot lose a document.
-                _write(
-                    cursor,
-                    f"INSERT INTO {schema}.docs_quarantine "
-                    "(doc_id, text, reason, quarantined_at) "
-                    "VALUES (?, ?, ?, CURRENT_TIMESTAMP)",
-                    [doc_id, text, reason],
-                    duplicate_is_done=True,
-                )
-                _write(
-                    cursor,
-                    f"DELETE FROM {schema}.docs WHERE id = ? AND graph_id IS NULL",
-                    [doc_id],
-                )
-            report.quarantine(reason, doc_id)
+                    _commit(conn)
+            for reason, ids in moved.items():
+                for chunk in _chunks(ids):
+                    marks = _marks(len(chunk))
+                    # The row is *moved*: the copy carries the text, and only then is the
+                    # original removed. A pass killed between the two statements leaves the
+                    # row in both tables; NOT EXISTS makes the copy idempotent, so the next
+                    # run finishes the delete -- the order that cannot lose a document.
+                    _write(
+                        cursor,
+                        f"INSERT INTO {schema}.docs_quarantine "
+                        "(doc_id, text, reason, quarantined_at) "
+                        "SELECT id, text, ?, CURRENT_TIMESTAMP "
+                        f"FROM {schema}.docs d WHERE d.graph_id IS NULL AND d.id IN ({marks}) "
+                        f"AND NOT EXISTS (SELECT 1 FROM {schema}.docs_quarantine q "
+                        "WHERE q.doc_id = d.id)",
+                        [reason, *chunk],
+                    )
+                    _write(
+                        cursor,
+                        f"DELETE FROM {schema}.docs WHERE graph_id IS NULL AND id IN ({marks})",
+                        list(chunk),
+                    )
+                    _commit(conn)
         if not dry_run:
             _commit(conn)
         return report.report()
@@ -420,25 +462,58 @@ def place_edge_vectors(
         done = _quarantined_keys(cursor, f"SELECT s, p, o_id FROM {staging}")
         done |= _quarantined_keys(cursor, f"SELECT s, p, o_id, reason FROM {quarantine}")
         rows = _read(cursor, f"SELECT s, p, o_id FROM {source} ORDER BY s, p, o_id")
+        graphs_of = _graph_map(
+            cursor,
+            f"SELECT DISTINCT r.s, r.p, r.o_id, r.graph_id FROM {schema}.rdf_edges r "
+            f"JOIN {source} e ON e.s = r.s AND e.p = r.p AND e.o_id = r.o_id",
+            3,
+        )
         report = _Tally()
+        each_row: List[Tuple[str, str, str]] = []  # decided only by a resolver: one statement each
         for row in rows:
             key = (str(row[0]), str(row[1]), str(row[2]))
             if key in done:
                 continue
-            graphs = _graphs_of(
-                cursor,
-                f"SELECT DISTINCT graph_id FROM {schema}.rdf_edges "
-                "WHERE s = ? AND p = ? AND o_id = ?",
-                list(key),
-            )
+            graphs = graphs_of.get(key, [])
             graph, reason = _decide(
                 graphs,
                 resolver,
                 lambda: AmbiguousEdgeVector(*key, candidate_graphs=tuple(graphs)),
                 missing_reason="no_edge",
             )
+            if len(graphs) > 1:
+                each_row.append((key, graph, reason))
             if graph is not None:
-                if not dry_run:
+                report.place(graph)
+            else:
+                report.quarantine(reason, "|".join(key))
+        if not dry_run:
+            # Edge vectors asserted by exactly one graph, and those no edge asserts, move in one
+            # statement each: the join carries the decision the loop above reached for them.
+            _write(
+                cursor,
+                f"INSERT INTO {staging} (graph_id, s, p, o_id, emb, metadata) "
+                "SELECT COALESCE(r.graph_id, ''), e.s, e.p, e.o_id, e.emb, NULL "
+                f"FROM {source} e JOIN ("
+                "SELECT s, p, o_id, MIN(graph_id) AS graph_id "
+                f"FROM {schema}.rdf_edges GROUP BY s, p, o_id HAVING COUNT(DISTINCT graph_id) = 1"
+                ") r ON r.s = e.s AND r.p = e.p AND r.o_id = e.o_id "
+                f"WHERE NOT EXISTS (SELECT 1 FROM {staging} t "
+                "WHERE t.s = e.s AND t.p = e.p AND t.o_id = e.o_id)",
+                [],
+            )
+            _write(
+                cursor,
+                f"INSERT INTO {quarantine} (s, p, o_id, reason, emb, quarantined_at) "
+                "SELECT e.s, e.p, e.o_id, 'no_edge', e.emb, CURRENT_TIMESTAMP "
+                f"FROM {source} e WHERE NOT EXISTS (SELECT 1 FROM {schema}.rdf_edges r "
+                "WHERE r.s = e.s AND r.p = e.p AND r.o_id = e.o_id) "
+                f"AND NOT EXISTS (SELECT 1 FROM {quarantine} q "
+                "WHERE q.s = e.s AND q.p = e.p AND q.o_id = e.o_id)",
+                [],
+            )
+            for key, graph, reason in each_row:
+                if graph is not None:
                     _write(
                         cursor,
                         f"INSERT INTO {staging} (graph_id, s, p, o_id, emb, metadata) "
@@ -447,19 +522,16 @@ def place_edge_vectors(
                         [graph, *key],
                         duplicate_is_done=True,
                     )
-                report.place(graph)
-                continue
-            if not dry_run:
-                _write(
-                    cursor,
-                    f"INSERT INTO {quarantine} "
-                    "(s, p, o_id, reason, emb, quarantined_at) "
-                    "SELECT s, p, o_id, ?, emb, CURRENT_TIMESTAMP "
-                    f"FROM {source} WHERE s = ? AND p = ? AND o_id = ?",
-                    [reason, *key],
-                    duplicate_is_done=True,
-                )
-            report.quarantine(reason, "|".join(key))
+                else:
+                    _write(
+                        cursor,
+                        f"INSERT INTO {quarantine} "
+                        "(s, p, o_id, reason, emb, quarantined_at) "
+                        "SELECT s, p, o_id, ?, emb, CURRENT_TIMESTAMP "
+                        f"FROM {source} WHERE s = ? AND p = ? AND o_id = ?",
+                        [reason, *key],
+                        duplicate_is_done=True,
+                    )
         if not dry_run:
             _commit(conn)
         return report.report()
@@ -569,6 +641,31 @@ def _read(cursor, sql: str, params=None) -> List[tuple]:
     return list(cursor.fetchall() or [])
 
 
+#: Rows per statement when a placement is written; see graph_scoped_embeddings.BATCH_ROWS.
+BATCH_ROWS = 400
+
+
+def _chunks(items, size: Optional[int] = None):
+    size = size or BATCH_ROWS  # read at call time, so a test can shrink the batch
+    for start in range(0, len(items), size):
+        yield items[start : start + size]
+
+
+def _marks(count: int) -> str:
+    return ", ".join("?" * count)
+
+
+def _graph_map(cursor, sql: str, width: int) -> Dict[tuple, List[str]]:
+    """``{key: sorted graphs}`` from one read of ``key columns..., graph_id`` rows."""
+    found: Dict[tuple, set] = {}
+    for row in _read(cursor, sql):
+        if row:
+            found.setdefault(tuple(str(v) for v in row[:width]), set()).add(
+                "" if row[width] is None else str(row[width])
+            )
+    return {key: sorted(graphs) for key, graphs in found.items()}
+
+
 def _graphs_of(cursor, sql: str, params) -> List[str]:
     rows = _read(cursor, sql, params)
     return sorted({("" if r[0] is None else str(r[0])) for r in rows if r})