iris-vector-graph 4.4.2__tar.gz → 4.5.1__tar.gz

{iris_vector_graph-4.4.2 → iris_vector_graph-4.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: iris-vector-graph
-Version: 4.4.2
+Version: 4.5.1
 Summary: Native graph database layer for InterSystems IRIS — openCypher, hybrid vector search, temporal property graph, RDF export, SHACL validation
 Project-URL: Homepage, https://github.com/intersystems-community/iris-vector-graph
 Project-URL: Documentation, https://github.com/intersystems-community/iris-vector-graph/tree/main/docs
@@ -264,7 +264,7 @@ and [docs/performance/GRAPH_ALGORITHMS.md](docs/performance/GRAPH_ALGORITHMS.md)
 └─────────────────────────────────────────────────────────────────────┘
 ```
 
-Full schema and ObjectScript class reference: [docs/architecture/ARCHITECTURE.md](docs/architecture/ARCHITECTURE.md).
+The Rust tier is used for an algorithm only when the loaded library passes IVG's reader gate for it; as of arno-callout 1.2.0, pagerank, ppr, wcc, cdlp and subgraph do not, and answer from ObjectScript ([KNOWN_ISSUES](docs/KNOWN_ISSUES.md), "Arno callout defects"). Full schema and ObjectScript class reference: [docs/architecture/ARCHITECTURE.md](docs/architecture/ARCHITECTURE.md).
 
 ---
 

{iris_vector_graph-4.4.2 → iris_vector_graph-4.5.1}/README.md

@@ -162,7 +162,7 @@ and [docs/performance/GRAPH_ALGORITHMS.md](docs/performance/GRAPH_ALGORITHMS.md)
 └─────────────────────────────────────────────────────────────────────┘
 ```
 
-Full schema and ObjectScript class reference: [docs/architecture/ARCHITECTURE.md](docs/architecture/ARCHITECTURE.md).
+The Rust tier is used for an algorithm only when the loaded library passes IVG's reader gate for it; as of arno-callout 1.2.0, pagerank, ppr, wcc, cdlp and subgraph do not, and answer from ObjectScript ([KNOWN_ISSUES](docs/KNOWN_ISSUES.md), "Arno callout defects"). Full schema and ObjectScript class reference: [docs/architecture/ARCHITECTURE.md](docs/architecture/ARCHITECTURE.md).
 
 ---
 

{iris_vector_graph-4.4.2 → iris_vector_graph-4.5.1}/iris_src/src/Graph/KG/Adjacency.cls

@@ -528,7 +528,18 @@ ClassMethod Indexed() As %Boolean
 /// key (0 for the default graph). With pTrustIndex the SQL is skipped when ^KG already has the triple,
 /// which the index keeps in step with the rows: further timestamps of one triple cost a $Data. The
 /// upgrade backfill passes 0, because there ^KG may hold a pre-ADR-0009 shadow with no row behind it.
-ClassMethod AssertTemporalRow(pGKey, pSrc As %String, pPred As %String, pDst As %String, pWeight As %String = "1.0", pTrustIndex As %Boolean = 1) As %Status
+///
+/// pNodes (by reference) is the caller's memory of the endpoint nodes it has already seen in this graph:
+/// pNodes(id) set means "the row exists, do not insert it again". A batch that writes many edges from a few
+/// sources passes one array and pays for each source's INSERT once. The memory can be stale, because a
+/// purge that retires bare nodes (Graph.KG.Eraser.RetireBareNodes) or any other delete can remove a node in
+/// between; the edge INSERT then fails its foreign key (SQLCODE -121), the entries are dropped and the
+/// nodes and the edge are written again, up to three times. Rows and ^KG are the same as without pNodes.
+///
+/// Embedded SQL: measured at 55 us for the three statements against 129 us as three %ExecDirect (a dynamic
+/// statement is prepared on every call), with the same rows. The remaining ~50 us is the foreign keys and the
+/// two indexes the edge row maintains.
+ClassMethod AssertTemporalRow(pGKey, pSrc As %String, pPred As %String, pDst As %String, pWeight As %String = "1.0", pTrustIndex As %Boolean = 1, ByRef pNodes) As %Status
 {
 	If (pTrustIndex) && ($Data(^KG("out", pGKey, pSrc, pPred, pDst))) {
 		Quit $$$OK
@@ -537,24 +548,35 @@ ClassMethod AssertTemporalRow(pGKey, pSrc As %String, pPred As %String, pDst As
 	Try {
 		// $Char(0) is SQL's '': a bound "" arrives as NULL, which graph_id refuses (the default graph is '')
 		Set tGraphId = $Select(pGKey = 0: $Char(0), 1: pGKey)
-		Set tNodeSQL = "INSERT INTO Graph_KG.nodes (node_id, graph_id) SELECT ?, ? WHERE NOT EXISTS "
-		Set tNodeSQL = tNodeSQL_"(SELECT 1 FROM Graph_KG.nodes WHERE node_id = ? AND COALESCE(graph_id, '') = COALESCE(?, ''))"
-		For tNode = pSrc, pDst {
-			Set tRS = ##class(%SQL.Statement).%ExecDirect(, tNodeSQL, tNode, tGraphId, tNode, tGraphId)
-			If tRS.%SQLCODE < 0 {
-				$$$ThrowStatus($$$ERROR($$$GeneralError, "AssertTemporalRow node "_tNode_": SQLCODE "_tRS.%SQLCODE_" "_tRS.%Message))
-			}
-		}
 		Set tQuals = ##class(%DynamicObject).%New()
 		Do tQuals.%Set("temporal", 1, "boolean")
 		If (pWeight '= "") && (+pWeight '= 1) {
 			Do tQuals.%Set("weight", +pWeight, "number")
 		}
-		Set tEdgeSQL = "INSERT INTO Graph_KG.rdf_edges (s, p, o_id, graph_id, qualifiers) SELECT ?, ?, ?, ?, ? WHERE NOT EXISTS "
-		Set tEdgeSQL = tEdgeSQL_"(SELECT 1 FROM Graph_KG.rdf_edges WHERE s = ? AND p = ? AND o_id = ? AND COALESCE(graph_id, '') = COALESCE(?, ''))"
-		Set tRS = ##class(%SQL.Statement).%ExecDirect(, tEdgeSQL, pSrc, pPred, pDst, tGraphId, tQuals.%ToJSON(), pSrc, pPred, pDst, tGraphId)
-		If tRS.%SQLCODE < 0 {
-			$$$ThrowStatus($$$ERROR($$$GeneralError, "AssertTemporalRow edge: SQLCODE "_tRS.%SQLCODE_" "_tRS.%Message))
+		Set tQ = tQuals.%ToJSON()
+		Set tS = pSrc, tP = pPred, tO = pDst, tG = tGraphId
+		For tTry = 1:1:3 {
+			For tNode = pSrc, pDst {
+				If $Data(pNodes(tNode)) {
+					Continue
+				}
+				Set tN = tNode
+				&sql(INSERT INTO Graph_KG.nodes (node_id, graph_id) SELECT :tN, :tG WHERE NOT EXISTS (SELECT 1 FROM Graph_KG.nodes WHERE node_id = :tN AND COALESCE(graph_id, '') = COALESCE(:tG, '')))
+				If SQLCODE < 0 {
+					$$$ThrowStatus($$$ERROR($$$GeneralError, "AssertTemporalRow node "_tNode_": SQLCODE "_SQLCODE_" "_$Get(%msg)))
+				}
+				Set pNodes(tNode) = ""
+			}
+			&sql(INSERT INTO Graph_KG.rdf_edges (s, p, o_id, graph_id, qualifiers) SELECT :tS, :tP, :tO, :tG, :tQ WHERE NOT EXISTS (SELECT 1 FROM Graph_KG.rdf_edges WHERE s = :tS AND p = :tP AND o_id = :tO AND COALESCE(graph_id, '') = COALESCE(:tG, '')))
+			If (SQLCODE = -121) && (tTry < 3) {
+				// a node row this call relied on is gone: it was remembered, or deleted since it was written
+				Kill pNodes(pSrc), pNodes(pDst)
+				Continue
+			}
+			If SQLCODE < 0 {
+				$$$ThrowStatus($$$ERROR($$$GeneralError, "AssertTemporalRow edge: SQLCODE "_SQLCODE_" "_$Get(%msg)))
+			}
+			Quit
 		}
 	} Catch tEx {
 		Set tSC = tEx.AsStatus()

{iris_vector_graph-4.4.2 → iris_vector_graph-4.5.1}/iris_src/src/Graph/KG/Eraser.cls

@@ -235,6 +235,9 @@ ClassMethod EraseGraph(pGraph As %String = "") As %Integer
 		Kill ^KG("deg2p_exact", tKey)
 		// Spec 234: an erased graph is no longer a multigraph.
 		Kill ^IVG.GraphMode(tKey)
+		// ADR-0015: ^IVG.TemporalStructural(tKey, predicate), which predicates are temporal-only, is
+		// configuration and not content, so it is NOT killed here (as the registration in fhir_graphs
+		// outlives an erase): a re-ingest after the erase must stay temporal-only. EraseAll empties it.
 		// Spec 236: cached algorithm answers derived from the erased content.
 		Kill ^IVG.AlgoCache(tKey)
 		// ^KG("labelset") is content-addressed interning of temporal label sets and
@@ -778,6 +781,7 @@ ClassMethod EraseAll() As %Integer
 		Do ..InvalidateDerived(1)
 		Kill ^ArnoKG
 		Kill ^IVG.GraphMode
+		Kill ^IVG.TemporalStructural
 		Kill ^IVG.AlgoCache
 		Kill ^IVG.Ledger("stmt")
 		Kill ^IVG.Ledger("tuple")
@@ -1089,6 +1093,177 @@ ClassMethod EraseFHIRGraphRows(pName As %String, pAll As %Boolean) [ Private ]
 	}
 }
 
+/// Remove the nodes of one graph that are BARE: no edge names them (as source or target), and they hold no
+/// label, property, document or embedding (the two legacy tables and every routed one). The maintenance sweep for
+/// the nodes a temporal workload leaves behind (opsreview, 4.4.1: one per span, for ever).
+///
+/// pGraph       the graph; "" is the default graph. Another graph's row of the same id is not touched.
+/// pBatch       nodes per transaction (default 10000); the list of what to delete is built first, so a
+///              node that stays (see below) is not looked at again in the same call.
+/// pDryRun      1 counts and changes nothing at all: no row, no global, not ^KG("__version").
+/// pMinAgeSec   leave nodes created less than this long ago. A writer that has just inserted its endpoint
+///              nodes and not yet its edge is the only one that can lose a race here, and it loses loudly
+///              (its edge INSERT fails the foreign key); this keeps a sweep off such nodes altogether.
+///
+/// RACE SAFETY. The statement that removes a node carries every bareness test again, and the table's own
+/// constraints decide the rest: fk_edges_source, fk_edges_dest, fk_labels_node and the embedding tables'
+/// foreign keys refuse a DELETE of a node with a dependant (SQLCODE -124), and they see a dependant another
+/// connection has written and not committed. The two child tables with no foreign key (properties,
+/// documents) are covered by the statement's NOT EXISTS only: a writer that finds the node present, and
+/// writes a property after this statement has evaluated it, leaves a property row without a node. Writers
+/// insert the node first, so that needs a node this sweep is already deleting; the age floor keeps the sweep
+/// off every node created since.
+///
+/// A node whose ^KG("out"/"in") entry exists is left (it is not bare, whatever the rows say).
+///
+/// Returns JSON {"graph", "bare", "erased", "kept", "batches", "dry_run"}. bare: nodes found bare;
+/// erased: removed; kept: found bare and left (adjacency present, a dependant appeared, a foreign key).
+/// Throws, after rolling back the batch it was in, when a statement fails; earlier batches stay committed.
+ClassMethod EraseBareNodes(pGraph As %String = "", pBatch As %Integer = 10000, pDryRun As %Boolean = 0, pMinAgeSec As %Integer = 0) As %String
+{
+	Set tKey = ##class(Graph.KG.GraphKey).ForIndex(pGraph)
+	Set tName = ##class(Graph.KG.GraphKey).ForName(pGraph)
+	Set pBatch = $Select(+pBatch < 1: 10000, 1: +pBatch \ 1)
+	Kill tRoute
+	Do ..CollectOrphanRoutes(.tRoute)
+	Set tAge = ""
+	If +pMinAgeSec > 0 {
+		Set tAge = " AND (n.created_at IS NULL OR n.created_at <= DATEADD(ss, -" _ (+pMinAgeSec \ 1) _ ", CURRENT_TIMESTAMP))"
+	}
+	Set tWhere = " FROM Graph_KG.nodes n WHERE COALESCE(n.graph_id, '') = COALESCE(?, '')" _ ..BareClauses(.tRoute) _ tAge
+	Set tBare = 0, tErased = 0, tKept = 0, tBatches = 0
+	Try {
+		If pDryRun {
+			Set tRS = ##class(%SQL.Statement).%ExecDirect(, "SELECT COUNT(*)" _ tWhere, tName)
+			If tRS.%SQLCODE < 0 {
+				$$$ThrowStatus($$$ERROR($$$GeneralError, "EraseBareNodes count: SQLCODE " _ tRS.%SQLCODE _ " " _ tRS.%Message))
+			}
+			If tRS.%Next() {
+				Set tBare = +tRS.%GetData(1)
+			}
+		} Else {
+			Kill ^||IVG.BareNodes
+			Set tRS = ##class(%SQL.Statement).%ExecDirect(, "SELECT n.node_id" _ tWhere, tName)
+			If tRS.%SQLCODE < 0 {
+				$$$ThrowStatus($$$ERROR($$$GeneralError, "EraseBareNodes scan: SQLCODE " _ tRS.%SQLCODE _ " " _ tRS.%Message))
+			}
+			While tRS.%Next() {
+				Set ^||IVG.BareNodes(tRS.%GetData(1)) = ""
+				Set tBare = tBare + 1
+			}
+			Set tErased = ..DeleteBareSet(tKey, tName, $Name(^||IVG.BareNodes), pBatch, .tKept, .tBatches)
+			Kill ^||IVG.BareNodes
+		}
+	} Catch tEx {
+		Kill ^||IVG.BareNodes
+		Throw tEx
+	}
+	Quit {"graph": (tName), "bare": (tBare), "erased": (tErased), "kept": (tKept), "batches": (tBatches), "dry_run": (+pDryRun)}.%ToJSON()
+}
+
+/// Retire the nodes named in @pRoot@(id) from graph pName (index key pKey) that are bare, as EraseBareNodes
+/// defines it; the others stay. For a purge that has just retracted the triples those ids were ends of
+/// (Graph.KG.TemporalIndex.RetireExpired). One transaction per 10,000 ids. Returns the nodes removed.
+ClassMethod RetireBareNodes(pKey, pName As %String, pRoot As %String) As %Integer
+{
+	Quit ..DeleteBareSet(pKey, pName, pRoot, 10000, .tKept, .tBatches)
+}
+
+/// The shared body: delete each id of @pRoot@(id) from graph pName while it is bare, pBatch ids to a
+/// transaction. pKept: ids left. pBatches: transactions run. Returns the number of nodes removed.
+ClassMethod DeleteBareSet(pKey, pName As %String, pRoot As %String, pBatch As %Integer, Output pKept As %Integer, Output pBatches As %Integer) As %Integer [ Private ]
+{
+	Set pKept = 0, pBatches = 0
+	Set tId = $Order(@pRoot@(""))
+	If tId = "" {
+		Quit 0
+	}
+	Kill tRoute
+	Do ..CollectOrphanRoutes(.tRoute)
+	Set tDel = ##class(%SQL.Statement).%New()
+	$$$ThrowOnError(tDel.%Prepare("DELETE FROM Graph_KG.nodes AS n WHERE n.node_id = ? AND COALESCE(n.graph_id, '') = COALESCE(?, '')" _ ..BareClauses(.tRoute)))
+	Set tErased = 0, tN = 0
+	Kill tDone
+	Try {
+		TSTART
+		While tId '= "" {
+			// an adjacency entry says the node has an edge, whatever the rows say
+			If $Data(^KG("out", pKey, tId)) || $Data(^KG("in", pKey, tId)) {
+				Set pKept = pKept + 1
+			} Else {
+				Set tRS = tDel.%Execute(tId, pName)
+				If tRS.%SQLCODE = 0 {
+					If tRS.%ROWCOUNT > 0 {
+						Set tErased = tErased + 1, tN = tN + 1, tDone(tId) = ""
+					}
+				} ElseIf tRS.%SQLCODE = 100 {
+					// no row: gone already, or another statement's dependant appeared
+				} ElseIf tRS.%SQLCODE = -124 {
+					// a foreign key found a dependant a writer added since the scan: the node stays
+					Set pKept = pKept + 1
+				} Else {
+					$$$ThrowStatus($$$ERROR($$$GeneralError, "bare node delete of " _ tId _ ": SQLCODE " _ tRS.%SQLCODE _ " " _ tRS.%Message))
+				}
+			}
+			Set tId = $Order(@pRoot@(tId))
+			If (tN >= pBatch) || (tId = "") {
+				TCOMMIT
+				Set pBatches = pBatches + 1
+				Do ..ForgetNodes(pKey, .tDone)
+				Kill tDone
+				Set tN = 0
+				If tId '= "" {
+					TSTART
+				}
+			}
+		}
+	} Catch tEx {
+		If $TLevel > 0 { TROLLBACK }
+		Throw tEx
+	}
+	If tErased > 0 {
+		Do ..InvalidateDerived(0)
+	}
+	Quit tErased
+}
+
+/// The derived entries of nodes that no longer exist: Adjacency.RemoveSource kills the generation, the cached
+/// degrees and the stored counters an older version left (ADR-0011: Adjacency is their only writer), and the
+/// property mirror goes here. A bare node has no edge, so RemoveSource finds no adjacency to remove.
+ClassMethod ForgetNodes(pKey, ByRef pIds) [ Private ]
+{
+	Set tId = ""
+	For {
+		Set tId = $Order(pIds(tId))
+		Quit:tId=""
+		$$$ThrowOnError(##class(Graph.KG.Adjacency).RemoveSource(pKey, tId))
+		Kill ^KG("prop", pKey, tId)
+	}
+}
+
+/// The "has none of these" tests for a node n (alias n, Graph_KG.nodes) as SQL text: no edge either way, no label,
+/// no property, no document, no embedding in the legacy tables or a routed node table.
+ClassMethod BareClauses(ByRef pRoute) As %String [ Private ]
+{
+	Set tEq = "COALESCE(x.graph_id, '') = COALESCE(n.graph_id, '')"
+	Set tC = " AND NOT EXISTS (SELECT 1 FROM Graph_KG.rdf_edges x WHERE x.s = n.node_id AND " _ tEq _ ")"
+	Set tC = tC _ " AND NOT EXISTS (SELECT 1 FROM Graph_KG.rdf_edges x WHERE x.o_id = n.node_id AND " _ tEq _ ")"
+	Set tC = tC _ " AND NOT EXISTS (SELECT 1 FROM Graph_KG.rdf_labels x WHERE x.s = n.node_id AND " _ tEq _ ")"
+	Set tC = tC _ " AND NOT EXISTS (SELECT 1 FROM Graph_KG.rdf_props x WHERE x.s = n.node_id AND " _ tEq _ ")"
+	Set tC = tC _ " AND NOT EXISTS (SELECT 1 FROM Graph_KG.docs x WHERE x.id = n.node_id AND " _ tEq _ ")"
+	Set tC = tC _ " AND NOT EXISTS (SELECT 1 FROM Graph_KG.kg_NodeEmbeddings x WHERE x.node_id = n.node_id AND " _ tEq _ ")"
+	Set tC = tC _ " AND NOT EXISTS (SELECT 1 FROM Graph_KG.kg_NodeEmbeddings_optimized x WHERE x.node_id = n.node_id AND " _ tEq _ ")"
+	Set tT = ""
+	For {
+		Set tT = $Order(pRoute(tT))
+		Quit:tT=""
+		Continue:$Extract(tT, 1, 8)="kg_eemb_"
+		Continue:'##class(%SYSTEM.SQL.Schema).TableExists("Graph_KG." _ tT)
+		Set tC = tC _ " AND NOT EXISTS (SELECT 1 FROM Graph_KG." _ tT _ " x WHERE x.node_id = n.node_id AND " _ tEq _ ")"
+	}
+	Quit tC
+}
+
 /// One error shape for every failed step, so the rollback path has one reason.
 ClassMethod SqlErr(pStore As %String, pCode As %Integer) As %Status [ Private ]
 {

{iris_vector_graph-4.4.2 → iris_vector_graph-4.5.1}/iris_src/src/Graph/KG/GraphStores.cls

@@ -104,6 +104,11 @@ ClassMethod Inventory() As %DynamicArray
 	// ^IVG.GraphMode(gKey)="multi" for each graph that allows parallel edges.
 	Do tStores.%Push(..Entry("^IVG.GraphMode", "global", "subscript 1", 1))
 
+	// --- graph-scoped globals: temporal-only predicates (ADR-0015) -------------
+	// ^IVG.TemporalStructural(gKey, predicate)=0 for each predicate of a graph whose temporal edges imply no
+	// structural row. Configuration: EraseGraph keeps it, EraseAll empties it, a snapshot carries it.
+	Do tStores.%Push(..Entry("^IVG.TemporalStructural", "global", "subscript 1", 1))
+
 	// --- graph-scoped globals: algorithm result cache (spec 236) -----------
 	// ^IVG.AlgoCache(gKey, algorithm, paramHash) = $LB(token, payload). Derived from
 	// the graph and recomputable, so erasing a graph drops its entries (ADR-0004: no

{iris_vector_graph-4.4.2 → iris_vector_graph-4.5.1}/iris_src/src/Graph/KG/NKGAccelAdjacency.cls

@@ -57,8 +57,11 @@ ClassMethod ExportAdjacencyNKG() As %String [ Private ]
     If chunkNum > 0 {
         Set n = $Get(^NKG("$meta", "nodeCount"), 0)
         Set nodemap = "NODEMAP:" _ n _ $Char(10)
+        // n bounds the indices; an index with no $ND entry (a gap a rebuild left after
+        // removing a node, spec 260 D-NKG-SWAP) is no node and is not listed
         For i = 0:1:n-1 {
-            Set nodemap = nodemap _ i _ "=" _ $Get(^NKG("$ND", i), "") _ $Char(10)
+            Continue:'$Data(^NKG("$ND", i))
+            Set nodemap = nodemap _ i _ "=" _ ^NKG("$ND", i) _ $Char(10)
         }
         Set nodemap = nodemap _ "---" _ $Char(10)
         Set ^||adj = chunkNum + 1

{iris_vector_graph-4.4.2 → iris_vector_graph-4.5.1}/iris_src/src/Graph/KG/NKGAccelCentrality.cls

@@ -111,11 +111,23 @@ ClassMethod EigenvectorGlobal(maxIter As %Integer = 30, tol As %Float = 0.0001,
         Set n = n + 1
     }
     If n = 0 Quit "ERROR:no nodes"
-    For i = 0:1:n-1 { Set ^||x(i) = 1/n }
+    // Iterate the node list, not 0..n-1: indices are stable and can have gaps below
+    // $meta("nodeCount") once a rebuild has removed nodes (spec 260 D-NKG-SWAP).
+    Set i = ""
+    For {
+        Set i = $Order(^||nodeList(i))
+        Quit:i=""
+        Set ^||x(i) = 1/n
+    }
 
     For iter = 1:1:maxIter {
         Set normSq = 0
-        For i = 0:1:n-1 { Set ^||xNew(i) = 0 }
+        Set i = ""
+        For {
+            Set i = $Order(^||nodeList(i))
+            Quit:i=""
+            Set ^||xNew(i) = 0
+        }
         Set src = ""
         For {
             Set src = $Order(^NKG("$NI",src))
@@ -133,11 +145,19 @@ ClassMethod EigenvectorGlobal(maxIter As %Integer = 30, tol As %Float = 0.0001,
                 Set negP = $Order(^NKG(-1,sIdx,negP))
             }
         }
-        For i = 0:1:n-1 { Set normSq = normSq + ($Get(^||xNew(i),0) * $Get(^||xNew(i),0)) }
+        Set i = ""
+        For {
+            Set i = $Order(^||nodeList(i))
+            Quit:i=""
+            Set normSq = normSq + ($Get(^||xNew(i),0) * $Get(^||xNew(i),0))
+        }
         If normSq <= 0 Quit
         Set norm = $ZSqr(normSq)
         Set maxDelta = 0
-        For i = 0:1:n-1 {
+        Set i = ""
+        For {
+            Set i = $Order(^||nodeList(i))
+            Quit:i=""
             Set newVal = $Get(^||xNew(i),0) / norm
             Set delta = $ZAbs(newVal - $Get(^||x(i),0))
             If delta > maxDelta Set maxDelta = delta
@@ -508,8 +528,6 @@ ClassMethod ClosenessGlobalMSBFS(formula As %String = "harmonic", direction As %
 {
     Set firstNode = $Order(^NKG("$NI",""))
     If firstNode = "" Quit "ERROR:^NKG not built"
-    Set n = $Get(^NKG("$meta","nodeCount"), 0)
-    If n = 0 Quit "ERROR:^NKG nodeCount=0"
 
     Kill ^||msAdj, ^||msVisit, ^||msNext, ^||msSeen, ^||msFar, ^||msReach, ^||msIdxNid
     Set adjDir = $Select(direction="in": -2, 1: -1)
@@ -529,6 +547,19 @@ ClassMethod ClosenessGlobalMSBFS(formula As %String = "harmonic", direction As %
             Set negP = $Order(^NKG(adjDir,vIdx,negP))
         }
     }
+    // n is the number of nodes, counted. $meta("nodeCount") is the interning high-water mark:
+    // indices are stable and never reused, so after a rebuild has removed nodes it is larger
+    // than the node count and "reached = n-1" below would never hold (spec 260 D-NKG-SWAP).
+    Set n = 0, tIx = ""
+    For {
+        Set tIx = $Order(^||msIdxNid(tIx))
+        Quit:tIx=""
+        Set n = n + 1
+    }
+    If n = 0 {
+        Kill ^||msAdj, ^||msIdxNid
+        Quit "ERROR:^NKG nodeCount=0"
+    }
 
     Set sIdx = ""
     For {

{iris_vector_graph-4.4.2 → iris_vector_graph-4.5.1}/iris_src/src/Graph/KG/TemporalIndex.cls

@@ -77,7 +77,7 @@ ClassMethod InsertEdge(graphId As %String = "", source As %String, predicate As
 /// Known limitation (spec-221 FR-004/FR-011): "update" overwrites the weight in
 /// ^KG("tout") and ^KG("tin") but does not adjust the ^KG("tagg") bucket
 /// aggregates or the HLL sketch, which still carry the original weight.
-ClassMethod WriteStores(pGKey, pTs As %Integer, pS As %String, pP As %String, pO As %String, pW As %Double, pSuppR As %Boolean = 0, pMode As %String = "insert", pAttrs = "") As %Integer [ Private ]
+ClassMethod WriteStores(pGKey, pTs As %Integer, pS As %String, pP As %String, pO As %String, pW As %Double, pSuppR As %Boolean = 0, pMode As %String = "insert", pAttrs = "", ByRef pNodes) As %Integer [ Private ]
 {
 	If pMode = "update" {
 		Kill ^KG("edgeprop", pGKey, pTs, pS, pP, pO)
@@ -105,10 +105,13 @@ ClassMethod WriteStores(pGKey, pTs As %Integer, pS As %String, pP As %String, pO
 	// and PutShadow moves the degree cache's generation only for a new one.
 	// ADR-0009 with the adjacency functional index (spec 252): the temporal edge asserts its marked
 	// structural row, and the row's index writes ^KG; a shadow with no row would not survive a rebuild.
-	If ##class(Graph.KG.Adjacency).Indexed() {
-		$$$ThrowOnError(##class(Graph.KG.Adjacency).AssertTemporalRow(pGKey, pS, pP, pO, pW))
-	} Else {
-		Do ##class(Graph.KG.Adjacency).PutShadow(pGKey, pS, pP, pO, pW)
+	// ADR-0015: a predicate registered temporal-only in this graph gets the temporal stores and nothing structural.
+	If ##class(Graph.KG.TemporalPolicy).IsStructuralKey(pGKey, pP) {
+		If ##class(Graph.KG.Adjacency).Indexed() {
+			$$$ThrowOnError(##class(Graph.KG.Adjacency).AssertTemporalRow(pGKey, pS, pP, pO, pW, 1, .pNodes))
+		} Else {
+			Do ##class(Graph.KG.Adjacency).PutShadow(pGKey, pS, pP, pO, pW)
+		}
 	}
 	// count and sum are atomic via $Increment
 	Set tmp = $Increment(^KG("tagg", pGKey, tBucket, pS, pP, "count"))
@@ -151,6 +154,9 @@ ClassMethod BulkInsertCounts(graphId As %String = "", batchJSON As %String, upse
 	Set arr = ##class(%DynamicArray).%FromJSON(batchJSON)
 	Set n = arr.%Size()
 	Set tWritten = 0, tCollided = 0, tSkipped = 0
+	// the endpoint nodes this batch has written a row for: a source shared by many edges is inserted once
+	// (Adjacency.AssertTemporalRow recovers if one is deleted meanwhile)
+	Kill tNodes
 	// Resolved once for the batch, the same way InsertEdge resolves it per edge.
 	If mode = "" { Set mode = $Select(upsert: "update", 1: "insert") }
 	For i = 0:1:(n-1) {
@@ -164,7 +170,7 @@ ClassMethod BulkInsertCounts(graphId As %String = "", batchJSON As %String, upse
 		If ts = 0 Set ts = ##class(%SYSTEM.SQL).DATEDIFF("ss", "1970-01-01 00:00:00", $ZDateTime($ZTimeStamp, 3, 1))
 		Set attrs = item.%Get("attrs")
 		If '$IsObject(attrs) { Set attrs = "" }
-		Set tRC = ..WriteStores(tGKey, ts, source, predicate, target, weight, +item.%Get("sri"), mode, attrs)
+		Set tRC = ..WriteStores(tGKey, ts, source, predicate, target, weight, +item.%Get("sri"), mode, attrs, .tNodes)
 		If tRC = 1 {
 			Set tWritten = tWritten + 1
 		} ElseIf tRC = -1 {
@@ -219,16 +225,40 @@ ClassMethod HLLAdd(ByRef pHLL, pValue As %String) [ Private ]
 
 /// ADR-0009 upgrade: give every temporal triple that has no structural row a marked one (spec 252).
 /// Walks ^KG("tout", graph, ts, s, p, o) once and asserts each distinct triple's row without trusting
-/// ^KG("out"), which on an older install holds row-less shadows. Returns the number of triples visited.
+/// ^KG("out"), which on an older install holds row-less shadows. A predicate registered temporal-only in
+/// its graph (ADR-0015) is skipped: an application that registers before the upgrade pays for none of
+/// its rows. Returns the number of triples asserted.
 /// Callable from SQL as SELECT Graph_KG.backfill_temporal_rows().
 ClassMethod BackfillStructuralRows() As %Integer [ SqlName = backfill_temporal_rows, SqlProc ]
 {
-	Set tN = 0
-	Set tG = ""
+	Quit ..BackfillWalk("", 0, "", 100000).asserted
+}
+
+/// BackfillStructuralRows for one predicate of one graph (ADR-0015's way back from temporal-only, via
+/// Graph.KG.TemporalPolicy.BackfillRows). pPredicate "" means every predicate of the graph; a registered
+/// temporal-only predicate is skipped whatever is asked. pSeenLimit bounds the memory of triples already
+/// asserted: a triple forgotten past it is asserted again, which is idempotent and costs one existence
+/// check, where remembering every distinct triple took memory proportional to the graph (6.79 million for
+/// opsreview). Returns JSON {"asserted", "seen_peak", "skipped_optout"}.
+ClassMethod BackfillRows(pGraph As %String = "", pPredicate As %String = "", pSeenLimit As %Integer = 100000) As %String
+{
+	Quit ..BackfillWalk(##class(Graph.KG.GraphKey).ForIndex(pGraph), 1, pPredicate, pSeenLimit).%ToJSON()
+}
+
+/// The walk behind both. pOneGraph 1: only graph key pKey; 0: every graph.
+ClassMethod BackfillWalk(pKey, pOneGraph As %Boolean, pPredicate As %String, pSeenLimit As %Integer) As %DynamicObject [ Private ]
+{
+	Set pSeenLimit = $Select(+pSeenLimit < 1: 100000, 1: +pSeenLimit \ 1)
+	Set tN = 0, tPeak = 0, tOptOut = 0, tSeenN = 0
+	Set tG = $Select(pOneGraph: $Order(^KG("tout", pKey), -1), 1: "")
 	For {
 		Set tG = $Order(^KG("tout", tG))
 		Quit:tG=""
-		Kill tSeen
+		If pOneGraph && (tG '= pKey) {
+			Quit
+		}
+		Kill tSeen, tNodes
+		Set tSeenN = 0
 		Set tTs = ""
 		For {
 			Set tTs = $Order(^KG("tout", tG, tTs))
@@ -241,20 +271,34 @@ ClassMethod BackfillStructuralRows() As %Integer [ SqlName = backfill_temporal_r
 				For {
 					Set tP = $Order(^KG("tout", tG, tTs, tS, tP))
 					Quit:tP=""
+					If (pPredicate '= "") && (tP '= pPredicate) {
+						Continue
+					}
+					If '##class(Graph.KG.TemporalPolicy).IsStructuralKey(tG, tP) {
+						Set tOptOut = tOptOut + 1
+						Continue
+					}
 					Set tO = ""
 					For {
 						Set tO = $Order(^KG("tout", tG, tTs, tS, tP, tO), 1, tW)
 						Quit:tO=""
 						Continue:$Data(tSeen(tS, tP, tO))
-						Set tSeen(tS, tP, tO) = ""
-						$$$ThrowOnError(##class(Graph.KG.Adjacency).AssertTemporalRow(tG, tS, tP, tO, tW, 0))
+						If tSeenN >= pSeenLimit {
+							Kill tSeen
+							Set tSeenN = 0
+						}
+						Set tSeen(tS, tP, tO) = "", tSeenN = tSeenN + 1
+						If tSeenN > tPeak { Set tPeak = tSeenN }
+						$$$ThrowOnError(##class(Graph.KG.Adjacency).AssertTemporalRow(tG, tS, tP, tO, tW, 0, .tNodes))
 						Set tN = tN + 1
+						// bounded: a source repeats, a span id does not
+						If tN # 5000 = 0 { Kill tNodes }
 					}
 				}
 			}
 		}
 	}
-	Quit tN
+	Quit {"asserted": (tN), "seen_peak": (tPeak), "skipped_optout": (tOptOut)}
 }
 
 /// Remove one timestamped edge, and everything InsertEdge derived from it.
@@ -767,14 +811,15 @@ ClassMethod Purge()
 }
 
 /// Legacy method — purges raw edges and expired aggregate buckets for default graph.
-/// Retires the structural rows only purged timestamps held up, as PurgeRawBefore does.
-ClassMethod PurgeBefore(tsEnd As %Integer)
+/// Retires the structural rows only purged timestamps held up, as PurgeRawBefore does. With pRetireNodes
+/// it also deletes the endpoint nodes that leaves bare (see PurgeRawBefore); returns how many.
+ClassMethod PurgeBefore(tsEnd As %Integer, pRetireNodes As %Boolean = 0) As %Integer
 {
 	Set tGKey = 0  // default graph
 	Set tsEnd = +tsEnd
 	Set tBucketDiv = $Select(..#TSUNIT="ms": ..#BUCKETMS, 1: ..#BUCKET)
 	Set maxSafeBucket = (tsEnd \ tBucketDiv) - 1
-	Do ..RetireExpired(tGKey, "", tsEnd)
+	Do ..RetireExpired(tGKey, "", tsEnd, pRetireNodes, .tRetired, "")
 	Set ts = ""
 	For {
 		Set ts = $Order(^KG("tout", tGKey, ts))
@@ -805,6 +850,7 @@ ClassMethod PurgeBefore(tsEnd As %Integer)
 		Kill ^KG("tout", tGKey, ts)
 	}
 	Do ..RelatestRange(tGKey, "", tsEnd)
+	Return +$Get(tRetired)
 }
 
 /// Delete raw temporal edges in [tsStart, tsEnd) for the specified graph.
@@ -813,14 +859,16 @@ ClassMethod PurgeBefore(tsEnd As %Integer)
 ///
 /// Spec 254 (FR-001, FR-002): the structural row of a (source, predicate, target) whose every
 /// timestamp is purged goes too, with its ^KG("out"/"in") entries, while the row still carries
-/// the temporal marker (ADR-0009, amended): a row a non-temporal write asserted stays. Nodes stay.
-/// See RetireExpired.
-ClassMethod PurgeRawBefore(graphId As %String = "", tsEnd As %Integer, tsStart As %Integer = 0) As %String
+/// the temporal marker (ADR-0009, amended): a row a non-temporal write asserted stays. Nodes stay, unless
+/// pRetireNodes is 1: then each endpoint of a retired triple that is left bare in this graph (no edge, label,
+/// property, document or embedding) is deleted too, by Graph.KG.Eraser.RetireBareNodes, and the result is
+/// "deleted:skipped:retired". See RetireExpired.
+ClassMethod PurgeRawBefore(graphId As %String = "", tsEnd As %Integer, tsStart As %Integer = 0, pRetireNodes As %Boolean = 0) As %String
 {
 	Set tGKey = ##class(Graph.KG.GraphKey).ForIndex(graphId)
 	Set tsEnd = +tsEnd
 	Set tsStart = $Select(+tsStart < 0: 0, 1: +tsStart)
-	Do ..RetireExpired(tGKey, tsStart, tsEnd)
+	Do ..RetireExpired(tGKey, tsStart, tsEnd, pRetireNodes, .tRetired, graphId)
 	Set deleted = 0
 	Set skipped = 0
 	Set ts = ""
@@ -868,7 +916,7 @@ ClassMethod PurgeRawBefore(graphId As %String = "", tsEnd As %Integer, tsStart A
 		Kill ^KG("tout", tGKey, ts)
 	}
 	Do:deleted ..RelatestRange(tGKey, tsStart, tsEnd)
-	Return deleted _ ":" _ skipped
+	Return deleted _ ":" _ skipped _ $Select(pRetireNodes: ":" _ +$Get(tRetired), 1: "")
 }
 
 /// Spec 254: retire the structural rows that only the timestamps in [pTsStart, pTsEnd) hold up,
@@ -887,8 +935,15 @@ ClassMethod PurgeRawBefore(graphId As %String = "", tsEnd As %Integer, tsStart A
 /// Rows first, raw entries after (the caller's kill loop): a purge that fails part way
 /// leaves timestamps without rows, and running it again finishes the job. The other order
 /// would leave marked rows that no later purge can find.
-ClassMethod RetireExpired(pGKey, pTsStart, pTsEnd As %Integer) As %Integer [ Private ]
+///
+/// 4. With pRetireNodes, the endpoints of those triples go to Graph.KG.Eraser.RetireBareNodes once their
+///    rows are gone: it deletes the ones nothing else names (pRetired: how many). Only endpoints of a retired
+///    triple are looked at, so a node someone created and never used in a temporal edge is never a candidate.
+///    The nodes go after the rows: a purge that fails between the two leaves bare nodes that no later purge
+///    names (their triples are gone), and Graph.KG.Eraser.EraseBareNodes takes those.
+ClassMethod RetireExpired(pGKey, pTsStart, pTsEnd As %Integer, pRetireNodes As %Boolean = 0, ByRef pRetired As %Integer, pGraphName As %String = "") As %Integer [ Private ]
 {
+	Set pRetired = 0
 	Kill ^||IVG.TPurge
 	Set tLeft = 0
 	Set tTs = ""
@@ -907,6 +962,8 @@ ClassMethod RetireExpired(pGKey, pTsStart, pTsEnd As %Integer) As %Integer [ Pri
 			For {
 				Set tP = $Order(^KG("tout", pGKey, tTs, tS, tP))
 				Quit:tP=""
+				// ADR-0015: a temporal-only predicate has no rows to retract and no endpoints to retire
+				Continue:('##class(Graph.KG.TemporalPolicy).IsStructuralKey(pGKey, tP))
 				Set tO = ""
 				For {
 					Set tO = $Order(^KG("tout", pGKey, tTs, tS, tP, tO))
@@ -960,6 +1017,29 @@ ClassMethod RetireExpired(pGKey, pTsStart, pTsEnd As %Integer) As %Integer [ Pri
 		} Else {
 			$$$ThrowOnError(##class(Graph.KG.Adjacency).RetractTemporalShadows(pGKey, $Name(^||IVG.TPurge)))
 		}
+		If pRetireNodes && ##class(Graph.KG.Adjacency).Indexed() {
+			// the ends of what is left in TPurge: the triples whose rows have just been retracted
+			Kill ^||IVG.TNodes
+			Set tS = ""
+			For {
+				Set tS = $Order(^||IVG.TPurge(tS))
+				Quit:tS=""
+				Set ^||IVG.TNodes(tS) = ""
+				Set tP = ""
+				For {
+					Set tP = $Order(^||IVG.TPurge(tS, tP))
+					Quit:tP=""
+					Set tO = ""
+					For {
+						Set tO = $Order(^||IVG.TPurge(tS, tP, tO))
+						Quit:tO=""
+						Set ^||IVG.TNodes(tO) = ""
+					}
+				}
+			}
+			Set pRetired = ##class(Graph.KG.Eraser).RetireBareNodes(pGKey, ##class(Graph.KG.GraphKey).ForName(pGraphName), $Name(^||IVG.TNodes))
+			Kill ^||IVG.TNodes
+		}
 	}
 	Kill ^||IVG.TPurge
 	Return tLeft