cjm-capability-graph-sqlite 0.0.29__tar.gz → 0.0.30__tar.gz

{cjm_capability_graph_sqlite-0.0.29/cjm_capability_graph_sqlite.egg-info → cjm_capability_graph_sqlite-0.0.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjm-capability-graph-sqlite
-Version: 0.0.29
+Version: 0.0.30
 Summary: A local, file-backed context-graph storage capability for the cjm-substrate runtime that implements graph storage, traversal, and querying using SQLite.
 Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
 License: Apache-2.0
@@ -15,7 +15,7 @@ Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: cjm-substrate>=0.0.51
-Requires-Dist: cjm-context-graph-primitives>=0.0.12
+Requires-Dist: cjm-context-graph-primitives>=0.0.18
 Dynamic: license-file
 
 # cjm-capability-graph-sqlite

cjm_capability_graph_sqlite-0.0.30/cjm_capability_graph_sqlite/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.30"

{cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.30}/cjm_capability_graph_sqlite/capability.py

@@ -19,8 +19,9 @@ import logging
 import os
 import sqlite3
 import time
+from contextlib import contextmanager
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 
 from cjm_capability_graph_sqlite.query_translation import (decode_projected_row,
                                                            EDGE_STRUCTURAL_KEYS,
@@ -40,6 +41,7 @@ from cjm_substrate.utils.validation import (config_to_dict, dataclass_to_jsonsch
 @dataclass
 class SQLiteGraphCapabilityConfig:
     """Configuration for SQLite Graph Capability."""
+
     db_path: Optional[str] = field(
         default=None,
         metadata={
@@ -173,8 +175,9 @@ class SQLiteGraphCapability(ToolCapability):
         return os.path.join(data_dir, "context_graph.db")
 
     def _init_db(self) -> None:
-        """Create tables and indices."""
-        with self._connect() as con:
+        """Create tables and indices -- the schema only. Statistics are written by the
+        paths that change data (`_writing`), never by an open (finding ff3ede3d)."""
+        with self._writing() as con:
             # Enable Foreign Keys
             con.execute("PRAGMA foreign_keys = ON;")
 
@@ -189,7 +192,11 @@ class SQLiteGraphCapability(ToolCapability):
                     updated_at REAL
                 )
             """)
-            con.execute("CREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(label);")
+            # (label, id): a label read in id order -- a keyset page, a bounded read -- is a
+            # range scan, never a sort of the whole label (findings a107e2f3 / ae2b196e); it
+            # serves a label-only lookup too, so the label-only index it replaces is dropped.
+            con.execute("CREATE INDEX IF NOT EXISTS idx_nodes_label_id ON nodes(label, id);")
+            con.execute("DROP INDEX IF EXISTS idx_nodes_label;")
 
             # Edges Table
             con.execute("""
@@ -227,15 +234,10 @@ class SQLiteGraphCapability(ToolCapability):
             # H7 companion: planner statistics. Without sqlite_stat1 the
             # correlated-probe plans pick the WRONG composite index (the
             # target-side probe scans a whole document's PART_OF rows per
-            # outer row: 5s; source-side probe: 8ms at corpus scale).
-            # Cheap (~40ms at 13k segments); writable opens only. Guarded:
-            # stats are an OPTIMIZATION — a corrupted DB must still LOAD so
-            # integrity_check (the designed G3 gate) can diagnose it, never
-            # fail at initialize via a stats side-effect.
-            try:
-                con.execute("ANALYZE;")
-            except sqlite3.DatabaseError as e:
-                logging.warning(f"ANALYZE skipped (run integrity_check): {e}")
+            # outer row: 5s; source-side probe: 8ms at corpus scale). They are
+            # kept by `_writing`'s PRAGMA optimize at the close of every write,
+            # this one included (a new index is analyzed there) -- no longer a
+            # full ANALYZE per open, O(db) on every read (finding ff3ede3d).
 
     def _connect(self) -> sqlite3.Connection:  # Open DB connection (honors readonly; sets WAL + FKs)
         """Open a SQLite connection honoring the `readonly` config (SG-41).
@@ -253,6 +255,28 @@ class SQLiteGraphCapability(ToolCapability):
         con.execute("PRAGMA foreign_keys = ON;")
         return con
 
+    @contextmanager
+    def _writing(self) -> Iterator[sqlite3.Connection]:  # One unit of write work: commit, planner upkeep, close
+        """Open a connection for one unit of write work: commit on success (roll back on
+        error), then keep the planner's statistics current, then close.
+
+        Statistics are written on the paths that change data, never on an open (finding
+        ff3ede3d: a read paid a full ANALYZE and took the write lock). PRAGMA optimize
+        analyzes a table only when it was never analyzed, gained an index, or moved ~10x
+        in size since, so a steady-state write pays a no-op check and a fresh rebuild db
+        gains statistics as it fills. Guarded: statistics are an OPTIMIZATION — a corrupted
+        DB must still answer integrity_check (the designed G3 gate)."""
+        con = self._connect()
+        try:
+            with con:
+                yield con
+            try:
+                con.execute("PRAGMA optimize=0x10002;")
+            except sqlite3.DatabaseError as e:
+                logging.warning(f"PRAGMA optimize skipped (run integrity_check): {e}")
+        finally:
+            con.close()
+
     # -------------------------------------------------------------------------
     # Helpers
     # -------------------------------------------------------------------------
@@ -321,7 +345,7 @@ class SQLiteGraphCapability(ToolCapability):
         stamping now() here clamped every rebuilt node to rebuild time)."""
         ids = []
         now = time.time()
-        with self._connect() as con:
+        with self._writing() as con:
             for n in nodes:
                 sources_json = json.dumps([s.to_dict() for s in n.sources])
                 props_json = json.dumps(n.properties)
@@ -353,7 +377,7 @@ class SQLiteGraphCapability(ToolCapability):
         ids = []
         now = time.time()
         missing: List[Tuple[str, str, str]] = []  # (source_id, target_id, relation_type) of FK-skipped edges
-        with self._connect() as con:
+        with self._writing() as con:
             con.execute("PRAGMA foreign_keys = ON;")
             for e in edges:
                 props_json = json.dumps(e.properties)
@@ -436,11 +460,13 @@ class SQLiteGraphCapability(ToolCapability):
         label: str,  # Node label to search for
         limit: int = 100  # Max results
     ) -> List[GraphNode]:  # Matching nodes
-        """Find nodes by label."""
+        """Find nodes by label, in id order: a bounded read sorts before it limits, so a
+        page is never an arbitrary subset (finding a107e2f3)."""
         results = []
         with self._connect() as con:
             cur = con.execute(
-                "SELECT id, label, properties, sources, created_at, updated_at FROM nodes WHERE label = ? LIMIT ?",
+                "SELECT id, label, properties, sources, created_at, updated_at FROM nodes "
+                "WHERE label = ? ORDER BY id LIMIT ?",
                 (label, limit)
             )
             for row in cur:
@@ -546,7 +572,7 @@ class SQLiteGraphCapability(ToolCapability):
         to rebuild time)."""
         properties = dict(properties)
         stamp = properties.pop("updated_at", None)
-        with self._connect() as con:
+        with self._writing() as con:
             # Fetch existing to merge
             cur = con.execute("SELECT properties FROM nodes WHERE id = ?", (node_id,))
             row = cur.fetchone()
@@ -574,7 +600,7 @@ class SQLiteGraphCapability(ToolCapability):
         exactly as `update_node` takes it."""
         properties = dict(properties)
         stamp = properties.pop("updated_at", None)
-        with self._connect() as con:
+        with self._writing() as con:
             cur = con.execute("SELECT properties FROM edges WHERE id = ?", (edge_id,))
             row = cur.fetchone()
             if not row:
@@ -599,7 +625,7 @@ class SQLiteGraphCapability(ToolCapability):
         cascade: bool = True  # Also delete connected edges
     ) -> int:  # Number of nodes deleted
         """Delete nodes (and optionally connected edges)."""
-        with self._connect() as con:
+        with self._writing() as con:
             if cascade:
                 con.execute("PRAGMA foreign_keys = ON;")  # Ensures cascade works
             else:
@@ -617,7 +643,7 @@ class SQLiteGraphCapability(ToolCapability):
         edge_ids: List[str]  # UUIDs of edges to delete
     ) -> int:  # Number of edges deleted
         """Delete edges."""
-        with self._connect() as con:
+        with self._writing() as con:
             placeholders = ','.join('?' for _ in edge_ids)
             cur = con.execute(
                 f"DELETE FROM edges WHERE id IN ({placeholders})",
@@ -678,7 +704,7 @@ class SQLiteGraphCapability(ToolCapability):
         created_at / updated_at are honored like add_nodes (0d50b921, design 8f6f2343)."""
         now = time.time()
         written = 0
-        with self._connect() as con:
+        with self._writing() as con:
             for node in nodes:
                 props_json = json.dumps(node.properties)
                 sources_json = json.dumps([s.to_dict() for s in node.sources])
@@ -729,7 +755,7 @@ class SQLiteGraphCapability(ToolCapability):
         created_at / updated_at are honored like add_edges (0d50b921, design 8f6f2343)."""
         now = time.time()
         written = 0
-        with self._connect() as con:
+        with self._writing() as con:
             for edge in edges:
                 props_json = json.dumps(edge.properties)
                 row = con.execute("SELECT properties FROM edges WHERE id = ?", (edge.id,)).fetchone()
@@ -925,7 +951,7 @@ def record_ingest_sources(
     rows it describes swap in together; it is provenance about the db, not a graph row, so
     it is neither stamped by the op clock nor seen by the write observer."""
     rows = sorted((str(k), str(v)) for k, v in dict(sources).items())
-    with self._connect() as con:
+    with self._writing() as con:
         con.execute("DELETE FROM ingest_sources")
         con.executemany("INSERT INTO ingest_sources (source, head) VALUES (?, ?)", rows)
     return len(rows)

{cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.30}/cjm_capability_graph_sqlite/query_translation.py

@@ -171,16 +171,22 @@ EDGE_FULL_COLUMNS = "e.id, e.source_id, e.target_id, e.relation_type, e.properti
 
 
 def _order_limit_sql(
-    query,  # NodeQuery or EdgeQuery (order_by / limit / offset fields)
+    query,  # NodeQuery or EdgeQuery (order_by / limit / offset / after fields)
     props_col: str,  # Properties column expression for ORDER BY paths
+    id_col: str,  # Row id column: the keyset key and the tie-break
     params: List[Any],  # Bound-params list (appended in place)
 ) -> str:  # ORDER BY / LIMIT / OFFSET tail
-    """Compile the shared ordering + paging tail."""
+    """Compile the shared ordering + paging tail. A property ordering breaks ties on the
+    id, and a bounded or keyset read with no ordering is ordered by the id, so a page is
+    never an arbitrary subset (findings a107e2f3 / ae2b196e)."""
     tail = ""
     if query.order_by is not None:
         tail += (f" ORDER BY json_extract({props_col}, "
                  f"{_json_path(query.order_by.prop)})"
-                 + (" DESC" if query.order_by.descending else " ASC"))
+                 + (" DESC" if query.order_by.descending else " ASC")
+                 + f", {id_col}")
+    elif query.limit is not None or query.offset or query.after is not None:
+        tail += f" ORDER BY {id_col}"
     if query.limit is not None or query.offset:
         tail += " LIMIT ?"
         params.append(query.limit if query.limit is not None else -1)
@@ -206,6 +212,9 @@ def translate_node_query(
     if q.ids is not None:
         where.append("n.id IN (SELECT value FROM json_each(?))")
         params.append(json.dumps(list(q.ids)))
+    if q.after is not None:
+        where.append("n.id > ?")
+        params.append(q.after)
     if q.label is not None:
         where.append("n.label = ?")
         params.append(q.label)
@@ -244,11 +253,11 @@ def translate_node_query(
             else:
                 select_parts.append(f"n.properties -> {_json_path(name)}")
                 keys.append(name)
-        tail = _order_limit_sql(q, "n.properties", params)
+        tail = _order_limit_sql(q, "n.properties", "n.id", params)
         return (f"SELECT {', '.join(select_parts)} FROM nodes AS n{where_sql}{tail}",
                 params, "rows", keys)
 
-    tail = _order_limit_sql(q, "n.properties", params)
+    tail = _order_limit_sql(q, "n.properties", "n.id", params)
     return (f"SELECT {NODE_FULL_COLUMNS} FROM nodes AS n{where_sql}{tail}",
             params, "full", None)
 
@@ -268,6 +277,9 @@ def translate_edge_query(
     if q.ids is not None:
         where.append("e.id IN (SELECT value FROM json_each(?))")
         params.append(json.dumps(list(q.ids)))
+    if q.after is not None:
+        where.append("e.id > ?")
+        params.append(q.after)
     if q.relation_type is not None:
         where.append("e.relation_type = ?")
         params.append(q.relation_type)
@@ -312,11 +324,11 @@ def translate_edge_query(
             else:
                 select_parts.append(f"e.properties -> {_json_path(name)}")
                 keys.append(name)
-        tail = _order_limit_sql(q, "e.properties", params)
+        tail = _order_limit_sql(q, "e.properties", "e.id", params)
         return (f"SELECT {', '.join(select_parts)} FROM edges AS e{where_sql}{tail}",
                 params, "rows", keys)
 
-    tail = _order_limit_sql(q, "e.properties", params)
+    tail = _order_limit_sql(q, "e.properties", "e.id", params)
     return (f"SELECT {EDGE_FULL_COLUMNS} FROM edges AS e{where_sql}{tail}",
             params, "full", None)
 

{cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.30/cjm_capability_graph_sqlite.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjm-capability-graph-sqlite
-Version: 0.0.29
+Version: 0.0.30
 Summary: A local, file-backed context-graph storage capability for the cjm-substrate runtime that implements graph storage, traversal, and querying using SQLite.
 Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
 License: Apache-2.0
@@ -15,7 +15,7 @@ Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: cjm-substrate>=0.0.51
-Requires-Dist: cjm-context-graph-primitives>=0.0.12
+Requires-Dist: cjm-context-graph-primitives>=0.0.18
 Dynamic: license-file
 
 # cjm-capability-graph-sqlite

cjm_capability_graph_sqlite-0.0.30/cjm_capability_graph_sqlite.egg-info/requires.txt

@@ -0,0 +1,2 @@
+cjm-substrate>=0.0.51
+cjm-context-graph-primitives>=0.0.18

{cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.30}/pyproject.toml

@@ -12,7 +12,7 @@ license = {text = "Apache-2.0"}
 authors = [{name = "Christian J. Mills", email = "9126128+cj-mills@users.noreply.github.com"}]
 keywords = ['python']
 classifiers = ["Natural Language :: English", "Intended Audience :: Developers", "Development Status :: 3 - Alpha", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only"]
-dependencies = ['cjm-substrate>=0.0.51', 'cjm-context-graph-primitives>=0.0.12']
+dependencies = ['cjm-substrate>=0.0.51', 'cjm-context-graph-primitives>=0.0.18']
 
 [project.urls]
 Repository = "https://github.com/cj-mills/cjm-capability-graph-sqlite"

{cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.30}/tests/test_capability.py

@@ -3,18 +3,17 @@
 Projected from the capability notebook's demo/test cells at the c25780e8 flip:
 the CRUD/round-trip walk, the SG-41 smells (WAL / readonly / guarded query /
 merge_strategy), and the stage-4 typed-surface smoke test."""
+
 import sqlite3
 import uuid
 
 import pytest
-
 from cjm_capability_graph_sqlite.capability import SQLiteGraphCapability
 from cjm_context_graph_primitives.graph import GraphContext, GraphEdge, GraphNode
 from cjm_context_graph_primitives.locators import FileRef
 from cjm_context_graph_primitives.provenance import SourceRef
-from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy,
-                                                PropertyPredicate, RawQuery,
-                                                RelationPredicate)
+from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy, PropertyPredicate,
+                                                RawQuery, RelationPredicate)
 from cjm_context_graph_primitives.slices import FullContent
 from cjm_substrate.core.errors import CapabilityInputError
 
@@ -140,6 +139,34 @@ def test_sg41_wal_query_guard_merge_readonly(tmp_path):
     ro.cleanup()
 
 
+def test_reads_and_opens_never_write_writes_keep_statistics(tmp_path):
+    # Finding ff3ede3d: statistics are written by the paths that change data (PRAGMA
+    # optimize at the close of each write), so an open or a read commits nothing -- an
+    # observer's data_version moves only when ANOTHER connection commits.
+    db = str(tmp_path / "stats.db")
+    w = SQLiteGraphCapability()
+    w.initialize({"db_path": db})
+    indexes = {r[0] for r in sqlite3.connect(db).execute(
+        "SELECT name FROM sqlite_master WHERE type = 'index'")}
+    assert "idx_nodes_label_id" in indexes and "idx_nodes_label" not in indexes
+    w.add_nodes([GraphNode(id=f"n{i:03d}", label="Doc", properties={"i": i}) for i in range(50)])
+    stats = {r[0] for r in sqlite3.connect(db).execute(
+        "SELECT idx FROM sqlite_stat1 WHERE tbl = 'nodes'")}
+    assert "idx_nodes_label_id" in stats
+    w.cleanup()
+
+    observer = sqlite3.connect(db)
+    before = observer.execute("PRAGMA data_version").fetchone()[0]
+    for readonly in (True, False):
+        r = SQLiteGraphCapability()
+        r.initialize({"db_path": db, "readonly": readonly})
+        assert [n.id for n in r.find_nodes_by_label("Doc", limit=3)] == ["n000", "n001", "n002"]
+        assert r.query_nodes(NodeQuery(label="Doc", count=True)).count == 50
+        r.cleanup()
+    assert observer.execute("PRAGMA data_version").fetchone()[0] == before
+    observer.close()
+
+
 def test_typed_surface_smoke(cap):
     doc = str(uuid.uuid4())
     segs = [str(uuid.uuid4()) for _ in range(3)]

{cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.30}/tests/test_query_translation.py

@@ -3,20 +3,19 @@
 Projected from the query_translation notebook's behavior-test cell at the
 c25780e8 flip: the translation contract is verified against the PRODUCTION
 schema in an in-memory DB — by what it returns, not by SQL strings."""
+
 import json
 import sqlite3
 
 import pytest
-
-from cjm_capability_graph_sqlite.query_translation import (EDGE_STRUCTURAL_KEYS,
+from cjm_capability_graph_sqlite.query_translation import (decode_projected_row,
+                                                           EDGE_STRUCTURAL_KEYS,
                                                            NODE_STRUCTURAL_KEYS,
-                                                           decode_projected_row,
                                                            translate_edge_query,
                                                            translate_node_query)
 from cjm_context_graph_primitives.locators import FileRef
-from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy,
-                                                PropertyPredicate, RelationPredicate,
-                                                SourcePredicate)
+from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy, PropertyPredicate,
+                                                RelationPredicate, SourcePredicate)
 from cjm_substrate.core.errors import CapabilityInputError
 
 
@@ -182,6 +181,27 @@ def test_ne_order_desc_limit_and_paging(con):
     assert [r[0] for r in rows] == ["s1", "s2"]
 
 
+def test_keyset_pages_and_deterministic_bounded_order(con):
+    # The keyset cursor (finding ae2b196e): a page is the rows after the last id seen, in
+    # id order, so a pass over the whole answer visits every row exactly once.
+    every = sorted(r["id"] for r in run(con, NodeQuery(label="Segment", project=[])))
+    seen, after = [], None
+    while True:
+        rows = run(con, NodeQuery(label="Segment", project=[], limit=2, after=after))
+        seen.extend(r["id"] for r in rows)
+        if len(rows) < 2:
+            break
+        after = rows[-1]["id"]
+    assert seen == every
+    # A bounded read with no ordering is ordered by id, never an arbitrary subset
+    # (finding a107e2f3); a property ordering breaks its ties on the id.
+    assert [r[0] for r in run(con, NodeQuery(label="Segment", limit=3))] == every[:3]
+    tied = run(con, NodeQuery(label="Segment", where=[PropertyPredicate("index", "eq", 0)],
+                              order_by=OrderBy(prop="index")))
+    assert [r[0] for r in tied] == ["other", "s0"]
+    assert [r[0] for r in run(con, EdgeQuery(relation_type="NEXT", after="n0"))] == ["n1", "nx"]
+
+
 def test_in_op_ids_batch_and_top_level_source(con):
     rows = run(con, NodeQuery(label="Segment", where=[PropertyPredicate("index", "in", [0, 2])],
                               related=RelationPredicate("PART_OF", node_id="doc-1")))

cjm_capability_graph_sqlite-0.0.29/cjm_capability_graph_sqlite/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.29"

cjm_capability_graph_sqlite-0.0.29/cjm_capability_graph_sqlite.egg-info/requires.txt

@@ -1,2 +0,0 @@
-cjm-substrate>=0.0.51
-cjm-context-graph-primitives>=0.0.12