cjm-capability-graph-sqlite 0.0.30__tar.gz → 0.0.31__tar.gz

{cjm_capability_graph_sqlite-0.0.30/cjm_capability_graph_sqlite.egg-info → cjm_capability_graph_sqlite-0.0.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjm-capability-graph-sqlite
-Version: 0.0.30
+Version: 0.0.31
 Summary: A local, file-backed context-graph storage capability for the cjm-substrate runtime that implements graph storage, traversal, and querying using SQLite.
 Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
 License: Apache-2.0

cjm_capability_graph_sqlite-0.0.31/cjm_capability_graph_sqlite/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.31"

{cjm_capability_graph_sqlite-0.0.30 → cjm_capability_graph_sqlite-0.0.31}/cjm_capability_graph_sqlite/capability.py

@@ -235,7 +235,7 @@ class SQLiteGraphCapability(ToolCapability):
             # correlated-probe plans pick the WRONG composite index (the
             # target-side probe scans a whole document's PART_OF rows per
             # outer row: 5s; source-side probe: 8ms at corpus scale). They are
-            # kept by `_writing`'s PRAGMA optimize at the close of every write,
+            # kept by `_writing`'s statistics rule at the close of every write,
             # this one included (a new index is analyzed there) -- no longer a
             # full ANALYZE per open, O(db) on every read (finding ff3ede3d).
 
@@ -261,22 +261,48 @@ class SQLiteGraphCapability(ToolCapability):
         error), then keep the planner's statistics current, then close.
 
         Statistics are written on the paths that change data, never on an open (finding
-        ff3ede3d: a read paid a full ANALYZE and took the write lock). PRAGMA optimize
-        analyzes a table only when it was never analyzed, gained an index, or moved ~10x
-        in size since, so a steady-state write pays a no-op check and a fresh rebuild db
-        gains statistics as it fills. Guarded: statistics are an OPTIMIZATION — a corrupted
-        DB must still answer integrity_check (the designed G3 gate)."""
+        ff3ede3d: a read paid a full ANALYZE and took the write lock): `_refresh_statistics`
+        analyzes a table only when its statistics are missing or stale, so a steady-state write
+        pays a ~0.01 ms check and a fresh rebuild db gains statistics as it fills. Guarded:
+        statistics are an OPTIMIZATION — a corrupted DB must still answer integrity_check (the
+        designed G3 gate)."""
         con = self._connect()
         try:
             with con:
                 yield con
             try:
-                con.execute("PRAGMA optimize=0x10002;")
+                self._refresh_statistics(con)
             except sqlite3.DatabaseError as e:
-                logging.warning(f"PRAGMA optimize skipped (run integrity_check): {e}")
+                logging.warning(f"statistics skipped (run integrity_check): {e}")
         finally:
             con.close()
 
+    def _refresh_statistics(
+        self,
+        con: sqlite3.Connection,  # A write connection, after its unit of work committed
+    ) -> None:
+        """Analyze each graph table whose planner statistics are missing or stale: an index
+        with no sqlite_stat1 row (never analyzed, or created since), or a size ~10x the
+        analyzed one (max(rowid), an O(log n) estimate). One rule on every SQLite: PRAGMA
+        optimize analyzes a never-analyzed table only from SQLite 3.46 on, so on 3.45 (CI's
+        Ubuntu 24.04) the write path kept no statistics at all -- the guard of the 162 s
+        planner regression gone."""
+        for table in ("nodes", "edges"):
+            rows = con.execute(f"SELECT max(rowid) FROM {table}").fetchone()[0] or 0
+            if not rows:
+                continue
+            indexes = {r[0] for r in con.execute(
+                "SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = ?", (table,))}
+            try:
+                stats = dict(con.execute(
+                    "SELECT idx, stat FROM sqlite_stat1 WHERE tbl = ?", (table,)).fetchall())
+            except sqlite3.OperationalError:  # no sqlite_stat1 yet: never analyzed
+                stats = {}
+            analyzed = max((int(str(s).split()[0]) for s in stats.values() if s), default=0)
+            if indexes - set(stats) or rows >= 10 * max(analyzed, 1):
+                con.execute(f"ANALYZE {table};")
+                con.commit()
+
     # -------------------------------------------------------------------------
     # Helpers
     # -------------------------------------------------------------------------

{cjm_capability_graph_sqlite-0.0.30 → cjm_capability_graph_sqlite-0.0.31/cjm_capability_graph_sqlite.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjm-capability-graph-sqlite
-Version: 0.0.30
+Version: 0.0.31
 Summary: A local, file-backed context-graph storage capability for the cjm-substrate runtime that implements graph storage, traversal, and querying using SQLite.
 Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
 License: Apache-2.0

{cjm_capability_graph_sqlite-0.0.30 → cjm_capability_graph_sqlite-0.0.31}/tests/test_capability.py

@@ -140,8 +140,8 @@ def test_sg41_wal_query_guard_merge_readonly(tmp_path):
 
 
 def test_reads_and_opens_never_write_writes_keep_statistics(tmp_path):
-    # Finding ff3ede3d: statistics are written by the paths that change data (PRAGMA
-    # optimize at the close of each write), so an open or a read commits nothing -- an
+    # Finding ff3ede3d: statistics are written by the paths that change data (the
+    # statistics rule at the close of each write), so an open or a read commits nothing -- an
     # observer's data_version moves only when ANOTHER connection commits.
     db = str(tmp_path / "stats.db")
     w = SQLiteGraphCapability()
@@ -167,6 +167,28 @@ def test_reads_and_opens_never_write_writes_keep_statistics(tmp_path):
     observer.close()
 
 
+def test_statistics_follow_growth_on_every_sqlite(tmp_path):
+    # The write path's own rule, never PRAGMA optimize's version-specific one (SQLite < 3.46
+    # never analyzed a fresh table, so CI kept no statistics): the first write analyzes, a
+    # write that leaves the table ~10x its analyzed size re-analyzes, a small one does not.
+    db = str(tmp_path / "grow.db")
+    cap = SQLiteGraphCapability()
+    cap.initialize({"db_path": db})
+
+    def analyzed_rows():
+        stats = sqlite3.connect(db).execute(
+            "SELECT stat FROM sqlite_stat1 WHERE tbl = 'nodes' AND idx = 'idx_nodes_label_id'").fetchone()
+        return int(stats[0].split()[0])
+
+    cap.add_nodes([GraphNode(id=f"a{i:03d}", label="Doc", properties={}) for i in range(5)])
+    assert analyzed_rows() == 5
+    cap.add_nodes([GraphNode(id=f"b{i:03d}", label="Doc", properties={}) for i in range(10)])
+    assert analyzed_rows() == 5      # 15 rows: under 10x, the statistics stand
+    cap.add_nodes([GraphNode(id=f"c{i:03d}", label="Doc", properties={}) for i in range(40)])
+    assert analyzed_rows() == 55     # 55 rows: 10x the analyzed 5, re-analyzed
+    cap.cleanup()
+
+
 def test_typed_surface_smoke(cap):
     doc = str(uuid.uuid4())
     segs = [str(uuid.uuid4()) for _ in range(3)]

cjm_capability_graph_sqlite-0.0.30/cjm_capability_graph_sqlite/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.30"