cjm-capability-graph-sqlite 0.0.28__tar.gz → 0.0.30__tar.gz

{cjm_capability_graph_sqlite-0.0.28/cjm_capability_graph_sqlite.egg-info → cjm_capability_graph_sqlite-0.0.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjm-capability-graph-sqlite
-Version: 0.0.28
+Version: 0.0.30
 Summary: A local, file-backed context-graph storage capability for the cjm-substrate runtime that implements graph storage, traversal, and querying using SQLite.
 Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
 License: Apache-2.0
@@ -15,7 +15,7 @@ Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: cjm-substrate>=0.0.51
-Requires-Dist: cjm-context-graph-primitives>=0.0.12
+Requires-Dist: cjm-context-graph-primitives>=0.0.18
 Dynamic: license-file
 
 # cjm-capability-graph-sqlite
@@ -26,6 +26,7 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 ## Modules
 
+- **`cjm_capability_graph_sqlite.__init__`**
 - **`cjm_capability_graph_sqlite.capability`** — Capability implementation for Context Graph using SQLite.
 - **`cjm_capability_graph_sqlite.query_translation`** — The per-backend translation of the typed query expressions (pass-2 Thread 5; stage 4): NodeQuery/EdgeQuery -> parameterized SQLite SQL over the nodes/edges schema. THIS module is what makes the typed surface portable - the expression is domain- and backend-neutral; every backend tool owns a translation like this one (the ratified stage-4 split: backend owns translation; the adapter stays generic). Pure functions, unit-tested against an in-memory DB with the production schema - no capability runtime needed.
 
@@ -35,13 +36,16 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 - `SQLiteGraphCapability` _class_ — Local, file-backed Context Graph implementation using SQLite.
 - `SQLiteGraphCapabilityConfig` _class_ — Configuration for SQLite Graph Capability.
+- `ingest_sources` _function_ — The source HEADs this db was ingested from, on a fresh read-only connection. A db built
 - `integrity_check` _function_ — Backend self-check (stage 4; the G3 corruption find institutionalized):
 - `query_edges` _function_ — Execute a typed edge query (stage 4) — same contract as `query_nodes`.
 - `query_nodes` _function_ — Execute a typed node query (stage 4) — translated to parameterized SQL
 - `raw_query` _function_ — Execute the raw escape — refuses backend mismatches (non-portable by
+- `record_ingest_sources` _function_ — Record the source HEADs this db was ingested from, REPLACING any earlier record
 
 ### `cjm_capability_graph_sqlite.query_translation`
 
+- `decode_projected_row` _function_ — Zip a projected row, decoding every property column from its JSON representation.
 - `translate_edge_query` _function_ — Translate an `EdgeQuery` to parameterized SQLite SQL.
 - `translate_node_query` _function_ — Translate a `NodeQuery` to parameterized SQLite SQL.
 

{cjm_capability_graph_sqlite-0.0.28 → cjm_capability_graph_sqlite-0.0.30}/README.md

@@ -6,6 +6,7 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 ## Modules
 
+- **`cjm_capability_graph_sqlite.__init__`**
 - **`cjm_capability_graph_sqlite.capability`** — Capability implementation for Context Graph using SQLite.
 - **`cjm_capability_graph_sqlite.query_translation`** — The per-backend translation of the typed query expressions (pass-2 Thread 5; stage 4): NodeQuery/EdgeQuery -> parameterized SQLite SQL over the nodes/edges schema. THIS module is what makes the typed surface portable - the expression is domain- and backend-neutral; every backend tool owns a translation like this one (the ratified stage-4 split: backend owns translation; the adapter stays generic). Pure functions, unit-tested against an in-memory DB with the production schema - no capability runtime needed.
 
@@ -15,13 +16,16 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 - `SQLiteGraphCapability` _class_ — Local, file-backed Context Graph implementation using SQLite.
 - `SQLiteGraphCapabilityConfig` _class_ — Configuration for SQLite Graph Capability.
+- `ingest_sources` _function_ — The source HEADs this db was ingested from, on a fresh read-only connection. A db built
 - `integrity_check` _function_ — Backend self-check (stage 4; the G3 corruption find institutionalized):
 - `query_edges` _function_ — Execute a typed edge query (stage 4) — same contract as `query_nodes`.
 - `query_nodes` _function_ — Execute a typed node query (stage 4) — translated to parameterized SQL
 - `raw_query` _function_ — Execute the raw escape — refuses backend mismatches (non-portable by
+- `record_ingest_sources` _function_ — Record the source HEADs this db was ingested from, REPLACING any earlier record
 
 ### `cjm_capability_graph_sqlite.query_translation`
 
+- `decode_projected_row` _function_ — Zip a projected row, decoding every property column from its JSON representation.
 - `translate_edge_query` _function_ — Translate an `EdgeQuery` to parameterized SQLite SQL.
 - `translate_node_query` _function_ — Translate a `NodeQuery` to parameterized SQLite SQL.
 

cjm_capability_graph_sqlite-0.0.30/cjm_capability_graph_sqlite/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.30"

{cjm_capability_graph_sqlite-0.0.28 → cjm_capability_graph_sqlite-0.0.30}/cjm_capability_graph_sqlite/capability.py

@@ -19,10 +19,15 @@ import logging
 import os
 import sqlite3
 import time
+from contextlib import contextmanager
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 
-from cjm_capability_graph_sqlite.query_translation import translate_edge_query, translate_node_query
+from cjm_capability_graph_sqlite.query_translation import (decode_projected_row,
+                                                           EDGE_STRUCTURAL_KEYS,
+                                                           NODE_STRUCTURAL_KEYS,
+                                                           translate_edge_query,
+                                                           translate_node_query)
 from cjm_context_graph_primitives.graph import GraphContext, GraphEdge, GraphNode
 from cjm_context_graph_primitives.provenance import SourceRef
 from cjm_context_graph_primitives.query import (EdgeQuery, EdgeQueryResult, NodeQuery,
@@ -36,6 +41,7 @@ from cjm_substrate.utils.validation import (config_to_dict, dataclass_to_jsonsch
 @dataclass
 class SQLiteGraphCapabilityConfig:
     """Configuration for SQLite Graph Capability."""
+
     db_path: Optional[str] = field(
         default=None,
         metadata={
@@ -78,6 +84,11 @@ class SQLiteGraphCapability(ToolCapability):
             FOREIGN KEY(source_id) REFERENCES nodes(id) ON DELETE CASCADE,
             FOREIGN KEY(target_id) REFERENCES nodes(id) ON DELETE CASCADE
         );
+
+        CREATE TABLE ingest_sources (  -- the source HEADs the rebuild ingested (DEC a9176261)
+            source TEXT PRIMARY KEY,
+            head TEXT NOT NULL
+        );
     """
 
     config_class = SQLiteGraphCapabilityConfig
@@ -164,8 +175,9 @@ class SQLiteGraphCapability(ToolCapability):
         return os.path.join(data_dir, "context_graph.db")
 
     def _init_db(self) -> None:
-        """Create tables and indices."""
-        with self._connect() as con:
+        """Create tables and indices -- the schema only. Statistics are written by the
+        paths that change data (`_writing`), never by an open (finding ff3ede3d)."""
+        with self._writing() as con:
             # Enable Foreign Keys
             con.execute("PRAGMA foreign_keys = ON;")
 
@@ -180,7 +192,11 @@ class SQLiteGraphCapability(ToolCapability):
                     updated_at REAL
                 )
             """)
-            con.execute("CREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(label);")
+            # (label, id): a label read in id order -- a keyset page, a bounded read -- is a
+            # range scan, never a sort of the whole label (findings a107e2f3 / ae2b196e); it
+            # serves a label-only lookup too, so the label-only index it replaces is dropped.
+            con.execute("CREATE INDEX IF NOT EXISTS idx_nodes_label_id ON nodes(label, id);")
+            con.execute("DROP INDEX IF EXISTS idx_nodes_label;")
 
             # Edges Table
             con.execute("""
@@ -207,18 +223,21 @@ class SQLiteGraphCapability(ToolCapability):
             # them up on the next initialize.
             con.execute("CREATE INDEX IF NOT EXISTS idx_edges_source_type ON edges(source_id, relation_type);")
             con.execute("CREATE INDEX IF NOT EXISTS idx_edges_target_type ON edges(target_id, relation_type);")
+            # The ingest record (DEC a9176261): the source HEADs this db was ingested from —
+            # provenance about the db itself, beside the graph and never in it.
+            con.execute("""
+                CREATE TABLE IF NOT EXISTS ingest_sources (
+                    source TEXT PRIMARY KEY,
+                    head TEXT NOT NULL
+                )
+            """)
             # H7 companion: planner statistics. Without sqlite_stat1 the
             # correlated-probe plans pick the WRONG composite index (the
             # target-side probe scans a whole document's PART_OF rows per
-            # outer row: 5s; source-side probe: 8ms at corpus scale).
-            # Cheap (~40ms at 13k segments); writable opens only. Guarded:
-            # stats are an OPTIMIZATION — a corrupted DB must still LOAD so
-            # integrity_check (the designed G3 gate) can diagnose it, never
-            # fail at initialize via a stats side-effect.
-            try:
-                con.execute("ANALYZE;")
-            except sqlite3.DatabaseError as e:
-                logging.warning(f"ANALYZE skipped (run integrity_check): {e}")
+            # outer row: 5s; source-side probe: 8ms at corpus scale). They are
+            # kept by `_writing`'s PRAGMA optimize at the close of every write,
+            # this one included (a new index is analyzed there) -- no longer a
+            # full ANALYZE per open, O(db) on every read (finding ff3ede3d).
 
     def _connect(self) -> sqlite3.Connection:  # Open DB connection (honors readonly; sets WAL + FKs)
         """Open a SQLite connection honoring the `readonly` config (SG-41).
@@ -236,6 +255,28 @@ class SQLiteGraphCapability(ToolCapability):
         con.execute("PRAGMA foreign_keys = ON;")
         return con
 
+    @contextmanager
+    def _writing(self) -> Iterator[sqlite3.Connection]:  # One unit of write work: commit, planner upkeep, close
+        """Open a connection for one unit of write work: commit on success (roll back on
+        error), then keep the planner's statistics current, then close.
+
+        Statistics are written on the paths that change data, never on an open (finding
+        ff3ede3d: a read paid a full ANALYZE and took the write lock). PRAGMA optimize
+        analyzes a table only when it was never analyzed, gained an index, or moved ~10x
+        in size since, so a steady-state write pays a no-op check and a fresh rebuild db
+        gains statistics as it fills. Guarded: statistics are an OPTIMIZATION — a corrupted
+        DB must still answer integrity_check (the designed G3 gate)."""
+        con = self._connect()
+        try:
+            with con:
+                yield con
+            try:
+                con.execute("PRAGMA optimize=0x10002;")
+            except sqlite3.DatabaseError as e:
+                logging.warning(f"PRAGMA optimize skipped (run integrity_check): {e}")
+        finally:
+            con.close()
+
     # -------------------------------------------------------------------------
     # Helpers
     # -------------------------------------------------------------------------
@@ -304,7 +345,7 @@ class SQLiteGraphCapability(ToolCapability):
         stamping now() here clamped every rebuilt node to rebuild time)."""
         ids = []
         now = time.time()
-        with self._connect() as con:
+        with self._writing() as con:
             for n in nodes:
                 sources_json = json.dumps([s.to_dict() for s in n.sources])
                 props_json = json.dumps(n.properties)
@@ -336,7 +377,7 @@ class SQLiteGraphCapability(ToolCapability):
         ids = []
         now = time.time()
         missing: List[Tuple[str, str, str]] = []  # (source_id, target_id, relation_type) of FK-skipped edges
-        with self._connect() as con:
+        with self._writing() as con:
             con.execute("PRAGMA foreign_keys = ON;")
             for e in edges:
                 props_json = json.dumps(e.properties)
@@ -419,11 +460,13 @@ class SQLiteGraphCapability(ToolCapability):
         label: str,  # Node label to search for
         limit: int = 100  # Max results
     ) -> List[GraphNode]:  # Matching nodes
-        """Find nodes by label."""
+        """Find nodes by label, in id order: a bounded read sorts before it limits, so a
+        page is never an arbitrary subset (finding a107e2f3)."""
         results = []
         with self._connect() as con:
             cur = con.execute(
-                "SELECT id, label, properties, sources, created_at, updated_at FROM nodes WHERE label = ? LIMIT ?",
+                "SELECT id, label, properties, sources, created_at, updated_at FROM nodes "
+                "WHERE label = ? ORDER BY id LIMIT ?",
                 (label, limit)
             )
             for row in cur:
@@ -529,7 +572,7 @@ class SQLiteGraphCapability(ToolCapability):
         to rebuild time)."""
         properties = dict(properties)
         stamp = properties.pop("updated_at", None)
-        with self._connect() as con:
+        with self._writing() as con:
             # Fetch existing to merge
             cur = con.execute("SELECT properties FROM nodes WHERE id = ?", (node_id,))
             row = cur.fetchone()
@@ -550,8 +593,14 @@ class SQLiteGraphCapability(ToolCapability):
         edge_id: str,  # UUID of edge to update
         properties: Dict[str, Any]  # Properties to merge/update
     ) -> bool:  # True if successful
-        """Partial update of edge properties."""
-        with self._connect() as con:
+        """Partial update of edge properties.
+
+        A reserved `updated_at` key in `properties` sets the COLUMN instead of now()
+        and never lands in the JSON blob — the op clock's stamp (design 8f6f2343),
+        exactly as `update_node` takes it."""
+        properties = dict(properties)
+        stamp = properties.pop("updated_at", None)
+        with self._writing() as con:
             cur = con.execute("SELECT properties FROM edges WHERE id = ?", (edge_id,))
             row = cur.fetchone()
             if not row:
@@ -562,7 +611,7 @@ class SQLiteGraphCapability(ToolCapability):
 
             con.execute(
                 "UPDATE edges SET properties = ?, updated_at = ? WHERE id = ?",
-                (json.dumps(existing), time.time(), edge_id)
+                (json.dumps(existing), stamp if stamp is not None else time.time(), edge_id)
             )
             return True
 
@@ -576,7 +625,7 @@ class SQLiteGraphCapability(ToolCapability):
         cascade: bool = True  # Also delete connected edges
     ) -> int:  # Number of nodes deleted
         """Delete nodes (and optionally connected edges)."""
-        with self._connect() as con:
+        with self._writing() as con:
             if cascade:
                 con.execute("PRAGMA foreign_keys = ON;")  # Ensures cascade works
             else:
@@ -594,7 +643,7 @@ class SQLiteGraphCapability(ToolCapability):
         edge_ids: List[str]  # UUIDs of edges to delete
     ) -> int:  # Number of edges deleted
         """Delete edges."""
-        with self._connect() as con:
+        with self._writing() as con:
             placeholders = ','.join('?' for _ in edge_ids)
             cur = con.execute(
                 f"DELETE FROM edges WHERE id IN ({placeholders})",
@@ -651,10 +700,11 @@ class SQLiteGraphCapability(ToolCapability):
         nodes: List[GraphNode],  # Nodes to import
         merge_strategy: str  # "overwrite" | "skip" | "merge"
     ) -> int:  # Count of rows written (inserted or updated)
-        """Import nodes honoring merge_strategy (see import_graph)."""
+        """Import nodes honoring merge_strategy (see import_graph). Carried
+        created_at / updated_at are honored like add_nodes (0d50b921, design 8f6f2343)."""
         now = time.time()
         written = 0
-        with self._connect() as con:
+        with self._writing() as con:
             for node in nodes:
                 props_json = json.dumps(node.properties)
                 sources_json = json.dumps([s.to_dict() for s in node.sources])
@@ -662,7 +712,10 @@ class SQLiteGraphCapability(ToolCapability):
                 if row is None:
                     con.execute(
                         "INSERT INTO nodes (id, label, properties, sources, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)",
-                        (node.id, node.label, props_json, sources_json, now, now),
+                        (node.id, node.label, props_json, sources_json,
+                         node.created_at if node.created_at is not None else now,
+                         node.updated_at if node.updated_at is not None
+                         else (node.created_at if node.created_at is not None else now)),
                     )
                     written += 1
                 elif merge_strategy == "skip":
@@ -670,7 +723,8 @@ class SQLiteGraphCapability(ToolCapability):
                 elif merge_strategy == "overwrite":
                     con.execute(
                         "UPDATE nodes SET label = ?, properties = ?, sources = ?, updated_at = ? WHERE id = ?",
-                        (node.label, props_json, sources_json, now, node.id),
+                        (node.label, props_json, sources_json,
+                         node.updated_at if node.updated_at is not None else now, node.id),
                     )
                     written += 1
                 else:  # merge
@@ -686,7 +740,8 @@ class SQLiteGraphCapability(ToolCapability):
                             seen.add(key)
                     con.execute(
                         "UPDATE nodes SET label = ?, properties = ?, sources = ?, updated_at = ? WHERE id = ?",
-                        (node.label, json.dumps(merged_props), json.dumps(existing_sources), now, node.id),
+                        (node.label, json.dumps(merged_props), json.dumps(existing_sources),
+                         node.updated_at if node.updated_at is not None else now, node.id),
                     )
                     written += 1
         return written
@@ -696,10 +751,11 @@ class SQLiteGraphCapability(ToolCapability):
         edges: List[GraphEdge],  # Edges to import
         merge_strategy: str  # "overwrite" | "skip" | "merge"
     ) -> int:  # Count of rows written (inserted or updated)
-        """Import edges honoring merge_strategy (see import_graph)."""
+        """Import edges honoring merge_strategy (see import_graph). Carried
+        created_at / updated_at are honored like add_edges (0d50b921, design 8f6f2343)."""
         now = time.time()
         written = 0
-        with self._connect() as con:
+        with self._writing() as con:
             for edge in edges:
                 props_json = json.dumps(edge.properties)
                 row = con.execute("SELECT properties FROM edges WHERE id = ?", (edge.id,)).fetchone()
@@ -707,7 +763,10 @@ class SQLiteGraphCapability(ToolCapability):
                     try:
                         con.execute(
                             "INSERT INTO edges (id, source_id, target_id, relation_type, properties, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
-                            (edge.id, edge.source_id, edge.target_id, edge.relation_type, props_json, now, now),
+                            (edge.id, edge.source_id, edge.target_id, edge.relation_type, props_json,
+                             edge.created_at if edge.created_at is not None else now,
+                             edge.updated_at if edge.updated_at is not None
+                             else (edge.created_at if edge.created_at is not None else now)),
                         )
                         written += 1
                     except sqlite3.IntegrityError as err:
@@ -717,7 +776,8 @@ class SQLiteGraphCapability(ToolCapability):
                 elif merge_strategy == "overwrite":
                     con.execute(
                         "UPDATE edges SET source_id = ?, target_id = ?, relation_type = ?, properties = ?, updated_at = ? WHERE id = ?",
-                        (edge.source_id, edge.target_id, edge.relation_type, props_json, now, edge.id),
+                        (edge.source_id, edge.target_id, edge.relation_type, props_json,
+                         edge.updated_at if edge.updated_at is not None else now, edge.id),
                     )
                     written += 1
                 else:  # merge
@@ -725,7 +785,8 @@ class SQLiteGraphCapability(ToolCapability):
                     merged.update(edge.properties)
                     con.execute(
                         "UPDATE edges SET properties = ?, updated_at = ? WHERE id = ?",
-                        (json.dumps(merged), now, edge.id),
+                        (json.dumps(merged), edge.updated_at if edge.updated_at is not None else now,
+                         edge.id),
                     )
                     written += 1
         return written
@@ -820,7 +881,7 @@ def query_nodes(
         return NodeQueryResult(nodes=[self._row_to_node(r) for r in rows])
     out = []
     for r in rows:
-        d = dict(zip(keys, r))
+        d = decode_projected_row(keys, r, NODE_STRUCTURAL_KEYS)  # properties keep their JSON types
         if "sources" in d and d["sources"]:
             d["sources"] = json.loads(d["sources"])  # ref dicts (CR-19 shape)
         out.append(d)
@@ -843,7 +904,7 @@ def query_edges(
         con.close()
     if mode == "full":
         return EdgeQueryResult(edges=[self._row_to_edge(r) for r in rows])
-    return EdgeQueryResult(rows=[dict(zip(keys, r)) for r in rows])
+    return EdgeQueryResult(rows=[decode_projected_row(keys, r, EDGE_STRUCTURAL_KEYS) for r in rows])
 
 
 def raw_query(
@@ -881,7 +942,39 @@ def integrity_check(self) -> Dict[str, Any]:  # {"ok": bool, "errors": [...], "b
     return {"ok": ok, "errors": [] if ok else rows, "backend": "sqlite"}
 
 
+def record_ingest_sources(
+    self,
+    sources: Dict[str, str],  # source id -> the HEAD commit the ingest read
+) -> int:  # Sources recorded
+    """Record the source HEADs this db was ingested from, REPLACING any earlier record
+    (DEC a9176261). The rebuild writes it into the fresh db it builds, so the record and the
+    rows it describes swap in together; it is provenance about the db, not a graph row, so
+    it is neither stamped by the op clock nor seen by the write observer."""
+    rows = sorted((str(k), str(v)) for k, v in dict(sources).items())
+    with self._writing() as con:
+        con.execute("DELETE FROM ingest_sources")
+        con.executemany("INSERT INTO ingest_sources (source, head) VALUES (?, ?)", rows)
+    return len(rows)
+
+
+def ingest_sources(self) -> Dict[str, str]:  # source id -> HEAD; {} when the db holds no record
+    """The source HEADs this db was ingested from, on a fresh read-only connection. A db built
+    before the record existed has no table: it reads as no record, never as an error."""
+    con = sqlite3.connect(f"file:{self._db_path}?mode=ro", uri=True)
+    try:
+        rows = con.execute("SELECT source, head FROM ingest_sources ORDER BY source").fetchall()
+    except sqlite3.OperationalError as e:
+        if "no such table" not in str(e):
+            raise
+        rows = []
+    finally:
+        con.close()
+    return {s: h for s, h in rows}
+
+
 SQLiteGraphCapability.query_nodes = query_nodes
 SQLiteGraphCapability.query_edges = query_edges
 SQLiteGraphCapability.raw_query = raw_query
 SQLiteGraphCapability.integrity_check = integrity_check
+SQLiteGraphCapability.record_ingest_sources = record_ingest_sources
+SQLiteGraphCapability.ingest_sources = ingest_sources

{cjm_capability_graph_sqlite-0.0.28 → cjm_capability_graph_sqlite-0.0.30}/cjm_capability_graph_sqlite/query_translation.py

@@ -25,7 +25,11 @@ Translation rules:
   Projected node rows always carry `id` (+ `label`/`created_at`/`updated_at`
   projectable structurally — the storage stamps, never shadowed by a
   same-named property; `sources` projectable as parsed ref dicts); edge rows
-  always carry `id`/`source_id`/`target_id` (+ `relation_type` projectable)."""
+  always carry `id`/`source_id`/`target_id` (+ `relation_type` projectable).
+  A projected PROPERTY is selected as its JSON representation (`->`) and decoded
+  by the capability, so it keeps its type — a list stays a list, an object an
+  object (`json_extract` returned arrays and objects as JSON TEXT: a projected
+  `calls` list read as a string, the live code fold's index keyed characters)."""
 
 import json
 import re
@@ -167,16 +171,22 @@ EDGE_FULL_COLUMNS = "e.id, e.source_id, e.target_id, e.relation_type, e.properti
 
 
 def _order_limit_sql(
-    query,  # NodeQuery or EdgeQuery (order_by / limit / offset fields)
+    query,  # NodeQuery or EdgeQuery (order_by / limit / offset / after fields)
     props_col: str,  # Properties column expression for ORDER BY paths
+    id_col: str,  # Row id column: the keyset key and the tie-break
     params: List[Any],  # Bound-params list (appended in place)
 ) -> str:  # ORDER BY / LIMIT / OFFSET tail
-    """Compile the shared ordering + paging tail."""
+    """Compile the shared ordering + paging tail. A property ordering breaks ties on the
+    id, and a bounded or keyset read with no ordering is ordered by the id, so a page is
+    never an arbitrary subset (findings a107e2f3 / ae2b196e)."""
     tail = ""
     if query.order_by is not None:
         tail += (f" ORDER BY json_extract({props_col}, "
                  f"{_json_path(query.order_by.prop)})"
-                 + (" DESC" if query.order_by.descending else " ASC"))
+                 + (" DESC" if query.order_by.descending else " ASC")
+                 + f", {id_col}")
+    elif query.limit is not None or query.offset or query.after is not None:
+        tail += f" ORDER BY {id_col}"
     if query.limit is not None or query.offset:
         tail += " LIMIT ?"
         params.append(query.limit if query.limit is not None else -1)
@@ -202,6 +212,9 @@ def translate_node_query(
     if q.ids is not None:
         where.append("n.id IN (SELECT value FROM json_each(?))")
         params.append(json.dumps(list(q.ids)))
+    if q.after is not None:
+        where.append("n.id > ?")
+        params.append(q.after)
     if q.label is not None:
         where.append("n.label = ?")
         params.append(q.label)
@@ -238,13 +251,13 @@ def translate_node_query(
                 select_parts.append("n.sources")
                 keys.append("sources")
             else:
-                select_parts.append(f"json_extract(n.properties, {_json_path(name)})")
+                select_parts.append(f"n.properties -> {_json_path(name)}")
                 keys.append(name)
-        tail = _order_limit_sql(q, "n.properties", params)
+        tail = _order_limit_sql(q, "n.properties", "n.id", params)
         return (f"SELECT {', '.join(select_parts)} FROM nodes AS n{where_sql}{tail}",
                 params, "rows", keys)
 
-    tail = _order_limit_sql(q, "n.properties", params)
+    tail = _order_limit_sql(q, "n.properties", "n.id", params)
     return (f"SELECT {NODE_FULL_COLUMNS} FROM nodes AS n{where_sql}{tail}",
             params, "full", None)
 
@@ -264,6 +277,9 @@ def translate_edge_query(
     if q.ids is not None:
         where.append("e.id IN (SELECT value FROM json_each(?))")
         params.append(json.dumps(list(q.ids)))
+    if q.after is not None:
+        where.append("e.id > ?")
+        params.append(q.after)
     if q.relation_type is not None:
         where.append("e.relation_type = ?")
         params.append(q.relation_type)
@@ -306,12 +322,30 @@ def translate_edge_query(
                 select_parts.append("e.relation_type")
                 keys.append("relation_type")
             else:
-                select_parts.append(f"json_extract(e.properties, {_json_path(name)})")
+                select_parts.append(f"e.properties -> {_json_path(name)}")
                 keys.append(name)
-        tail = _order_limit_sql(q, "e.properties", params)
+        tail = _order_limit_sql(q, "e.properties", "e.id", params)
         return (f"SELECT {', '.join(select_parts)} FROM edges AS e{where_sql}{tail}",
                 params, "rows", keys)
 
-    tail = _order_limit_sql(q, "e.properties", params)
+    tail = _order_limit_sql(q, "e.properties", "e.id", params)
     return (f"SELECT {EDGE_FULL_COLUMNS} FROM edges AS e{where_sql}{tail}",
             params, "full", None)
+
+
+# The structural columns a projection selects as-is; every other projected key is a property,
+# selected as JSON (`->`) and decoded by `decode_projected_row`.
+NODE_STRUCTURAL_KEYS = frozenset({"id", "label", "created_at", "updated_at", "sources"})
+EDGE_STRUCTURAL_KEYS = frozenset({"id", "source_id", "target_id", "relation_type"})
+
+
+def decode_projected_row(
+    keys: List[str],        # The projected keys, in select order
+    values: Any,            # One cursor row
+    structural: frozenset,  # The keys selected as plain columns
+) -> dict:  # The row: structural columns as-is, each property decoded to its JSON type (absent -> None)
+    """Zip a projected row, decoding every property column from its JSON representation."""
+    out = {}
+    for k, v in zip(keys, values):
+        out[k] = v if k in structural or v is None else json.loads(v)
+    return out

{cjm_capability_graph_sqlite-0.0.28 → cjm_capability_graph_sqlite-0.0.30/cjm_capability_graph_sqlite.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjm-capability-graph-sqlite
-Version: 0.0.28
+Version: 0.0.30
 Summary: A local, file-backed context-graph storage capability for the cjm-substrate runtime that implements graph storage, traversal, and querying using SQLite.
 Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
 License: Apache-2.0
@@ -15,7 +15,7 @@ Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: cjm-substrate>=0.0.51
-Requires-Dist: cjm-context-graph-primitives>=0.0.12
+Requires-Dist: cjm-context-graph-primitives>=0.0.18
 Dynamic: license-file
 
 # cjm-capability-graph-sqlite
@@ -26,6 +26,7 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 ## Modules
 
+- **`cjm_capability_graph_sqlite.__init__`**
 - **`cjm_capability_graph_sqlite.capability`** — Capability implementation for Context Graph using SQLite.
 - **`cjm_capability_graph_sqlite.query_translation`** — The per-backend translation of the typed query expressions (pass-2 Thread 5; stage 4): NodeQuery/EdgeQuery -> parameterized SQLite SQL over the nodes/edges schema. THIS module is what makes the typed surface portable - the expression is domain- and backend-neutral; every backend tool owns a translation like this one (the ratified stage-4 split: backend owns translation; the adapter stays generic). Pure functions, unit-tested against an in-memory DB with the production schema - no capability runtime needed.
 
@@ -35,13 +36,16 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 - `SQLiteGraphCapability` _class_ — Local, file-backed Context Graph implementation using SQLite.
 - `SQLiteGraphCapabilityConfig` _class_ — Configuration for SQLite Graph Capability.
+- `ingest_sources` _function_ — The source HEADs this db was ingested from, on a fresh read-only connection. A db built
 - `integrity_check` _function_ — Backend self-check (stage 4; the G3 corruption find institutionalized):
 - `query_edges` _function_ — Execute a typed edge query (stage 4) — same contract as `query_nodes`.
 - `query_nodes` _function_ — Execute a typed node query (stage 4) — translated to parameterized SQL
 - `raw_query` _function_ — Execute the raw escape — refuses backend mismatches (non-portable by
+- `record_ingest_sources` _function_ — Record the source HEADs this db was ingested from, REPLACING any earlier record
 
 ### `cjm_capability_graph_sqlite.query_translation`
 
+- `decode_projected_row` _function_ — Zip a projected row, decoding every property column from its JSON representation.
 - `translate_edge_query` _function_ — Translate an `EdgeQuery` to parameterized SQLite SQL.
 - `translate_node_query` _function_ — Translate a `NodeQuery` to parameterized SQLite SQL.
 

cjm_capability_graph_sqlite-0.0.30/cjm_capability_graph_sqlite.egg-info/requires.txt

@@ -0,0 +1,2 @@
+cjm-substrate>=0.0.51
+cjm-context-graph-primitives>=0.0.18

{cjm_capability_graph_sqlite-0.0.28 → cjm_capability_graph_sqlite-0.0.30}/pyproject.toml

@@ -12,7 +12,7 @@ license = {text = "Apache-2.0"}
 authors = [{name = "Christian J. Mills", email = "9126128+cj-mills@users.noreply.github.com"}]
 keywords = ['python']
 classifiers = ["Natural Language :: English", "Intended Audience :: Developers", "Development Status :: 3 - Alpha", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only"]
-dependencies = ['cjm-substrate>=0.0.51', 'cjm-context-graph-primitives>=0.0.12']
+dependencies = ['cjm-substrate>=0.0.51', 'cjm-context-graph-primitives>=0.0.18']
 
 [project.urls]
 Repository = "https://github.com/cj-mills/cjm-capability-graph-sqlite"

{cjm_capability_graph_sqlite-0.0.28 → cjm_capability_graph_sqlite-0.0.30}/tests/test_capability.py

@@ -3,18 +3,17 @@
 Projected from the capability notebook's demo/test cells at the c25780e8 flip:
 the CRUD/round-trip walk, the SG-41 smells (WAL / readonly / guarded query /
 merge_strategy), and the stage-4 typed-surface smoke test."""
+
 import sqlite3
 import uuid
 
 import pytest
-
 from cjm_capability_graph_sqlite.capability import SQLiteGraphCapability
 from cjm_context_graph_primitives.graph import GraphContext, GraphEdge, GraphNode
 from cjm_context_graph_primitives.locators import FileRef
 from cjm_context_graph_primitives.provenance import SourceRef
-from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy,
-                                                PropertyPredicate, RawQuery,
-                                                RelationPredicate)
+from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy, PropertyPredicate,
+                                                RawQuery, RelationPredicate)
 from cjm_context_graph_primitives.slices import FullContent
 from cjm_substrate.core.errors import CapabilityInputError
 
@@ -140,6 +139,34 @@ def test_sg41_wal_query_guard_merge_readonly(tmp_path):
     ro.cleanup()
 
 
+def test_reads_and_opens_never_write_writes_keep_statistics(tmp_path):
+    # Finding ff3ede3d: statistics are written by the paths that change data (PRAGMA
+    # optimize at the close of each write), so an open or a read commits nothing -- an
+    # observer's data_version moves only when ANOTHER connection commits.
+    db = str(tmp_path / "stats.db")
+    w = SQLiteGraphCapability()
+    w.initialize({"db_path": db})
+    indexes = {r[0] for r in sqlite3.connect(db).execute(
+        "SELECT name FROM sqlite_master WHERE type = 'index'")}
+    assert "idx_nodes_label_id" in indexes and "idx_nodes_label" not in indexes
+    w.add_nodes([GraphNode(id=f"n{i:03d}", label="Doc", properties={"i": i}) for i in range(50)])
+    stats = {r[0] for r in sqlite3.connect(db).execute(
+        "SELECT idx FROM sqlite_stat1 WHERE tbl = 'nodes'")}
+    assert "idx_nodes_label_id" in stats
+    w.cleanup()
+
+    observer = sqlite3.connect(db)
+    before = observer.execute("PRAGMA data_version").fetchone()[0]
+    for readonly in (True, False):
+        r = SQLiteGraphCapability()
+        r.initialize({"db_path": db, "readonly": readonly})
+        assert [n.id for n in r.find_nodes_by_label("Doc", limit=3)] == ["n000", "n001", "n002"]
+        assert r.query_nodes(NodeQuery(label="Doc", count=True)).count == 50
+        r.cleanup()
+    assert observer.execute("PRAGMA data_version").fetchone()[0] == before
+    observer.close()
+
+
 def test_typed_surface_smoke(cap):
     doc = str(uuid.uuid4())
     segs = [str(uuid.uuid4()) for _ in range(3)]
@@ -221,3 +248,67 @@ def test_add_edges_aggregates_missing_endpoint_warnings(cap, caplog):
     msg = fk_lines[0].getMessage()
     assert "25 of 26" in msg and "sample:" in msg
     assert not any("Edge creation error (likely missing node)" in r.getMessage() for r in caplog.records)
+
+
+def test_update_edge_reserved_updated_at_sets_column(cap):
+    # The op clock (design 8f6f2343): update_edge takes the reserved `updated_at` exactly as
+    # update_node does — the COLUMN, never the JSON blob; without it, now()-stamping stands.
+    import json
+
+    a, b = str(uuid.uuid4()), str(uuid.uuid4())
+    cap.add_nodes([GraphNode(id=a, label="Person"), GraphNode(id=b, label="Person")])
+    eid = str(uuid.uuid4())
+    cap.add_edges([GraphEdge(id=eid, source_id=a, target_id=b, relation_type="KNOWS")])
+    assert cap.update_edge(eid, {"since": 1999, "updated_at": 1234.5})
+    con = sqlite3.connect(cap._db_path)
+    props, stamp = con.execute("SELECT properties, updated_at FROM edges WHERE id = ?",
+                               (eid,)).fetchone()
+    con.close()
+    assert json.loads(props) == {"since": 1999} and stamp == 1234.5
+    assert cap.update_edge(eid, {"since": 2000})
+    con = sqlite3.connect(cap._db_path)
+    (stamp2,) = con.execute("SELECT updated_at FROM edges WHERE id = ?", (eid,)).fetchone()
+    con.close()
+    assert stamp2 > 1234.5
+
+
+def test_import_honors_carried_times(cap):
+    # import_graph keeps carried created_at / updated_at like add_nodes / add_edges.
+    a, b = str(uuid.uuid4()), str(uuid.uuid4())
+    eid = str(uuid.uuid4())
+    cap.import_graph(GraphContext(
+        nodes=[GraphNode(id=a, label="P", created_at=10.0, updated_at=20.0),
+               GraphNode(id=b, label="P", created_at=30.0)],
+        edges=[GraphEdge(id=eid, source_id=a, target_id=b, relation_type="R", created_at=40.0)]))
+    con = sqlite3.connect(cap._db_path)
+    rows = dict((r[0], r[1:]) for r in con.execute("SELECT id, created_at, updated_at FROM nodes"))
+    erow = con.execute("SELECT created_at, updated_at FROM edges WHERE id = ?", (eid,)).fetchone()
+    con.close()
+    assert rows[a] == (10.0, 20.0) and rows[b] == (30.0, 30.0) and erow == (40.0, 40.0)
+
+
+def test_ingest_record_replaces_reads_readonly_and_tolerates_a_legacy_db(cap, tmp_path):
+    """The ingest record (DEC a9176261): a replace, never a merge; beside the graph (no row,
+    no op-clock stamp); readable through a read-only open; a db built before the table
+    existed reads as no record."""
+    assert cap.ingest_sources() == {}
+    assert cap.record_ingest_sources({"archive:/site/posts": "a" * 40, "repo:/r/x": "b" * 40}) == 2
+    assert cap.record_ingest_sources({"repo:/r/x": "c" * 40}) == 1
+    assert cap.ingest_sources() == {"repo:/r/x": "c" * 40}
+    assert cap.get_schema()["counts"] == {}  # beside the graph, never in it
+
+    ro = SQLiteGraphCapability()
+    ro.initialize({"db_path": cap._db_path, "readonly": True})
+    assert ro.ingest_sources() == {"repo:/r/x": "c" * 40}
+    ro.cleanup()
+
+    legacy = str(tmp_path / "legacy.db")
+    con = sqlite3.connect(legacy)
+    con.execute("CREATE TABLE nodes (id TEXT PRIMARY KEY, label TEXT NOT NULL, properties JSON, "
+                "sources JSON, created_at REAL, updated_at REAL)")
+    con.commit()
+    con.close()
+    old = SQLiteGraphCapability()
+    old.initialize({"db_path": legacy, "readonly": True})
+    assert old.ingest_sources() == {}
+    old.cleanup()

{cjm_capability_graph_sqlite-0.0.28 → cjm_capability_graph_sqlite-0.0.30}/tests/test_query_translation.py

@@ -3,17 +3,19 @@
 Projected from the query_translation notebook's behavior-test cell at the
 c25780e8 flip: the translation contract is verified against the PRODUCTION
 schema in an in-memory DB — by what it returns, not by SQL strings."""
+
 import json
 import sqlite3
 
 import pytest
-
-from cjm_capability_graph_sqlite.query_translation import (translate_edge_query,
+from cjm_capability_graph_sqlite.query_translation import (decode_projected_row,
+                                                           EDGE_STRUCTURAL_KEYS,
+                                                           NODE_STRUCTURAL_KEYS,
+                                                           translate_edge_query,
                                                            translate_node_query)
 from cjm_context_graph_primitives.locators import FileRef
-from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy,
-                                                PropertyPredicate, RelationPredicate,
-                                                SourcePredicate)
+from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy, PropertyPredicate,
+                                                RelationPredicate, SourcePredicate)
 from cjm_substrate.core.errors import CapabilityInputError
 
 
@@ -71,7 +73,8 @@ def run(con, q):
         return cur.fetchone()[0]
     rows = cur.fetchall()
     if mode == "rows":
-        return [dict(zip(keys, r)) for r in rows]
+        structural = NODE_STRUCTURAL_KEYS if isinstance(q, NodeQuery) else EDGE_STRUCTURAL_KEYS
+        return [decode_projected_row(keys, r, structural) for r in rows]
     return rows
 
 
@@ -85,6 +88,21 @@ def test_spine_read_ordered_projection(con):
     assert json.loads(rows[1]["sources"])[0]["content_hash"] == "sha256:ab"
 
 
+def test_projected_properties_keep_their_json_types(con):
+    # A projected object stays an object and a missing key is None (json_extract used to
+    # hand arrays and objects back as JSON TEXT — a projected list read as a string).
+    rows = run(con, NodeQuery(label="Correction", project=["payload", "correction_type"]))
+    by = {r["id"]: r for r in rows}
+    assert by["c1"]["payload"] == {"document_id": "doc-1", "segment_id": "s1"}
+    assert by["c2"]["correction_type"] == "text_content"
+    rows = run(con, NodeQuery(label="Segment", related=RelationPredicate("PART_OF", node_id="doc-1"),
+                              order_by=OrderBy(prop="index"), project=["text", "start_time"]))
+    assert [r["text"] for r in rows] == ["It's 100% real_deal", "hello world", "", None]
+    assert rows[1]["start_time"] == 1.0
+    rows = run(con, EdgeQuery(relation_type="REVIEWED", project=["decision"]))
+    assert rows == [{"id": "rev", "source_id": "sess-1", "target_id": "s0", "decision": "corrected"}]
+
+
 def test_structural_timestamp_projection(con):
     # created_at/updated_at project from the storage COLUMNS (the stamps),
     # not json_extract over properties — no Segment carries them as props.
@@ -163,6 +181,27 @@ def test_ne_order_desc_limit_and_paging(con):
     assert [r[0] for r in rows] == ["s1", "s2"]
 
 
+def test_keyset_pages_and_deterministic_bounded_order(con):
+    # The keyset cursor (finding ae2b196e): a page is the rows after the last id seen, in
+    # id order, so a pass over the whole answer visits every row exactly once.
+    every = sorted(r["id"] for r in run(con, NodeQuery(label="Segment", project=[])))
+    seen, after = [], None
+    while True:
+        rows = run(con, NodeQuery(label="Segment", project=[], limit=2, after=after))
+        seen.extend(r["id"] for r in rows)
+        if len(rows) < 2:
+            break
+        after = rows[-1]["id"]
+    assert seen == every
+    # A bounded read with no ordering is ordered by id, never an arbitrary subset
+    # (finding a107e2f3); a property ordering breaks its ties on the id.
+    assert [r[0] for r in run(con, NodeQuery(label="Segment", limit=3))] == every[:3]
+    tied = run(con, NodeQuery(label="Segment", where=[PropertyPredicate("index", "eq", 0)],
+                              order_by=OrderBy(prop="index")))
+    assert [r[0] for r in tied] == ["other", "s0"]
+    assert [r[0] for r in run(con, EdgeQuery(relation_type="NEXT", after="n0"))] == ["n1", "nx"]
+
+
 def test_in_op_ids_batch_and_top_level_source(con):
     rows = run(con, NodeQuery(label="Segment", where=[PropertyPredicate("index", "in", [0, 2])],
                               related=RelationPredicate("PART_OF", node_id="doc-1")))

cjm_capability_graph_sqlite-0.0.28/cjm_capability_graph_sqlite/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.28"

cjm_capability_graph_sqlite-0.0.28/cjm_capability_graph_sqlite.egg-info/requires.txt

@@ -1,2 +0,0 @@
-cjm-substrate>=0.0.51
-cjm-context-graph-primitives>=0.0.12