cjm-capability-graph-sqlite 0.0.26__tar.gz → 0.0.29__tar.gz

{cjm_capability_graph_sqlite-0.0.26/cjm_capability_graph_sqlite.egg-info → cjm_capability_graph_sqlite-0.0.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjm-capability-graph-sqlite
-Version: 0.0.26
+Version: 0.0.29
 Summary: A local, file-backed context-graph storage capability for the cjm-substrate runtime that implements graph storage, traversal, and querying using SQLite.
 Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
 License: Apache-2.0
@@ -26,6 +26,7 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 ## Modules
 
+- **`cjm_capability_graph_sqlite.__init__`**
 - **`cjm_capability_graph_sqlite.capability`** — Capability implementation for Context Graph using SQLite.
 - **`cjm_capability_graph_sqlite.query_translation`** — The per-backend translation of the typed query expressions (pass-2 Thread 5; stage 4): NodeQuery/EdgeQuery -> parameterized SQLite SQL over the nodes/edges schema. THIS module is what makes the typed surface portable - the expression is domain- and backend-neutral; every backend tool owns a translation like this one (the ratified stage-4 split: backend owns translation; the adapter stays generic). Pure functions, unit-tested against an in-memory DB with the production schema - no capability runtime needed.
 
@@ -35,13 +36,16 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 - `SQLiteGraphCapability` _class_ — Local, file-backed Context Graph implementation using SQLite.
 - `SQLiteGraphCapabilityConfig` _class_ — Configuration for SQLite Graph Capability.
+- `ingest_sources` _function_ — The source HEADs this db was ingested from, on a fresh read-only connection. A db built
 - `integrity_check` _function_ — Backend self-check (stage 4; the G3 corruption find institutionalized):
 - `query_edges` _function_ — Execute a typed edge query (stage 4) — same contract as `query_nodes`.
 - `query_nodes` _function_ — Execute a typed node query (stage 4) — translated to parameterized SQL
 - `raw_query` _function_ — Execute the raw escape — refuses backend mismatches (non-portable by
+- `record_ingest_sources` _function_ — Record the source HEADs this db was ingested from, REPLACING any earlier record
 
 ### `cjm_capability_graph_sqlite.query_translation`
 
+- `decode_projected_row` _function_ — Zip a projected row, decoding every property column from its JSON representation.
 - `translate_edge_query` _function_ — Translate an `EdgeQuery` to parameterized SQLite SQL.
 - `translate_node_query` _function_ — Translate a `NodeQuery` to parameterized SQLite SQL.
 

{cjm_capability_graph_sqlite-0.0.26 → cjm_capability_graph_sqlite-0.0.29}/README.md

@@ -6,6 +6,7 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 ## Modules
 
+- **`cjm_capability_graph_sqlite.__init__`**
 - **`cjm_capability_graph_sqlite.capability`** — Capability implementation for Context Graph using SQLite.
 - **`cjm_capability_graph_sqlite.query_translation`** — The per-backend translation of the typed query expressions (pass-2 Thread 5; stage 4): NodeQuery/EdgeQuery -> parameterized SQLite SQL over the nodes/edges schema. THIS module is what makes the typed surface portable - the expression is domain- and backend-neutral; every backend tool owns a translation like this one (the ratified stage-4 split: backend owns translation; the adapter stays generic). Pure functions, unit-tested against an in-memory DB with the production schema - no capability runtime needed.
 
@@ -15,13 +16,16 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 - `SQLiteGraphCapability` _class_ — Local, file-backed Context Graph implementation using SQLite.
 - `SQLiteGraphCapabilityConfig` _class_ — Configuration for SQLite Graph Capability.
+- `ingest_sources` _function_ — The source HEADs this db was ingested from, on a fresh read-only connection. A db built
 - `integrity_check` _function_ — Backend self-check (stage 4; the G3 corruption find institutionalized):
 - `query_edges` _function_ — Execute a typed edge query (stage 4) — same contract as `query_nodes`.
 - `query_nodes` _function_ — Execute a typed node query (stage 4) — translated to parameterized SQL
 - `raw_query` _function_ — Execute the raw escape — refuses backend mismatches (non-portable by
+- `record_ingest_sources` _function_ — Record the source HEADs this db was ingested from, REPLACING any earlier record
 
 ### `cjm_capability_graph_sqlite.query_translation`
 
+- `decode_projected_row` _function_ — Zip a projected row, decoding every property column from its JSON representation.
 - `translate_edge_query` _function_ — Translate an `EdgeQuery` to parameterized SQLite SQL.
 - `translate_node_query` _function_ — Translate a `NodeQuery` to parameterized SQLite SQL.
 

cjm_capability_graph_sqlite-0.0.29/cjm_capability_graph_sqlite/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.29"

{cjm_capability_graph_sqlite-0.0.26 → cjm_capability_graph_sqlite-0.0.29}/cjm_capability_graph_sqlite/capability.py

@@ -22,7 +22,11 @@ import time
 from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional, Tuple, Union
 
-from cjm_capability_graph_sqlite.query_translation import translate_edge_query, translate_node_query
+from cjm_capability_graph_sqlite.query_translation import (decode_projected_row,
+                                                           EDGE_STRUCTURAL_KEYS,
+                                                           NODE_STRUCTURAL_KEYS,
+                                                           translate_edge_query,
+                                                           translate_node_query)
 from cjm_context_graph_primitives.graph import GraphContext, GraphEdge, GraphNode
 from cjm_context_graph_primitives.provenance import SourceRef
 from cjm_context_graph_primitives.query import (EdgeQuery, EdgeQueryResult, NodeQuery,
@@ -78,10 +82,21 @@ class SQLiteGraphCapability(ToolCapability):
             FOREIGN KEY(source_id) REFERENCES nodes(id) ON DELETE CASCADE,
             FOREIGN KEY(target_id) REFERENCES nodes(id) ON DELETE CASCADE
         );
+
+        CREATE TABLE ingest_sources (  -- the source HEADs the rebuild ingested (DEC a9176261)
+            source TEXT PRIMARY KEY,
+            head TEXT NOT NULL
+        );
     """
 
     config_class = SQLiteGraphCapabilityConfig
 
+    # Substrate observability class (DEC 8bf656c0 as manifest DATA, finding 0d886ffe B):
+    # graph-storage ops are compute-light, re-derivable work — routine per-op journal
+    # rows and worker-lifecycle rows are noise (they filled 97% of journal.db); failures
+    # still journal. `cjm-ctl regenerate-manifest` records this on the manifest.
+    observability_class = "ambient"
+
     def __init__(self):
         self.logger = logging.getLogger(f"{__name__}.{type(self).__name__}")
         self.config: SQLiteGraphCapabilityConfig = None
@@ -201,6 +216,14 @@ class SQLiteGraphCapability(ToolCapability):
             # them up on the next initialize.
             con.execute("CREATE INDEX IF NOT EXISTS idx_edges_source_type ON edges(source_id, relation_type);")
             con.execute("CREATE INDEX IF NOT EXISTS idx_edges_target_type ON edges(target_id, relation_type);")
+            # The ingest record (DEC a9176261): the source HEADs this db was ingested from —
+            # provenance about the db itself, beside the graph and never in it.
+            con.execute("""
+                CREATE TABLE IF NOT EXISTS ingest_sources (
+                    source TEXT PRIMARY KEY,
+                    head TEXT NOT NULL
+                )
+            """)
             # H7 companion: planner statistics. Without sqlite_stat1 the
             # correlated-probe plans pick the WRONG composite index (the
             # target-side probe scans a whole document's PART_OF rows per
@@ -291,17 +314,23 @@ class SQLiteGraphCapability(ToolCapability):
         self,
         nodes: List[GraphNode]  # Nodes to create
     ) -> List[str]:  # Created node IDs
-        """Bulk create nodes."""
+        """Bulk create nodes.
+
+        A node arriving with created_at/updated_at already set keeps them —
+        journal REPLAY restores temporal provenance this way (finding 0d50b921:
+        stamping now() here clamped every rebuilt node to rebuild time)."""
         ids = []
         now = time.time()
         with self._connect() as con:
             for n in nodes:
                 sources_json = json.dumps([s.to_dict() for s in n.sources])
                 props_json = json.dumps(n.properties)
+                created = n.created_at if n.created_at is not None else now
+                updated = n.updated_at if n.updated_at is not None else created
                 try:
                     con.execute(
                         "INSERT INTO nodes (id, label, properties, sources, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)",
-                        (n.id, n.label, props_json, sources_json, now, now)
+                        (n.id, n.label, props_json, sources_json, created, updated)
                     )
                     ids.append(n.id)
                 except sqlite3.IntegrityError:
@@ -312,21 +341,40 @@ class SQLiteGraphCapability(ToolCapability):
         self,
         edges: List[GraphEdge]  # Edges to create
     ) -> List[str]:  # Created edge IDs
-        """Bulk create edges."""
+        """Bulk create edges.
+
+        Carried created_at/updated_at are honored like add_nodes (0d50b921).
+
+        Missing-endpoint edges (FOREIGN KEY failures) are AGGREGATED into one warning
+        per call — count + a bounded sample — never one line per edge: a rebuild replays
+        every journaled edge whose endpoint is absent, and per-edge logging flooded
+        diagnostics.db to 1.5M rows / 633 MB (finding 0d886ffe). The skipped count is
+        recoverable by the caller as `len(edges) - len(ids)`."""
         ids = []
         now = time.time()
+        missing: List[Tuple[str, str, str]] = []  # (source_id, target_id, relation_type) of FK-skipped edges
         with self._connect() as con:
             con.execute("PRAGMA foreign_keys = ON;")
             for e in edges:
                 props_json = json.dumps(e.properties)
+                created = e.created_at if e.created_at is not None else now
+                updated = e.updated_at if e.updated_at is not None else created
                 try:
                     con.execute(
                         "INSERT INTO edges (id, source_id, target_id, relation_type, properties, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
-                        (e.id, e.source_id, e.target_id, e.relation_type, props_json, now, now)
+                        (e.id, e.source_id, e.target_id, e.relation_type, props_json, created, updated)
                     )
                     ids.append(e.id)
                 except sqlite3.IntegrityError as err:
-                    self.logger.warning(f"Edge creation error (likely missing node): {err}")
+                    if "FOREIGN KEY" in str(err):
+                        missing.append((e.source_id, e.target_id, e.relation_type))
+                    else:
+                        self.logger.warning(f"Edge creation error ({e.id}): {err}")
+        if missing:
+            sample = "; ".join(f"{s[:8]}->{t[:8]} {r}" for s, t, r in missing[:3])
+            self.logger.warning(
+                f"add_edges: {len(missing)} of {len(edges)} edge(s) skipped — endpoint node missing "
+                f"(FOREIGN KEY); sample: {sample}")
         return ids
 
     # -------------------------------------------------------------------------
@@ -490,7 +538,14 @@ class SQLiteGraphCapability(ToolCapability):
         node_id: str,  # UUID of node to update
         properties: Dict[str, Any]  # Properties to merge/update
     ) -> bool:  # True if successful
-        """Partial update of node properties."""
+        """Partial update of node properties.
+
+        A reserved `updated_at` key in `properties` sets the COLUMN instead of
+        now() and never lands in the JSON blob — journal REPLAY restores a STATE
+        op's true time this way (0d50b921 residual: later-STATE nodes clamped
+        to rebuild time)."""
+        properties = dict(properties)
+        stamp = properties.pop("updated_at", None)
         with self._connect() as con:
             # Fetch existing to merge
             cur = con.execute("SELECT properties FROM nodes WHERE id = ?", (node_id,))
@@ -503,7 +558,7 @@ class SQLiteGraphCapability(ToolCapability):
 
             con.execute(
                 "UPDATE nodes SET properties = ?, updated_at = ? WHERE id = ?",
-                (json.dumps(existing), time.time(), node_id)
+                (json.dumps(existing), stamp if stamp is not None else time.time(), node_id)
             )
             return True
 
@@ -512,7 +567,13 @@ class SQLiteGraphCapability(ToolCapability):
         edge_id: str,  # UUID of edge to update
         properties: Dict[str, Any]  # Properties to merge/update
     ) -> bool:  # True if successful
-        """Partial update of edge properties."""
+        """Partial update of edge properties.
+
+        A reserved `updated_at` key in `properties` sets the COLUMN instead of now()
+        and never lands in the JSON blob — the op clock's stamp (design 8f6f2343),
+        exactly as `update_node` takes it."""
+        properties = dict(properties)
+        stamp = properties.pop("updated_at", None)
         with self._connect() as con:
             cur = con.execute("SELECT properties FROM edges WHERE id = ?", (edge_id,))
             row = cur.fetchone()
@@ -524,7 +585,7 @@ class SQLiteGraphCapability(ToolCapability):
 
             con.execute(
                 "UPDATE edges SET properties = ?, updated_at = ? WHERE id = ?",
-                (json.dumps(existing), time.time(), edge_id)
+                (json.dumps(existing), stamp if stamp is not None else time.time(), edge_id)
             )
             return True
 
@@ -613,7 +674,8 @@ class SQLiteGraphCapability(ToolCapability):
         nodes: List[GraphNode],  # Nodes to import
         merge_strategy: str  # "overwrite" | "skip" | "merge"
     ) -> int:  # Count of rows written (inserted or updated)
-        """Import nodes honoring merge_strategy (see import_graph)."""
+        """Import nodes honoring merge_strategy (see import_graph). Carried
+        created_at / updated_at are honored like add_nodes (0d50b921, design 8f6f2343)."""
         now = time.time()
         written = 0
         with self._connect() as con:
@@ -624,7 +686,10 @@ class SQLiteGraphCapability(ToolCapability):
                 if row is None:
                     con.execute(
                         "INSERT INTO nodes (id, label, properties, sources, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)",
-                        (node.id, node.label, props_json, sources_json, now, now),
+                        (node.id, node.label, props_json, sources_json,
+                         node.created_at if node.created_at is not None else now,
+                         node.updated_at if node.updated_at is not None
+                         else (node.created_at if node.created_at is not None else now)),
                     )
                     written += 1
                 elif merge_strategy == "skip":
@@ -632,7 +697,8 @@ class SQLiteGraphCapability(ToolCapability):
                 elif merge_strategy == "overwrite":
                     con.execute(
                         "UPDATE nodes SET label = ?, properties = ?, sources = ?, updated_at = ? WHERE id = ?",
-                        (node.label, props_json, sources_json, now, node.id),
+                        (node.label, props_json, sources_json,
+                         node.updated_at if node.updated_at is not None else now, node.id),
                     )
                     written += 1
                 else:  # merge
@@ -648,7 +714,8 @@ class SQLiteGraphCapability(ToolCapability):
                             seen.add(key)
                     con.execute(
                         "UPDATE nodes SET label = ?, properties = ?, sources = ?, updated_at = ? WHERE id = ?",
-                        (node.label, json.dumps(merged_props), json.dumps(existing_sources), now, node.id),
+                        (node.label, json.dumps(merged_props), json.dumps(existing_sources),
+                         node.updated_at if node.updated_at is not None else now, node.id),
                     )
                     written += 1
         return written
@@ -658,7 +725,8 @@ class SQLiteGraphCapability(ToolCapability):
         edges: List[GraphEdge],  # Edges to import
         merge_strategy: str  # "overwrite" | "skip" | "merge"
     ) -> int:  # Count of rows written (inserted or updated)
-        """Import edges honoring merge_strategy (see import_graph)."""
+        """Import edges honoring merge_strategy (see import_graph). Carried
+        created_at / updated_at are honored like add_edges (0d50b921, design 8f6f2343)."""
         now = time.time()
         written = 0
         with self._connect() as con:
@@ -669,7 +737,10 @@ class SQLiteGraphCapability(ToolCapability):
                     try:
                         con.execute(
                             "INSERT INTO edges (id, source_id, target_id, relation_type, properties, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
-                            (edge.id, edge.source_id, edge.target_id, edge.relation_type, props_json, now, now),
+                            (edge.id, edge.source_id, edge.target_id, edge.relation_type, props_json,
+                             edge.created_at if edge.created_at is not None else now,
+                             edge.updated_at if edge.updated_at is not None
+                             else (edge.created_at if edge.created_at is not None else now)),
                         )
                         written += 1
                     except sqlite3.IntegrityError as err:
@@ -679,7 +750,8 @@ class SQLiteGraphCapability(ToolCapability):
                 elif merge_strategy == "overwrite":
                     con.execute(
                         "UPDATE edges SET source_id = ?, target_id = ?, relation_type = ?, properties = ?, updated_at = ? WHERE id = ?",
-                        (edge.source_id, edge.target_id, edge.relation_type, props_json, now, edge.id),
+                        (edge.source_id, edge.target_id, edge.relation_type, props_json,
+                         edge.updated_at if edge.updated_at is not None else now, edge.id),
                     )
                     written += 1
                 else:  # merge
@@ -687,7 +759,8 @@ class SQLiteGraphCapability(ToolCapability):
                     merged.update(edge.properties)
                     con.execute(
                         "UPDATE edges SET properties = ?, updated_at = ? WHERE id = ?",
-                        (json.dumps(merged), now, edge.id),
+                        (json.dumps(merged), edge.updated_at if edge.updated_at is not None else now,
+                         edge.id),
                     )
                     written += 1
         return written
@@ -782,7 +855,7 @@ def query_nodes(
         return NodeQueryResult(nodes=[self._row_to_node(r) for r in rows])
     out = []
     for r in rows:
-        d = dict(zip(keys, r))
+        d = decode_projected_row(keys, r, NODE_STRUCTURAL_KEYS)  # properties keep their JSON types
         if "sources" in d and d["sources"]:
             d["sources"] = json.loads(d["sources"])  # ref dicts (CR-19 shape)
         out.append(d)
@@ -805,7 +878,7 @@ def query_edges(
         con.close()
     if mode == "full":
         return EdgeQueryResult(edges=[self._row_to_edge(r) for r in rows])
-    return EdgeQueryResult(rows=[dict(zip(keys, r)) for r in rows])
+    return EdgeQueryResult(rows=[decode_projected_row(keys, r, EDGE_STRUCTURAL_KEYS) for r in rows])
 
 
 def raw_query(
@@ -843,7 +916,39 @@ def integrity_check(self) -> Dict[str, Any]:  # {"ok": bool, "errors": [...], "b
     return {"ok": ok, "errors": [] if ok else rows, "backend": "sqlite"}
 
 
+def record_ingest_sources(
+    self,
+    sources: Dict[str, str],  # source id -> the HEAD commit the ingest read
+) -> int:  # Sources recorded
+    """Record the source HEADs this db was ingested from, REPLACING any earlier record
+    (DEC a9176261). The rebuild writes it into the fresh db it builds, so the record and the
+    rows it describes swap in together; it is provenance about the db, not a graph row, so
+    it is neither stamped by the op clock nor seen by the write observer."""
+    rows = sorted((str(k), str(v)) for k, v in dict(sources).items())
+    with self._connect() as con:
+        con.execute("DELETE FROM ingest_sources")
+        con.executemany("INSERT INTO ingest_sources (source, head) VALUES (?, ?)", rows)
+    return len(rows)
+
+
+def ingest_sources(self) -> Dict[str, str]:  # source id -> HEAD; {} when the db holds no record
+    """The source HEADs this db was ingested from, on a fresh read-only connection. A db built
+    before the record existed has no table: it reads as no record, never as an error."""
+    con = sqlite3.connect(f"file:{self._db_path}?mode=ro", uri=True)
+    try:
+        rows = con.execute("SELECT source, head FROM ingest_sources ORDER BY source").fetchall()
+    except sqlite3.OperationalError as e:
+        if "no such table" not in str(e):
+            raise
+        rows = []
+    finally:
+        con.close()
+    return {s: h for s, h in rows}
+
+
 SQLiteGraphCapability.query_nodes = query_nodes
 SQLiteGraphCapability.query_edges = query_edges
 SQLiteGraphCapability.raw_query = raw_query
 SQLiteGraphCapability.integrity_check = integrity_check
+SQLiteGraphCapability.record_ingest_sources = record_ingest_sources
+SQLiteGraphCapability.ingest_sources = ingest_sources

{cjm_capability_graph_sqlite-0.0.26 → cjm_capability_graph_sqlite-0.0.29}/cjm_capability_graph_sqlite/query_translation.py

@@ -22,9 +22,14 @@ Translation rules:
   does Python-side locator equality; a recurring need here is promotion
   evidence).
 - **Modes**: `count` > `project` > full (mirrors the result DTO contract).
-  Projected node rows always carry `id` (+ `label` projectable structurally;
-  `sources` projectable as parsed ref dicts); edge rows always carry
-  `id`/`source_id`/`target_id` (+ `relation_type` projectable)."""
+  Projected node rows always carry `id` (+ `label`/`created_at`/`updated_at`
+  projectable structurally — the storage stamps, never shadowed by a
+  same-named property; `sources` projectable as parsed ref dicts); edge rows
+  always carry `id`/`source_id`/`target_id` (+ `relation_type` projectable).
+  A projected PROPERTY is selected as its JSON representation (`->`) and decoded
+  by the capability, so it keeps its type — a list stays a list, an object an
+  object (`json_extract` returned arrays and objects as JSON TEXT: a projected
+  `calls` list read as a string, the live code fold's index keyed characters)."""
 
 import json
 import re
@@ -230,11 +235,14 @@ def translate_node_query(
             if name == "label":
                 select_parts.append("n.label")
                 keys.append("label")
+            elif name in ("created_at", "updated_at"):
+                select_parts.append("n." + name)   # storage stamp, not a property
+                keys.append(name)
             elif name == "sources":
                 select_parts.append("n.sources")
                 keys.append("sources")
             else:
-                select_parts.append(f"json_extract(n.properties, {_json_path(name)})")
+                select_parts.append(f"n.properties -> {_json_path(name)}")
                 keys.append(name)
         tail = _order_limit_sql(q, "n.properties", params)
         return (f"SELECT {', '.join(select_parts)} FROM nodes AS n{where_sql}{tail}",
@@ -302,7 +310,7 @@ def translate_edge_query(
                 select_parts.append("e.relation_type")
                 keys.append("relation_type")
             else:
-                select_parts.append(f"json_extract(e.properties, {_json_path(name)})")
+                select_parts.append(f"e.properties -> {_json_path(name)}")
                 keys.append(name)
         tail = _order_limit_sql(q, "e.properties", params)
         return (f"SELECT {', '.join(select_parts)} FROM edges AS e{where_sql}{tail}",
@@ -311,3 +319,21 @@ def translate_edge_query(
     tail = _order_limit_sql(q, "e.properties", params)
     return (f"SELECT {EDGE_FULL_COLUMNS} FROM edges AS e{where_sql}{tail}",
             params, "full", None)
+
+
+# The structural columns a projection selects as-is; every other projected key is a property,
+# selected as JSON (`->`) and decoded by `decode_projected_row`.
+NODE_STRUCTURAL_KEYS = frozenset({"id", "label", "created_at", "updated_at", "sources"})
+EDGE_STRUCTURAL_KEYS = frozenset({"id", "source_id", "target_id", "relation_type"})
+
+
+def decode_projected_row(
+    keys: List[str],        # The projected keys, in select order
+    values: Any,            # One cursor row
+    structural: frozenset,  # The keys selected as plain columns
+) -> dict:  # The row: structural columns as-is, each property decoded to its JSON type (absent -> None)
+    """Zip a projected row, decoding every property column from its JSON representation."""
+    out = {}
+    for k, v in zip(keys, values):
+        out[k] = v if k in structural or v is None else json.loads(v)
+    return out

{cjm_capability_graph_sqlite-0.0.26 → cjm_capability_graph_sqlite-0.0.29/cjm_capability_graph_sqlite.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjm-capability-graph-sqlite
-Version: 0.0.26
+Version: 0.0.29
 Summary: A local, file-backed context-graph storage capability for the cjm-substrate runtime that implements graph storage, traversal, and querying using SQLite.
 Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
 License: Apache-2.0
@@ -26,6 +26,7 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 ## Modules
 
+- **`cjm_capability_graph_sqlite.__init__`**
 - **`cjm_capability_graph_sqlite.capability`** — Capability implementation for Context Graph using SQLite.
 - **`cjm_capability_graph_sqlite.query_translation`** — The per-backend translation of the typed query expressions (pass-2 Thread 5; stage 4): NodeQuery/EdgeQuery -> parameterized SQLite SQL over the nodes/edges schema. THIS module is what makes the typed surface portable - the expression is domain- and backend-neutral; every backend tool owns a translation like this one (the ratified stage-4 split: backend owns translation; the adapter stays generic). Pure functions, unit-tested against an in-memory DB with the production schema - no capability runtime needed.
 
@@ -35,13 +36,16 @@ A local, file-backed context-graph storage capability for the cjm-substrate runt
 
 - `SQLiteGraphCapability` _class_ — Local, file-backed Context Graph implementation using SQLite.
 - `SQLiteGraphCapabilityConfig` _class_ — Configuration for SQLite Graph Capability.
+- `ingest_sources` _function_ — The source HEADs this db was ingested from, on a fresh read-only connection. A db built
 - `integrity_check` _function_ — Backend self-check (stage 4; the G3 corruption find institutionalized):
 - `query_edges` _function_ — Execute a typed edge query (stage 4) — same contract as `query_nodes`.
 - `query_nodes` _function_ — Execute a typed node query (stage 4) — translated to parameterized SQL
 - `raw_query` _function_ — Execute the raw escape — refuses backend mismatches (non-portable by
+- `record_ingest_sources` _function_ — Record the source HEADs this db was ingested from, REPLACING any earlier record
 
 ### `cjm_capability_graph_sqlite.query_translation`
 
+- `decode_projected_row` _function_ — Zip a projected row, decoding every property column from its JSON representation.
 - `translate_edge_query` _function_ — Translate an `EdgeQuery` to parameterized SQLite SQL.
 - `translate_node_query` _function_ — Translate a `NodeQuery` to parameterized SQLite SQL.
 

{cjm_capability_graph_sqlite-0.0.26 → cjm_capability_graph_sqlite-0.0.29}/tests/test_capability.py

@@ -177,3 +177,111 @@ def test_typed_surface_smoke(cap):
     ctx = cap.export_graph(NodeQuery(label="Segment",
                                      related=RelationPredicate("PART_OF", node_id=doc)))
     assert len(ctx.nodes) == 3 and len(ctx.edges) == 2  # NEXT only (PART_OF targets the doc)
+
+
+def test_update_node_reserved_updated_at_sets_column(cap):
+    # 0d50b921 residual: a reserved `updated_at` key in the update payload sets
+    # the COLUMN (journal replay restores a STATE op's true time) and never
+    # lands in the JSON properties blob; without it, now()-stamping stands.
+    import json
+
+    nid = str(uuid.uuid4())
+    cap.add_nodes([GraphNode(id=nid, label="Person", properties={"name": "Ada"})])
+    assert cap.update_node(nid, {"role": "host", "updated_at": 1234.5})
+    con = sqlite3.connect(cap._db_path)
+    row = con.execute("SELECT properties, updated_at FROM nodes WHERE id = ?",
+                      (nid,)).fetchone()
+    con.close()
+    props = json.loads(row[0])
+    assert props["role"] == "host" and "updated_at" not in props
+    assert row[1] == 1234.5
+    # Without the reserved key the column re-stamps to now().
+    assert cap.update_node(nid, {"role": "guest"})
+    con = sqlite3.connect(cap._db_path)
+    row2 = con.execute("SELECT updated_at FROM nodes WHERE id = ?", (nid,)).fetchone()
+    con.close()
+    assert row2[0] > 1234.5
+
+
+def test_add_edges_aggregates_missing_endpoint_warnings(cap, caplog):
+    """Finding 0d886ffe: FK-skipped edges log ONE warning per call (count + sample),
+    never one line per edge — per-edge logging flooded diagnostics.db during rebuilds."""
+    import logging
+    from cjm_context_graph_primitives.graph import GraphEdge, GraphNode
+    cap.add_nodes([GraphNode(id="n-live", label="T", properties={}, sources=[])])
+    edges = [GraphEdge(id=f"e{i}", source_id="n-live", target_id=f"n-missing-{i}",
+                       relation_type="REL", properties={}) for i in range(25)]
+    edges.append(GraphEdge(id="e-ok", source_id="n-live", target_id="n-live",
+                           relation_type="SELF", properties={}))
+    with caplog.at_level(logging.WARNING):
+        ids = cap.add_edges(edges)
+    assert ids == ["e-ok"]  # skipped count recoverable as len(edges) - len(ids)
+    fk_lines = [r for r in caplog.records if "endpoint node missing" in r.getMessage()]
+    assert len(fk_lines) == 1
+    msg = fk_lines[0].getMessage()
+    assert "25 of 26" in msg and "sample:" in msg
+    assert not any("Edge creation error (likely missing node)" in r.getMessage() for r in caplog.records)
+
+
+def test_update_edge_reserved_updated_at_sets_column(cap):
+    # The op clock (design 8f6f2343): update_edge takes the reserved `updated_at` exactly as
+    # update_node does — the COLUMN, never the JSON blob; without it, now()-stamping stands.
+    import json
+
+    a, b = str(uuid.uuid4()), str(uuid.uuid4())
+    cap.add_nodes([GraphNode(id=a, label="Person"), GraphNode(id=b, label="Person")])
+    eid = str(uuid.uuid4())
+    cap.add_edges([GraphEdge(id=eid, source_id=a, target_id=b, relation_type="KNOWS")])
+    assert cap.update_edge(eid, {"since": 1999, "updated_at": 1234.5})
+    con = sqlite3.connect(cap._db_path)
+    props, stamp = con.execute("SELECT properties, updated_at FROM edges WHERE id = ?",
+                               (eid,)).fetchone()
+    con.close()
+    assert json.loads(props) == {"since": 1999} and stamp == 1234.5
+    assert cap.update_edge(eid, {"since": 2000})
+    con = sqlite3.connect(cap._db_path)
+    (stamp2,) = con.execute("SELECT updated_at FROM edges WHERE id = ?", (eid,)).fetchone()
+    con.close()
+    assert stamp2 > 1234.5
+
+
+def test_import_honors_carried_times(cap):
+    # import_graph keeps carried created_at / updated_at like add_nodes / add_edges.
+    a, b = str(uuid.uuid4()), str(uuid.uuid4())
+    eid = str(uuid.uuid4())
+    cap.import_graph(GraphContext(
+        nodes=[GraphNode(id=a, label="P", created_at=10.0, updated_at=20.0),
+               GraphNode(id=b, label="P", created_at=30.0)],
+        edges=[GraphEdge(id=eid, source_id=a, target_id=b, relation_type="R", created_at=40.0)]))
+    con = sqlite3.connect(cap._db_path)
+    rows = dict((r[0], r[1:]) for r in con.execute("SELECT id, created_at, updated_at FROM nodes"))
+    erow = con.execute("SELECT created_at, updated_at FROM edges WHERE id = ?", (eid,)).fetchone()
+    con.close()
+    assert rows[a] == (10.0, 20.0) and rows[b] == (30.0, 30.0) and erow == (40.0, 40.0)
+
+
+def test_ingest_record_replaces_reads_readonly_and_tolerates_a_legacy_db(cap, tmp_path):
+    """The ingest record (DEC a9176261): a replace, never a merge; beside the graph (no row,
+    no op-clock stamp); readable through a read-only open; a db built before the table
+    existed reads as no record."""
+    assert cap.ingest_sources() == {}
+    assert cap.record_ingest_sources({"archive:/site/posts": "a" * 40, "repo:/r/x": "b" * 40}) == 2
+    assert cap.record_ingest_sources({"repo:/r/x": "c" * 40}) == 1
+    assert cap.ingest_sources() == {"repo:/r/x": "c" * 40}
+    assert cap.get_schema()["counts"] == {}  # beside the graph, never in it
+
+    ro = SQLiteGraphCapability()
+    ro.initialize({"db_path": cap._db_path, "readonly": True})
+    assert ro.ingest_sources() == {"repo:/r/x": "c" * 40}
+    ro.cleanup()
+
+    legacy = str(tmp_path / "legacy.db")
+    con = sqlite3.connect(legacy)
+    con.execute("CREATE TABLE nodes (id TEXT PRIMARY KEY, label TEXT NOT NULL, properties JSON, "
+                "sources JSON, created_at REAL, updated_at REAL)")
+    con.commit()
+    con.close()
+    old = SQLiteGraphCapability()
+    old.initialize({"db_path": legacy, "readonly": True})
+    assert old.ingest_sources() == {}
+    old.cleanup()

{cjm_capability_graph_sqlite-0.0.26 → cjm_capability_graph_sqlite-0.0.29}/tests/test_query_translation.py

@@ -8,7 +8,10 @@ import sqlite3
 
 import pytest
 
-from cjm_capability_graph_sqlite.query_translation import (translate_edge_query,
+from cjm_capability_graph_sqlite.query_translation import (EDGE_STRUCTURAL_KEYS,
+                                                           NODE_STRUCTURAL_KEYS,
+                                                           decode_projected_row,
+                                                           translate_edge_query,
                                                            translate_node_query)
 from cjm_context_graph_primitives.locators import FileRef
 from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy,
@@ -71,7 +74,8 @@ def run(con, q):
         return cur.fetchone()[0]
     rows = cur.fetchall()
     if mode == "rows":
-        return [dict(zip(keys, r)) for r in rows]
+        structural = NODE_STRUCTURAL_KEYS if isinstance(q, NodeQuery) else EDGE_STRUCTURAL_KEYS
+        return [decode_projected_row(keys, r, structural) for r in rows]
     return rows
 
 
@@ -85,6 +89,33 @@ def test_spine_read_ordered_projection(con):
     assert json.loads(rows[1]["sources"])[0]["content_hash"] == "sha256:ab"
 
 
+def test_projected_properties_keep_their_json_types(con):
+    # A projected object stays an object and a missing key is None (json_extract used to
+    # hand arrays and objects back as JSON TEXT — a projected list read as a string).
+    rows = run(con, NodeQuery(label="Correction", project=["payload", "correction_type"]))
+    by = {r["id"]: r for r in rows}
+    assert by["c1"]["payload"] == {"document_id": "doc-1", "segment_id": "s1"}
+    assert by["c2"]["correction_type"] == "text_content"
+    rows = run(con, NodeQuery(label="Segment", related=RelationPredicate("PART_OF", node_id="doc-1"),
+                              order_by=OrderBy(prop="index"), project=["text", "start_time"]))
+    assert [r["text"] for r in rows] == ["It's 100% real_deal", "hello world", "", None]
+    assert rows[1]["start_time"] == 1.0
+    rows = run(con, EdgeQuery(relation_type="REVIEWED", project=["decision"]))
+    assert rows == [{"id": "rev", "source_id": "sess-1", "target_id": "s0", "decision": "corrected"}]
+
+
+def test_structural_timestamp_projection(con):
+    # created_at/updated_at project from the storage COLUMNS (the stamps),
+    # not json_extract over properties — no Segment carries them as props.
+    rows = run(con, NodeQuery(label="Segment",
+                              related=RelationPredicate("PART_OF", node_id="doc-1"),
+                              order_by=OrderBy(prop="index"),
+                              project=["index", "created_at", "updated_at"]))
+    assert [r["created_at"] for r in rows] == [1.0, 1.0, 1.0, 1.0]
+    assert rows[0]["updated_at"] == 1.0
+    assert rows[0]["id"] == "s0"
+
+
 def test_empty_filter_or_case_two_counts(con):
     empty_eq = run(con, NodeQuery(label="Segment", where=[PropertyPredicate("text", "eq", "")],
                                   related=RelationPredicate("PART_OF", node_id="doc-1"), count=True))

cjm_capability_graph_sqlite-0.0.26/cjm_capability_graph_sqlite/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.26"