cjm-capability-graph-sqlite 0.0.29__tar.gz → 0.0.31__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cjm_capability_graph_sqlite-0.0.29/cjm_capability_graph_sqlite.egg-info → cjm_capability_graph_sqlite-0.0.31}/PKG-INFO +2 -2
- cjm_capability_graph_sqlite-0.0.31/cjm_capability_graph_sqlite/__init__.py +1 -0
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/cjm_capability_graph_sqlite/capability.py +76 -24
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/cjm_capability_graph_sqlite/query_translation.py +19 -7
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31/cjm_capability_graph_sqlite.egg-info}/PKG-INFO +2 -2
- cjm_capability_graph_sqlite-0.0.31/cjm_capability_graph_sqlite.egg-info/requires.txt +2 -0
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/pyproject.toml +1 -1
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/tests/test_capability.py +53 -4
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/tests/test_query_translation.py +26 -6
- cjm_capability_graph_sqlite-0.0.29/cjm_capability_graph_sqlite/__init__.py +0 -1
- cjm_capability_graph_sqlite-0.0.29/cjm_capability_graph_sqlite.egg-info/requires.txt +0 -2
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/LICENSE +0 -0
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/MANIFEST.in +0 -0
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/README.md +0 -0
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/cjm_capability_graph_sqlite.egg-info/SOURCES.txt +0 -0
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/cjm_capability_graph_sqlite.egg-info/dependency_links.txt +0 -0
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/cjm_capability_graph_sqlite.egg-info/top_level.txt +0 -0
- {cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cjm-capability-graph-sqlite
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.31
|
|
4
4
|
Summary: A local, file-backed context-graph storage capability for the cjm-substrate runtime that implements graph storage, traversal, and querying using SQLite.
|
|
5
5
|
Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -15,7 +15,7 @@ Requires-Python: >=3.12
|
|
|
15
15
|
Description-Content-Type: text/markdown
|
|
16
16
|
License-File: LICENSE
|
|
17
17
|
Requires-Dist: cjm-substrate>=0.0.51
|
|
18
|
-
Requires-Dist: cjm-context-graph-primitives>=0.0.
|
|
18
|
+
Requires-Dist: cjm-context-graph-primitives>=0.0.18
|
|
19
19
|
Dynamic: license-file
|
|
20
20
|
|
|
21
21
|
# cjm-capability-graph-sqlite
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.31"
|
|
@@ -19,8 +19,9 @@ import logging
|
|
|
19
19
|
import os
|
|
20
20
|
import sqlite3
|
|
21
21
|
import time
|
|
22
|
+
from contextlib import contextmanager
|
|
22
23
|
from dataclasses import dataclass, field
|
|
23
|
-
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
24
|
+
from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
|
|
24
25
|
|
|
25
26
|
from cjm_capability_graph_sqlite.query_translation import (decode_projected_row,
|
|
26
27
|
EDGE_STRUCTURAL_KEYS,
|
|
@@ -40,6 +41,7 @@ from cjm_substrate.utils.validation import (config_to_dict, dataclass_to_jsonsch
|
|
|
40
41
|
@dataclass
|
|
41
42
|
class SQLiteGraphCapabilityConfig:
|
|
42
43
|
"""Configuration for SQLite Graph Capability."""
|
|
44
|
+
|
|
43
45
|
db_path: Optional[str] = field(
|
|
44
46
|
default=None,
|
|
45
47
|
metadata={
|
|
@@ -173,8 +175,9 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
173
175
|
return os.path.join(data_dir, "context_graph.db")
|
|
174
176
|
|
|
175
177
|
def _init_db(self) -> None:
|
|
176
|
-
"""Create tables and indices.
|
|
177
|
-
|
|
178
|
+
"""Create tables and indices -- the schema only. Statistics are written by the
|
|
179
|
+
paths that change data (`_writing`), never by an open (finding ff3ede3d)."""
|
|
180
|
+
with self._writing() as con:
|
|
178
181
|
# Enable Foreign Keys
|
|
179
182
|
con.execute("PRAGMA foreign_keys = ON;")
|
|
180
183
|
|
|
@@ -189,7 +192,11 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
189
192
|
updated_at REAL
|
|
190
193
|
)
|
|
191
194
|
""")
|
|
192
|
-
|
|
195
|
+
# (label, id): a label read in id order -- a keyset page, a bounded read -- is a
|
|
196
|
+
# range scan, never a sort of the whole label (findings a107e2f3 / ae2b196e); it
|
|
197
|
+
# serves a label-only lookup too, so the label-only index it replaces is dropped.
|
|
198
|
+
con.execute("CREATE INDEX IF NOT EXISTS idx_nodes_label_id ON nodes(label, id);")
|
|
199
|
+
con.execute("DROP INDEX IF EXISTS idx_nodes_label;")
|
|
193
200
|
|
|
194
201
|
# Edges Table
|
|
195
202
|
con.execute("""
|
|
@@ -227,15 +234,10 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
227
234
|
# H7 companion: planner statistics. Without sqlite_stat1 the
|
|
228
235
|
# correlated-probe plans pick the WRONG composite index (the
|
|
229
236
|
# target-side probe scans a whole document's PART_OF rows per
|
|
230
|
-
# outer row: 5s; source-side probe: 8ms at corpus scale).
|
|
231
|
-
#
|
|
232
|
-
#
|
|
233
|
-
#
|
|
234
|
-
# fail at initialize via a stats side-effect.
|
|
235
|
-
try:
|
|
236
|
-
con.execute("ANALYZE;")
|
|
237
|
-
except sqlite3.DatabaseError as e:
|
|
238
|
-
logging.warning(f"ANALYZE skipped (run integrity_check): {e}")
|
|
237
|
+
# outer row: 5s; source-side probe: 8ms at corpus scale). They are
|
|
238
|
+
# kept by `_writing`'s statistics rule at the close of every write,
|
|
239
|
+
# this one included (a new index is analyzed there) -- no longer a
|
|
240
|
+
# full ANALYZE per open, O(db) on every read (finding ff3ede3d).
|
|
239
241
|
|
|
240
242
|
def _connect(self) -> sqlite3.Connection: # Open DB connection (honors readonly; sets WAL + FKs)
|
|
241
243
|
"""Open a SQLite connection honoring the `readonly` config (SG-41).
|
|
@@ -253,6 +255,54 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
253
255
|
con.execute("PRAGMA foreign_keys = ON;")
|
|
254
256
|
return con
|
|
255
257
|
|
|
258
|
+
@contextmanager
|
|
259
|
+
def _writing(self) -> Iterator[sqlite3.Connection]: # One unit of write work: commit, planner upkeep, close
|
|
260
|
+
"""Open a connection for one unit of write work: commit on success (roll back on
|
|
261
|
+
error), then keep the planner's statistics current, then close.
|
|
262
|
+
|
|
263
|
+
Statistics are written on the paths that change data, never on an open (finding
|
|
264
|
+
ff3ede3d: a read paid a full ANALYZE and took the write lock): `_refresh_statistics`
|
|
265
|
+
analyzes a table only when its statistics are missing or stale, so a steady-state write
|
|
266
|
+
pays a ~0.01 ms check and a fresh rebuild db gains statistics as it fills. Guarded:
|
|
267
|
+
statistics are an OPTIMIZATION — a corrupted DB must still answer integrity_check (the
|
|
268
|
+
designed G3 gate)."""
|
|
269
|
+
con = self._connect()
|
|
270
|
+
try:
|
|
271
|
+
with con:
|
|
272
|
+
yield con
|
|
273
|
+
try:
|
|
274
|
+
self._refresh_statistics(con)
|
|
275
|
+
except sqlite3.DatabaseError as e:
|
|
276
|
+
logging.warning(f"statistics skipped (run integrity_check): {e}")
|
|
277
|
+
finally:
|
|
278
|
+
con.close()
|
|
279
|
+
|
|
280
|
+
def _refresh_statistics(
|
|
281
|
+
self,
|
|
282
|
+
con: sqlite3.Connection, # A write connection, after its unit of work committed
|
|
283
|
+
) -> None:
|
|
284
|
+
"""Analyze each graph table whose planner statistics are missing or stale: an index
|
|
285
|
+
with no sqlite_stat1 row (never analyzed, or created since), or a size ~10x the
|
|
286
|
+
analyzed one (max(rowid), an O(log n) estimate). One rule on every SQLite: PRAGMA
|
|
287
|
+
optimize analyzes a never-analyzed table only from SQLite 3.46 on, so on 3.45 (CI's
|
|
288
|
+
Ubuntu 24.04) the write path kept no statistics at all -- the guard of the 162 s
|
|
289
|
+
planner regression gone."""
|
|
290
|
+
for table in ("nodes", "edges"):
|
|
291
|
+
rows = con.execute(f"SELECT max(rowid) FROM {table}").fetchone()[0] or 0
|
|
292
|
+
if not rows:
|
|
293
|
+
continue
|
|
294
|
+
indexes = {r[0] for r in con.execute(
|
|
295
|
+
"SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = ?", (table,))}
|
|
296
|
+
try:
|
|
297
|
+
stats = dict(con.execute(
|
|
298
|
+
"SELECT idx, stat FROM sqlite_stat1 WHERE tbl = ?", (table,)).fetchall())
|
|
299
|
+
except sqlite3.OperationalError: # no sqlite_stat1 yet: never analyzed
|
|
300
|
+
stats = {}
|
|
301
|
+
analyzed = max((int(str(s).split()[0]) for s in stats.values() if s), default=0)
|
|
302
|
+
if indexes - set(stats) or rows >= 10 * max(analyzed, 1):
|
|
303
|
+
con.execute(f"ANALYZE {table};")
|
|
304
|
+
con.commit()
|
|
305
|
+
|
|
256
306
|
# -------------------------------------------------------------------------
|
|
257
307
|
# Helpers
|
|
258
308
|
# -------------------------------------------------------------------------
|
|
@@ -321,7 +371,7 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
321
371
|
stamping now() here clamped every rebuilt node to rebuild time)."""
|
|
322
372
|
ids = []
|
|
323
373
|
now = time.time()
|
|
324
|
-
with self.
|
|
374
|
+
with self._writing() as con:
|
|
325
375
|
for n in nodes:
|
|
326
376
|
sources_json = json.dumps([s.to_dict() for s in n.sources])
|
|
327
377
|
props_json = json.dumps(n.properties)
|
|
@@ -353,7 +403,7 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
353
403
|
ids = []
|
|
354
404
|
now = time.time()
|
|
355
405
|
missing: List[Tuple[str, str, str]] = [] # (source_id, target_id, relation_type) of FK-skipped edges
|
|
356
|
-
with self.
|
|
406
|
+
with self._writing() as con:
|
|
357
407
|
con.execute("PRAGMA foreign_keys = ON;")
|
|
358
408
|
for e in edges:
|
|
359
409
|
props_json = json.dumps(e.properties)
|
|
@@ -436,11 +486,13 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
436
486
|
label: str, # Node label to search for
|
|
437
487
|
limit: int = 100 # Max results
|
|
438
488
|
) -> List[GraphNode]: # Matching nodes
|
|
439
|
-
"""Find nodes by label
|
|
489
|
+
"""Find nodes by label, in id order: a bounded read sorts before it limits, so a
|
|
490
|
+
page is never an arbitrary subset (finding a107e2f3)."""
|
|
440
491
|
results = []
|
|
441
492
|
with self._connect() as con:
|
|
442
493
|
cur = con.execute(
|
|
443
|
-
"SELECT id, label, properties, sources, created_at, updated_at FROM nodes
|
|
494
|
+
"SELECT id, label, properties, sources, created_at, updated_at FROM nodes "
|
|
495
|
+
"WHERE label = ? ORDER BY id LIMIT ?",
|
|
444
496
|
(label, limit)
|
|
445
497
|
)
|
|
446
498
|
for row in cur:
|
|
@@ -546,7 +598,7 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
546
598
|
to rebuild time)."""
|
|
547
599
|
properties = dict(properties)
|
|
548
600
|
stamp = properties.pop("updated_at", None)
|
|
549
|
-
with self.
|
|
601
|
+
with self._writing() as con:
|
|
550
602
|
# Fetch existing to merge
|
|
551
603
|
cur = con.execute("SELECT properties FROM nodes WHERE id = ?", (node_id,))
|
|
552
604
|
row = cur.fetchone()
|
|
@@ -574,7 +626,7 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
574
626
|
exactly as `update_node` takes it."""
|
|
575
627
|
properties = dict(properties)
|
|
576
628
|
stamp = properties.pop("updated_at", None)
|
|
577
|
-
with self.
|
|
629
|
+
with self._writing() as con:
|
|
578
630
|
cur = con.execute("SELECT properties FROM edges WHERE id = ?", (edge_id,))
|
|
579
631
|
row = cur.fetchone()
|
|
580
632
|
if not row:
|
|
@@ -599,7 +651,7 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
599
651
|
cascade: bool = True # Also delete connected edges
|
|
600
652
|
) -> int: # Number of nodes deleted
|
|
601
653
|
"""Delete nodes (and optionally connected edges)."""
|
|
602
|
-
with self.
|
|
654
|
+
with self._writing() as con:
|
|
603
655
|
if cascade:
|
|
604
656
|
con.execute("PRAGMA foreign_keys = ON;") # Ensures cascade works
|
|
605
657
|
else:
|
|
@@ -617,7 +669,7 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
617
669
|
edge_ids: List[str] # UUIDs of edges to delete
|
|
618
670
|
) -> int: # Number of edges deleted
|
|
619
671
|
"""Delete edges."""
|
|
620
|
-
with self.
|
|
672
|
+
with self._writing() as con:
|
|
621
673
|
placeholders = ','.join('?' for _ in edge_ids)
|
|
622
674
|
cur = con.execute(
|
|
623
675
|
f"DELETE FROM edges WHERE id IN ({placeholders})",
|
|
@@ -678,7 +730,7 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
678
730
|
created_at / updated_at are honored like add_nodes (0d50b921, design 8f6f2343)."""
|
|
679
731
|
now = time.time()
|
|
680
732
|
written = 0
|
|
681
|
-
with self.
|
|
733
|
+
with self._writing() as con:
|
|
682
734
|
for node in nodes:
|
|
683
735
|
props_json = json.dumps(node.properties)
|
|
684
736
|
sources_json = json.dumps([s.to_dict() for s in node.sources])
|
|
@@ -729,7 +781,7 @@ class SQLiteGraphCapability(ToolCapability):
|
|
|
729
781
|
created_at / updated_at are honored like add_edges (0d50b921, design 8f6f2343)."""
|
|
730
782
|
now = time.time()
|
|
731
783
|
written = 0
|
|
732
|
-
with self.
|
|
784
|
+
with self._writing() as con:
|
|
733
785
|
for edge in edges:
|
|
734
786
|
props_json = json.dumps(edge.properties)
|
|
735
787
|
row = con.execute("SELECT properties FROM edges WHERE id = ?", (edge.id,)).fetchone()
|
|
@@ -925,7 +977,7 @@ def record_ingest_sources(
|
|
|
925
977
|
rows it describes swap in together; it is provenance about the db, not a graph row, so
|
|
926
978
|
it is neither stamped by the op clock nor seen by the write observer."""
|
|
927
979
|
rows = sorted((str(k), str(v)) for k, v in dict(sources).items())
|
|
928
|
-
with self.
|
|
980
|
+
with self._writing() as con:
|
|
929
981
|
con.execute("DELETE FROM ingest_sources")
|
|
930
982
|
con.executemany("INSERT INTO ingest_sources (source, head) VALUES (?, ?)", rows)
|
|
931
983
|
return len(rows)
|
|
@@ -171,16 +171,22 @@ EDGE_FULL_COLUMNS = "e.id, e.source_id, e.target_id, e.relation_type, e.properti
|
|
|
171
171
|
|
|
172
172
|
|
|
173
173
|
def _order_limit_sql(
|
|
174
|
-
query, # NodeQuery or EdgeQuery (order_by / limit / offset fields)
|
|
174
|
+
query, # NodeQuery or EdgeQuery (order_by / limit / offset / after fields)
|
|
175
175
|
props_col: str, # Properties column expression for ORDER BY paths
|
|
176
|
+
id_col: str, # Row id column: the keyset key and the tie-break
|
|
176
177
|
params: List[Any], # Bound-params list (appended in place)
|
|
177
178
|
) -> str: # ORDER BY / LIMIT / OFFSET tail
|
|
178
|
-
"""Compile the shared ordering + paging tail.
|
|
179
|
+
"""Compile the shared ordering + paging tail. A property ordering breaks ties on the
|
|
180
|
+
id, and a bounded or keyset read with no ordering is ordered by the id, so a page is
|
|
181
|
+
never an arbitrary subset (findings a107e2f3 / ae2b196e)."""
|
|
179
182
|
tail = ""
|
|
180
183
|
if query.order_by is not None:
|
|
181
184
|
tail += (f" ORDER BY json_extract({props_col}, "
|
|
182
185
|
f"{_json_path(query.order_by.prop)})"
|
|
183
|
-
+ (" DESC" if query.order_by.descending else " ASC")
|
|
186
|
+
+ (" DESC" if query.order_by.descending else " ASC")
|
|
187
|
+
+ f", {id_col}")
|
|
188
|
+
elif query.limit is not None or query.offset or query.after is not None:
|
|
189
|
+
tail += f" ORDER BY {id_col}"
|
|
184
190
|
if query.limit is not None or query.offset:
|
|
185
191
|
tail += " LIMIT ?"
|
|
186
192
|
params.append(query.limit if query.limit is not None else -1)
|
|
@@ -206,6 +212,9 @@ def translate_node_query(
|
|
|
206
212
|
if q.ids is not None:
|
|
207
213
|
where.append("n.id IN (SELECT value FROM json_each(?))")
|
|
208
214
|
params.append(json.dumps(list(q.ids)))
|
|
215
|
+
if q.after is not None:
|
|
216
|
+
where.append("n.id > ?")
|
|
217
|
+
params.append(q.after)
|
|
209
218
|
if q.label is not None:
|
|
210
219
|
where.append("n.label = ?")
|
|
211
220
|
params.append(q.label)
|
|
@@ -244,11 +253,11 @@ def translate_node_query(
|
|
|
244
253
|
else:
|
|
245
254
|
select_parts.append(f"n.properties -> {_json_path(name)}")
|
|
246
255
|
keys.append(name)
|
|
247
|
-
tail = _order_limit_sql(q, "n.properties", params)
|
|
256
|
+
tail = _order_limit_sql(q, "n.properties", "n.id", params)
|
|
248
257
|
return (f"SELECT {', '.join(select_parts)} FROM nodes AS n{where_sql}{tail}",
|
|
249
258
|
params, "rows", keys)
|
|
250
259
|
|
|
251
|
-
tail = _order_limit_sql(q, "n.properties", params)
|
|
260
|
+
tail = _order_limit_sql(q, "n.properties", "n.id", params)
|
|
252
261
|
return (f"SELECT {NODE_FULL_COLUMNS} FROM nodes AS n{where_sql}{tail}",
|
|
253
262
|
params, "full", None)
|
|
254
263
|
|
|
@@ -268,6 +277,9 @@ def translate_edge_query(
|
|
|
268
277
|
if q.ids is not None:
|
|
269
278
|
where.append("e.id IN (SELECT value FROM json_each(?))")
|
|
270
279
|
params.append(json.dumps(list(q.ids)))
|
|
280
|
+
if q.after is not None:
|
|
281
|
+
where.append("e.id > ?")
|
|
282
|
+
params.append(q.after)
|
|
271
283
|
if q.relation_type is not None:
|
|
272
284
|
where.append("e.relation_type = ?")
|
|
273
285
|
params.append(q.relation_type)
|
|
@@ -312,11 +324,11 @@ def translate_edge_query(
|
|
|
312
324
|
else:
|
|
313
325
|
select_parts.append(f"e.properties -> {_json_path(name)}")
|
|
314
326
|
keys.append(name)
|
|
315
|
-
tail = _order_limit_sql(q, "e.properties", params)
|
|
327
|
+
tail = _order_limit_sql(q, "e.properties", "e.id", params)
|
|
316
328
|
return (f"SELECT {', '.join(select_parts)} FROM edges AS e{where_sql}{tail}",
|
|
317
329
|
params, "rows", keys)
|
|
318
330
|
|
|
319
|
-
tail = _order_limit_sql(q, "e.properties", params)
|
|
331
|
+
tail = _order_limit_sql(q, "e.properties", "e.id", params)
|
|
320
332
|
return (f"SELECT {EDGE_FULL_COLUMNS} FROM edges AS e{where_sql}{tail}",
|
|
321
333
|
params, "full", None)
|
|
322
334
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cjm-capability-graph-sqlite
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.31
|
|
4
4
|
Summary: A local, file-backed context-graph storage capability for the cjm-substrate runtime that implements graph storage, traversal, and querying using SQLite.
|
|
5
5
|
Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -15,7 +15,7 @@ Requires-Python: >=3.12
|
|
|
15
15
|
Description-Content-Type: text/markdown
|
|
16
16
|
License-File: LICENSE
|
|
17
17
|
Requires-Dist: cjm-substrate>=0.0.51
|
|
18
|
-
Requires-Dist: cjm-context-graph-primitives>=0.0.
|
|
18
|
+
Requires-Dist: cjm-context-graph-primitives>=0.0.18
|
|
19
19
|
Dynamic: license-file
|
|
20
20
|
|
|
21
21
|
# cjm-capability-graph-sqlite
|
|
@@ -12,7 +12,7 @@ license = {text = "Apache-2.0"}
|
|
|
12
12
|
authors = [{name = "Christian J. Mills", email = "9126128+cj-mills@users.noreply.github.com"}]
|
|
13
13
|
keywords = ['python']
|
|
14
14
|
classifiers = ["Natural Language :: English", "Intended Audience :: Developers", "Development Status :: 3 - Alpha", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only"]
|
|
15
|
-
dependencies = ['cjm-substrate>=0.0.51', 'cjm-context-graph-primitives>=0.0.
|
|
15
|
+
dependencies = ['cjm-substrate>=0.0.51', 'cjm-context-graph-primitives>=0.0.18']
|
|
16
16
|
|
|
17
17
|
[project.urls]
|
|
18
18
|
Repository = "https://github.com/cj-mills/cjm-capability-graph-sqlite"
|
{cjm_capability_graph_sqlite-0.0.29 → cjm_capability_graph_sqlite-0.0.31}/tests/test_capability.py
RENAMED
|
@@ -3,18 +3,17 @@
|
|
|
3
3
|
Projected from the capability notebook's demo/test cells at the c25780e8 flip:
|
|
4
4
|
the CRUD/round-trip walk, the SG-41 smells (WAL / readonly / guarded query /
|
|
5
5
|
merge_strategy), and the stage-4 typed-surface smoke test."""
|
|
6
|
+
|
|
6
7
|
import sqlite3
|
|
7
8
|
import uuid
|
|
8
9
|
|
|
9
10
|
import pytest
|
|
10
|
-
|
|
11
11
|
from cjm_capability_graph_sqlite.capability import SQLiteGraphCapability
|
|
12
12
|
from cjm_context_graph_primitives.graph import GraphContext, GraphEdge, GraphNode
|
|
13
13
|
from cjm_context_graph_primitives.locators import FileRef
|
|
14
14
|
from cjm_context_graph_primitives.provenance import SourceRef
|
|
15
|
-
from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy,
|
|
16
|
-
|
|
17
|
-
RelationPredicate)
|
|
15
|
+
from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy, PropertyPredicate,
|
|
16
|
+
RawQuery, RelationPredicate)
|
|
18
17
|
from cjm_context_graph_primitives.slices import FullContent
|
|
19
18
|
from cjm_substrate.core.errors import CapabilityInputError
|
|
20
19
|
|
|
@@ -140,6 +139,56 @@ def test_sg41_wal_query_guard_merge_readonly(tmp_path):
|
|
|
140
139
|
ro.cleanup()
|
|
141
140
|
|
|
142
141
|
|
|
142
|
+
def test_reads_and_opens_never_write_writes_keep_statistics(tmp_path):
|
|
143
|
+
# Finding ff3ede3d: statistics are written by the paths that change data (the
|
|
144
|
+
# statistics rule at the close of each write), so an open or a read commits nothing -- an
|
|
145
|
+
# observer's data_version moves only when ANOTHER connection commits.
|
|
146
|
+
db = str(tmp_path / "stats.db")
|
|
147
|
+
w = SQLiteGraphCapability()
|
|
148
|
+
w.initialize({"db_path": db})
|
|
149
|
+
indexes = {r[0] for r in sqlite3.connect(db).execute(
|
|
150
|
+
"SELECT name FROM sqlite_master WHERE type = 'index'")}
|
|
151
|
+
assert "idx_nodes_label_id" in indexes and "idx_nodes_label" not in indexes
|
|
152
|
+
w.add_nodes([GraphNode(id=f"n{i:03d}", label="Doc", properties={"i": i}) for i in range(50)])
|
|
153
|
+
stats = {r[0] for r in sqlite3.connect(db).execute(
|
|
154
|
+
"SELECT idx FROM sqlite_stat1 WHERE tbl = 'nodes'")}
|
|
155
|
+
assert "idx_nodes_label_id" in stats
|
|
156
|
+
w.cleanup()
|
|
157
|
+
|
|
158
|
+
observer = sqlite3.connect(db)
|
|
159
|
+
before = observer.execute("PRAGMA data_version").fetchone()[0]
|
|
160
|
+
for readonly in (True, False):
|
|
161
|
+
r = SQLiteGraphCapability()
|
|
162
|
+
r.initialize({"db_path": db, "readonly": readonly})
|
|
163
|
+
assert [n.id for n in r.find_nodes_by_label("Doc", limit=3)] == ["n000", "n001", "n002"]
|
|
164
|
+
assert r.query_nodes(NodeQuery(label="Doc", count=True)).count == 50
|
|
165
|
+
r.cleanup()
|
|
166
|
+
assert observer.execute("PRAGMA data_version").fetchone()[0] == before
|
|
167
|
+
observer.close()
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def test_statistics_follow_growth_on_every_sqlite(tmp_path):
|
|
171
|
+
# The write path's own rule, never PRAGMA optimize's version-specific one (SQLite < 3.46
|
|
172
|
+
# never analyzed a fresh table, so CI kept no statistics): the first write analyzes, a
|
|
173
|
+
# write that leaves the table ~10x its analyzed size re-analyzes, a small one does not.
|
|
174
|
+
db = str(tmp_path / "grow.db")
|
|
175
|
+
cap = SQLiteGraphCapability()
|
|
176
|
+
cap.initialize({"db_path": db})
|
|
177
|
+
|
|
178
|
+
def analyzed_rows():
|
|
179
|
+
stats = sqlite3.connect(db).execute(
|
|
180
|
+
"SELECT stat FROM sqlite_stat1 WHERE tbl = 'nodes' AND idx = 'idx_nodes_label_id'").fetchone()
|
|
181
|
+
return int(stats[0].split()[0])
|
|
182
|
+
|
|
183
|
+
cap.add_nodes([GraphNode(id=f"a{i:03d}", label="Doc", properties={}) for i in range(5)])
|
|
184
|
+
assert analyzed_rows() == 5
|
|
185
|
+
cap.add_nodes([GraphNode(id=f"b{i:03d}", label="Doc", properties={}) for i in range(10)])
|
|
186
|
+
assert analyzed_rows() == 5 # 15 rows: under 10x, the statistics stand
|
|
187
|
+
cap.add_nodes([GraphNode(id=f"c{i:03d}", label="Doc", properties={}) for i in range(40)])
|
|
188
|
+
assert analyzed_rows() == 55 # 55 rows: 10x the analyzed 5, re-analyzed
|
|
189
|
+
cap.cleanup()
|
|
190
|
+
|
|
191
|
+
|
|
143
192
|
def test_typed_surface_smoke(cap):
|
|
144
193
|
doc = str(uuid.uuid4())
|
|
145
194
|
segs = [str(uuid.uuid4()) for _ in range(3)]
|
|
@@ -3,20 +3,19 @@
|
|
|
3
3
|
Projected from the query_translation notebook's behavior-test cell at the
|
|
4
4
|
c25780e8 flip: the translation contract is verified against the PRODUCTION
|
|
5
5
|
schema in an in-memory DB — by what it returns, not by SQL strings."""
|
|
6
|
+
|
|
6
7
|
import json
|
|
7
8
|
import sqlite3
|
|
8
9
|
|
|
9
10
|
import pytest
|
|
10
|
-
|
|
11
|
-
|
|
11
|
+
from cjm_capability_graph_sqlite.query_translation import (decode_projected_row,
|
|
12
|
+
EDGE_STRUCTURAL_KEYS,
|
|
12
13
|
NODE_STRUCTURAL_KEYS,
|
|
13
|
-
decode_projected_row,
|
|
14
14
|
translate_edge_query,
|
|
15
15
|
translate_node_query)
|
|
16
16
|
from cjm_context_graph_primitives.locators import FileRef
|
|
17
|
-
from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy,
|
|
18
|
-
|
|
19
|
-
SourcePredicate)
|
|
17
|
+
from cjm_context_graph_primitives.query import (EdgeQuery, NodeQuery, OrderBy, PropertyPredicate,
|
|
18
|
+
RelationPredicate, SourcePredicate)
|
|
20
19
|
from cjm_substrate.core.errors import CapabilityInputError
|
|
21
20
|
|
|
22
21
|
|
|
@@ -182,6 +181,27 @@ def test_ne_order_desc_limit_and_paging(con):
|
|
|
182
181
|
assert [r[0] for r in rows] == ["s1", "s2"]
|
|
183
182
|
|
|
184
183
|
|
|
184
|
+
def test_keyset_pages_and_deterministic_bounded_order(con):
|
|
185
|
+
# The keyset cursor (finding ae2b196e): a page is the rows after the last id seen, in
|
|
186
|
+
# id order, so a pass over the whole answer visits every row exactly once.
|
|
187
|
+
every = sorted(r["id"] for r in run(con, NodeQuery(label="Segment", project=[])))
|
|
188
|
+
seen, after = [], None
|
|
189
|
+
while True:
|
|
190
|
+
rows = run(con, NodeQuery(label="Segment", project=[], limit=2, after=after))
|
|
191
|
+
seen.extend(r["id"] for r in rows)
|
|
192
|
+
if len(rows) < 2:
|
|
193
|
+
break
|
|
194
|
+
after = rows[-1]["id"]
|
|
195
|
+
assert seen == every
|
|
196
|
+
# A bounded read with no ordering is ordered by id, never an arbitrary subset
|
|
197
|
+
# (finding a107e2f3); a property ordering breaks its ties on the id.
|
|
198
|
+
assert [r[0] for r in run(con, NodeQuery(label="Segment", limit=3))] == every[:3]
|
|
199
|
+
tied = run(con, NodeQuery(label="Segment", where=[PropertyPredicate("index", "eq", 0)],
|
|
200
|
+
order_by=OrderBy(prop="index")))
|
|
201
|
+
assert [r[0] for r in tied] == ["other", "s0"]
|
|
202
|
+
assert [r[0] for r in run(con, EdgeQuery(relation_type="NEXT", after="n0"))] == ["n1", "nx"]
|
|
203
|
+
|
|
204
|
+
|
|
185
205
|
def test_in_op_ids_batch_and_top_level_source(con):
|
|
186
206
|
rows = run(con, NodeQuery(label="Segment", where=[PropertyPredicate("index", "in", [0, 2])],
|
|
187
207
|
related=RelationPredicate("PART_OF", node_id="doc-1")))
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.0.29"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|