prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,625 @@
1
+ """Whether stored OWL RL facts are provably current (docs/reference/owl-rl.md, "Freshness").
2
+
3
+ A run records, beside its cache key, each source table's token as the run read it (`read_run`), and the store's
4
+ own seq at its step. A query of an engine of profile='rl' reads the stored facts only where every source is
5
+ provably unchanged since that run, which only these prove:
6
+
7
+ - **a declared version column** (`version_columns={table: column}`): a column the user's writes keep
8
+ monotonic (an insert or update sets a value above every value before); its maximum and the row count (a
9
+ deleted row lowers it) are compared. Declared, it is used before anything else;
10
+ - **a Delta table version**: a Spark frame that reads Delta tables only (each table's latest version), a Delta
11
+ table on Databricks SQL (DESCRIBE HISTORY). No checksum is taken of a Delta table;
12
+ - **a full content checksum** of a small table: the row count and two sums of a 64-bit hash of every row,
13
+ taken per plan only where the table's size, read from the backend's metadata (`size_of`), is at most the
14
+ backend's threshold (`SMALL`), so that it costs at most about 20 ms.
15
+
16
+ Row counts alone, file times and cached versions prove nothing and are never used; the size read only decides
17
+ whether a checksum is taken.
18
+
19
+ Once per plan, where the dataset home reads the store (`dataset.store_now`), each source's token is read and
20
+ timed (`Check`, which `explain()` shows). Where a source changed (or the store took a write since the run), the
21
+ engine refreshes first (`refresh`: the closure computed again, its difference written). A source no token proves
22
+ (a table past its threshold with no declared column, not Delta; a view), and a refresh that cannot
23
+ write (a read-only store), refuse by name (`StoredFactsNotCurrentError`), naming `version_columns=` and
24
+ `freshness='last_run'`, which reads the stored facts of the last run with no check (`explain()` says when that
25
+ run was, and under which key).
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import json
31
+ from dataclasses import dataclass
32
+ from typing import Any, Dict, List, Optional, Tuple
33
+
34
+ from prograph.rules import module_home
35
+ from prograph.utils.exceptions import StoredFactsNotCurrentError
36
+
37
+ module_home("stored-facts-freshness", "Stored OWL RL facts used only where provably current: each source's "
38
+ "version recorded at the run and compared once per plan",
39
+ terms=("freshness", "last_run", "provably current", "source version", "version column",
40
+ "version_columns", "Delta version", "content checksum", "StoredFactsNotCurrentError",
41
+ "refresh", "stale facts", "stale-by-choice"))
42
+
43
+ # The engine's `freshness=`: None (stored facts used only where provably current) or 'last_run'.
44
+ FRESHNESS = (None, "last_run", "strict")
45
+
46
+
47
+ def engine_freshness(freshness: Optional[str], profile: Optional[str]) -> Optional[str]:
48
+ """`freshness=` checked: None, 'last_run' or 'strict', under profile='rl' (the stored facts) or 'ql' (the
49
+ consistency verdict, `consistency.recheck`). Under 'rl' 'strict' is the default's rule. Anything else
50
+ raises ValueError."""
51
+ if freshness not in FRESHNESS:
52
+ raise ValueError(f"freshness is None (the default), 'last_run' or 'strict', not {freshness!r}")
53
+ if freshness is not None and profile not in ("rl", "ql"):
54
+ raise ValueError(f"freshness={freshness!r} chooses how the ontology's stored facts or consistency verdict "
55
+ f"are trusted: build the engine with profile='rl' or profile='ql' too")
56
+ return None if freshness == "strict" and profile == "rl" else freshness
57
+
58
+
59
+ @dataclass(frozen=True)
60
+ class Source:
61
+ """One logical table the engine's own maps read: `name` as mapped (`Source.table`), `sql` the relation a
62
+ statement reads (a table name or a query in parentheses), the declared version `column` or None, `label`
63
+ the source in the user's words (the table, or a query's map and first line), and for a query the
64
+ (table, column) of each table it reads that `version_columns` declares (`reads`, empty where not all are)."""
65
+ name: str
66
+ sql: str
67
+ column: Optional[str]
68
+ label: str = ""
69
+ reads: Tuple[Tuple[str, str], ...] = ()
70
+
71
+ @property
72
+ def query(self) -> bool:
73
+ return self.sql.startswith("(")
74
+
75
+
76
+ def sources(engine: Any, maps: Any) -> Tuple[Source, ...]:
77
+ """The logical tables `maps` (the engine's own, `dataset.own_maps`) read, with their declared version
78
+ columns (`version_columns`): a mapped table's own, and for an rr:sqlQuery those of the tables it reads (a
79
+ query is proven by them where every table it reads has one). A declared table no map reads, nor any
80
+ query, raises ValueError."""
81
+ from prograph.plan.sql.dialect import get_dialect_instance
82
+ graph = getattr(getattr(engine, "executor", None), "graph", None)
83
+ dialect = graph.sql_dialect() if hasattr(graph, "sql_dialect") else None
84
+ d = get_dialect_instance(dialect or "duckdb")
85
+ declared = dict(getattr(engine, "_version_columns", None) or {})
86
+ maps_of: Dict[str, Any] = {}
87
+ for m in maps.nodes + maps.edges:
88
+ maps_of.setdefault(m.source.table, m)
89
+ out, used = [], set()
90
+ matched: Dict[str, set] = {} # a declared name: the relations of queries it names
91
+ for s in maps.tables():
92
+ if s.query is None:
93
+ used.add(s.table) if s.table in declared else None
94
+ out.append(Source(s.table, d.table_name(s.qualified), declared.get(s.table), s.table))
95
+ continue
96
+ m = maps_of.get(s.table)
97
+ first = s.query.strip().splitlines()[0][:80] if s.query.strip() else ""
98
+ label = f"the rr:sqlQuery of {getattr(m, 'id', s.table)} ({first})"
99
+ reads: Tuple[Tuple[str, str], ...] = ()
100
+ store = _reader(engine) if _kind(engine) == "sql" else None
101
+ tables = _query_tables(store, f"({s.query})") if store is not None and _kind(engine) == "sql" else None
102
+ if tables and declared:
103
+ found = [(t, _declared_for(t, declared, store.dialect)) for t in tables]
104
+ for t, c in found:
105
+ if c is not None:
106
+ matched.setdefault(c[0], set()).add(".".join(p for p, _ in name_parts(t)))
107
+ if all(c is not None for _, c in found):
108
+ reads = tuple((".".join(d.quote(p) for p, _ in name_parts(t)), c[1]) for t, c in found)
109
+ used.update(c[0] for _, c in found)
110
+ out.append(Source(s.table, f"({s.query})", None, label, reads))
111
+ for name, relations in matched.items():
112
+ if len(relations) > 1:
113
+ raise ValueError(f"version_columns names {name!r}, which is each of {sorted(relations)} that the "
114
+ f"rr:sqlQuery maps read: name the one you mean with its schema")
115
+ unused = sorted(set(declared) - used)
116
+ if unused:
117
+ raise ValueError(f"version_columns names {unused}, which no map of the engine reads, nor any rr:sqlQuery; "
118
+ f"the mapped tables are {sorted(x.name for x in out if not x.query)}")
119
+ return tuple(out)
120
+
121
+
122
+ def name_parts(name: str) -> List[Tuple[str, bool]]:
123
+ """A dotted name's parts, each (text, quoted): `"a"."b"`, `` `a`.`b` `` and `a.b` alike."""
124
+ import re
125
+ return [(m.group(1) if m.group(1) is not None else m.group(2) if m.group(2) is not None else m.group(3),
126
+ m.group(3) is None)
127
+ for m in re.finditer(r'"((?:[^"]|"")*)"|`((?:[^`]|``)*)`|([^."`]+)', name)]
128
+
129
+
130
+ def _same_part(declared: Tuple[str, bool], found: Tuple[str, bool], dialect: str) -> bool:
131
+ """Whether a declared name's part names a relation's part: on PostgreSQL an unquoted part folds to lower
132
+ case and a quoted one is exact; elsewhere (DuckDB, Spark, Databricks SQL) case does not matter."""
133
+ if dialect == "postgresql":
134
+ text = declared[0] if declared[1] else declared[0].lower()
135
+ return text == found[0]
136
+ return declared[0].lower() == found[0].lower()
137
+
138
+
139
+ def _declared_for(table: str, declared: Dict[str, str], dialect: str) -> Optional[Tuple[str, str]]:
140
+ """The (declared name, column) of `version_columns` that names the relation `table` (as the database names
141
+ it, qualified, maybe quoted): the declared name's parts are the relation's trailing parts (`t`,
142
+ `schema.t`, `catalog.schema.t`)."""
143
+ found = name_parts(table)
144
+ for name, column in declared.items():
145
+ parts = name_parts(name)
146
+ if 0 < len(parts) <= len(found) and all(_same_part(a, b, dialect)
147
+ for a, b in zip(parts, found[len(found) - len(parts):])):
148
+ return name, column
149
+ return None
150
+
151
+
152
+ def _reader(engine: Any) -> Any:
153
+ """What reads the sources' versions and checksums: the owned store's connection, or with no store (profile
154
+ 'ql') the engine's own connection, wrapped as the store wraps it (`store.store_connection`: no table made)."""
155
+ store = getattr(engine, "_store", None)
156
+ if store is not None:
157
+ return store
158
+ held = engine.__dict__.get("_freshness_reader")
159
+ if held is None:
160
+ from types import SimpleNamespace
161
+ from prograph.store import store_connection
162
+ db = store_connection(engine._connection)
163
+ held = SimpleNamespace(db=db, dialect=db.dialect)
164
+ engine.__dict__["_freshness_reader"] = held
165
+ return held
166
+
167
+
168
+ def _graph(engine: Any) -> Any:
169
+ return getattr(getattr(engine, "executor", None), "graph", None)
170
+
171
+
172
+ def _kind(engine: Any) -> str:
173
+ """How the engine reads its sources: 'sql' (a database connection), 'spark' (Spark frames) or 'frames'
174
+ (pandas frames on DuckDB)."""
175
+ from prograph.graph.sql_graph import SQLGraph
176
+ graph = _graph(engine)
177
+ if isinstance(graph, SQLGraph):
178
+ return "sql"
179
+ if getattr(engine, "backend", None) == "spark":
180
+ return "spark"
181
+ return "frames"
182
+
183
+
184
+ # -- how big a table may be for a checksum per plan ------------------------------------------------------------
185
+
186
+ # The largest table whose full checksum a plan takes, per backend, in the unit its metadata keeps (`size_of`).
187
+ # The target: a checksum per plan costs at most about 20 ms on DuckDB and PostgreSQL; on Spark a checksum is a
188
+ # job, whose floor is about 40 ms at any size, so there the target is that floor. Measured 2026-11-15 on a
189
+ # 16-core workstation: tables of three text columns (`s`, `o`, `label`, about 30 to 45 bytes each, like a mapped
190
+ # table's IRIs and labels), the median of 7 runs of `checksum_sql` / `_spark_checksum` per size:
191
+ # - DuckDB (rows, `duckdb_tables().estimated_size`): 100,000 rows 6.0 ms, 1,000,000 12.5 ms, 2,000,000
192
+ # 17.5 ms, 3,000,000 25.0 ms: 2,000,000 rows.
193
+ # - PostgreSQL (the test container; bytes, `pg_table_size`): 10,000 rows (1.6 MB) 6.2 ms, 30,000 (3.7 MB) 18.4 ms, 100,000
194
+ # (12 MB) 35.7 ms, 300,000 (35 MB) 69.7 ms: 4 MB.
195
+ # - Spark local[2] (bytes, the optimized plan's `sizeInBytes`): 3.4 KB 43 ms, 340 KB 39 ms, 3.4 MB 44 ms:
196
+ # 4 MB. Rows a session holds in memory (`createDataFrame`) have no size estimate (Long.MAX_VALUE): no
197
+ # checksum, so such a frame needs a declared column. A Delta table is proven by its version, never summed.
198
+ # - pandas frames (rows), summed on the engine's own DuckDB, which types each object column from every row
199
+ # (`pandas_graph.frames_connection`) on every statement: 10,000 rows 8.8 ms, 30,000 30.6 ms, 100,000
200
+ # 84.1 ms: 20,000 rows.
201
+ SMALL = {"duckdb": 2_000_000, "postgresql": 4_000_000, "spark": 4_000_000, "frames": 20_000}
202
+ SIZE_UNIT = {"duckdb": "rows", "postgresql": "bytes", "spark": "bytes", "frames": "rows"}
203
+
204
+
205
+ def size_of(engine: Any, s: Source) -> Optional[int]:
206
+ """A table's size from the backend's metadata, read now, cheaply: DuckDB `duckdb_tables().estimated_size`
207
+ (rows), PostgreSQL `pg_table_size` (bytes of a table or materialized view), Spark the optimized plan's
208
+ `sizeInBytes` (an estimate), a pandas frame its rows; a query or a view the sum of the tables it reads
209
+ (`_query_tables`). None where none is kept, so no checksum is taken."""
210
+ kind = _kind(engine)
211
+ if kind == "frames":
212
+ frame = (getattr(_graph(engine), "dataframes", None) or {}).get(s.name)
213
+ return None if frame is None else len(frame)
214
+ if kind == "spark":
215
+ frame = _spark_frame(engine, s)
216
+ try:
217
+ size = frame._jdf.queryExecution().optimizedPlan().stats().sizeInBytes() # a Scala BigInt
218
+ return int(size) if isinstance(size, int) else int(size.toString())
219
+ except Exception: # noqa: BLE001 - no estimate kept
220
+ return None
221
+ store = _reader(engine)
222
+ if store.dialect not in ("duckdb", "postgresql"):
223
+ return None
224
+ if not s.query:
225
+ size = _table_size(store, s.sql)
226
+ if size is not None:
227
+ return size
228
+ # A query, or a view: the sizes of the tables it reads, added, where each is known.
229
+ tables = _query_tables(store, s.sql if s.query else f"(SELECT * FROM {s.sql})")
230
+ if not tables:
231
+ return None
232
+ sizes = [_table_size(store, t) for t in tables]
233
+ return None if any(x is None for x in sizes) else sum(sizes)
234
+
235
+
236
+ def _table_size(store: Any, name: str) -> Optional[int]:
237
+ from prograph.plan.sql.dialect import get_dialect_instance
238
+ literal = get_dialect_instance(store.dialect).literal
239
+ if store.dialect == "duckdb":
240
+ schema, _, table = name.replace('"', "").rpartition(".")
241
+ where = f"table_name = {literal(table)}" + (f" AND schema_name = {literal(schema.split('.')[-1])}" if schema else "")
242
+ row = store.db.execute(f"SELECT max(estimated_size) FROM duckdb_tables() WHERE {where}").fetchone()
243
+ else:
244
+ row = store.db.execute(f"SELECT CASE WHEN c.relkind IN ('r', 'm') THEN pg_table_size(c.oid) END FROM pg_class "
245
+ f"AS c WHERE c.oid = to_regclass({literal(name)})").fetchone()
246
+ return None if row is None or row[0] is None else int(row[0])
247
+
248
+
249
+ def _query_tables(store: Any, sql: str) -> Optional[List[str]]:
250
+ """The base tables a query (`sql` in parentheses) reads, as the database names them, views read through:
251
+ DuckDB's parser (`duckdb.get_table_names`, a view by its own text in `duckdb_views()`), PostgreSQL's plan
252
+ (`StoreConnection.plan_json`: each scan's schema and relation; no row is read), Databricks SQL's analyzed
253
+ plan (`analyzed_relations` of `EXPLAIN EXTENDED`). None where they cannot be told."""
254
+ if store is None:
255
+ return None
256
+ try:
257
+ if store.dialect == "duckdb":
258
+ return _duckdb_tables(store, f"SELECT * FROM {sql} AS _q", 0)
259
+ if store.dialect in ("databricks_sql", "databricks"):
260
+ rows = store.db.execute(f"EXPLAIN EXTENDED SELECT * FROM {sql} AS _q").fetchall()
261
+ return analyzed_relations(str(rows[0][0])) if rows else None
262
+ if store.dialect != "postgresql":
263
+ return None
264
+ plan = store.db.plan_json(f"SELECT * FROM {sql} AS _q")
265
+ except Exception: # noqa: BLE001 - a query not told apart has no size
266
+ return None
267
+ if plan is None:
268
+ return None
269
+ plan = json.loads(plan) if isinstance(plan, str) else plan
270
+ found, stack = set(), list(plan)
271
+ while stack:
272
+ node = stack.pop()
273
+ node = node.get("Plan", node)
274
+ if node.get("Node Type") in ("Function Scan", "ProjectSet", "Table Function Scan"):
275
+ return None # rows a function makes: no table's size counts them
276
+ if "Relation Name" in node:
277
+ found.add(f'"{node.get("Schema", "public")}"."{node["Relation Name"]}"')
278
+ stack.extend(node.get("Plans", ()))
279
+ return sorted(found) or None
280
+
281
+
282
+ def _duckdb_tables(store: Any, sql: str, depth: int) -> Optional[List[str]]:
283
+ import duckdb
284
+ from prograph.plan.sql.dialect import get_dialect_instance
285
+ literal = get_dialect_instance("duckdb").literal
286
+ out: set = set()
287
+ serialized = store.db.execute(f"SELECT json_serialize_sql({literal(sql)})").fetchone()[0]
288
+ if '"TABLE_FUNCTION"' in serialized or any(f'"function_name":"{f}"' in serialized
289
+ for f in ("unnest", "generate_series", "range")):
290
+ return None # rows a function makes: no table's size counts them
291
+ for name in duckdb.get_table_names(sql, qualified=True):
292
+ parts = name.replace('"', "").split(".")
293
+ bare, schema = parts[-1], parts[-2] if len(parts) > 1 else None
294
+ where = f"view_name = {literal(bare)}" + (f" AND schema_name = {literal(schema)}" if schema else
295
+ " AND schema_name = current_schema()")
296
+ row = store.db.execute(f"SELECT sql FROM duckdb_views() WHERE {where} AND NOT internal").fetchone()
297
+ if row is None:
298
+ out.add(name)
299
+ continue
300
+ if depth >= 8:
301
+ return None
302
+ text = (row[0].split(" AS ", 1)[1] if " AS " in row[0] else row[0]).strip().rstrip(";")
303
+ inner = _duckdb_tables(store, text, depth + 1)
304
+ if inner is None:
305
+ return None
306
+ out.update(inner)
307
+ return sorted(out)
308
+
309
+
310
+ def small(engine: Any, s: Source) -> bool:
311
+ """Whether a plan takes `s`'s checksum: its size is known and at most its backend's threshold."""
312
+ kind = _kind(engine)
313
+ backend = kind if kind in ("spark", "frames") else _reader(engine).dialect
314
+ limit = SMALL.get(backend)
315
+ if limit is None:
316
+ return False
317
+ size = size_of(engine, s)
318
+ return size is not None and size <= limit
319
+
320
+
321
+ # -- the SQL of a token (DuckDB, PostgreSQL, Databricks SQL) ---------------------------------------------------
322
+
323
+ def _text(d: Any, expr: str) -> str:
324
+ return f"CAST({expr} AS {d.type_name('STRING')})"
325
+
326
+
327
+ def column_sql(d: Any, source: Source) -> str:
328
+ """One row, one text: the declared column's maximum and the row count (`column:<max>/<count>`)."""
329
+ col = d.quote(source.column)
330
+ top = f"CASE WHEN max(_t.{col}) IS NULL THEN '' ELSE {_text(d, f'max(_t.{col})')} END"
331
+ return f"SELECT 'column:' || {top} || '/' || {_text(d, 'count(*)')} AS a FROM {source.sql} AS _t"
332
+
333
+
334
+ def checksum_sql(d: Any, dialect: str, source: Source) -> str:
335
+ """One row, one text: the row count and two sums of a 64-bit hash of each whole row (order-free, so a
336
+ table's row order does not matter; a changed value changes its row's hash)."""
337
+ if dialect == "postgresql":
338
+ one, two = "hashtextextended(_t::text, 0)", "hashtextextended(_t::text, 1)"
339
+ elif dialect == "duckdb":
340
+ one, two = "hash(_t)", "hash(_t, 1)"
341
+ else: # Spark SQL, Databricks SQL
342
+ one, two = "xxhash64(*)", "hash(*)"
343
+ num = "DECIMAL(38, 0)"
344
+ return (f"SELECT 'checksum:' || {_text(d, 'count(*)')} || ':' || "
345
+ f"{_text(d, f'coalesce(sum(CAST({one} AS {num})), 0)')} || ':' || "
346
+ f"{_text(d, f'coalesce(sum(CAST({two} AS {num})), 0)')} AS a FROM {source.sql} AS _t")
347
+
348
+
349
+ # -- reading the tokens ---------------------------------------------------------------------------------------
350
+
351
+ @dataclass(frozen=True)
352
+ class Check:
353
+ """How one source was read for a plan: `how` ('declared version column', 'Delta version', 'checksum', or
354
+ why none), and the milliseconds it took."""
355
+ how: str
356
+ ms: float
357
+
358
+
359
+ def read_now(engine: Any, srcs: Tuple[Source, ...], checks: Optional[Dict[str, Check]] = None) \
360
+ -> Dict[str, Optional[str]]:
361
+ """Each source's token now: by its declared column, else its Delta version, else, for a small table
362
+ (`small`), its checksum; None for a source none of them proves. `checks` gets how each was read, timed."""
363
+ import time
364
+ out: Dict[str, Optional[str]] = {}
365
+ for s in srcs:
366
+ started = time.perf_counter()
367
+ token = _proof(engine, s)
368
+ if token is None and small(engine, s):
369
+ token = _checksum(engine, s)
370
+ out[s.name] = token
371
+ if checks is not None:
372
+ how = proof_name(token) if token is not None else "no version, too big for a checksum per query"
373
+ checks[s.name] = Check(how, (time.perf_counter() - started) * 1000)
374
+ return out
375
+
376
+
377
+ def read_run(engine: Any, srcs: Tuple[Source, ...]) -> Dict[str, List[str]]:
378
+ """What a run records of each source: its token (`read_now`), and for a small table its checksum too, so
379
+ an engine that declares no column over the same store compares as well. A Delta table: its version alone."""
380
+ now = read_now(engine, srcs)
381
+ out: Dict[str, List[str]] = {}
382
+ for s in srcs:
383
+ held = [now[s.name]] if now[s.name] is not None else []
384
+ if (s.column is not None or s.reads) and small(engine, s):
385
+ held.append(_checksum(engine, s))
386
+ out[s.name] = held
387
+ return out
388
+
389
+
390
+ def _proof(engine: Any, s: Source) -> Optional[str]:
391
+ """A declared column's token, else a Delta version; None for neither."""
392
+ kind = _kind(engine)
393
+ if kind == "spark":
394
+ from pyspark.sql import functions as F
395
+ from prograph.plan.sql.dialect import get_dialect_instance
396
+ frame = _spark_frame(engine, s)
397
+ if s.column is not None:
398
+ col = F.col(get_dialect_instance("spark").quote(s.column))
399
+ row = frame.agg(F.max(col).alias("m"), F.count(F.lit(1)).alias("n")).first()
400
+ return f"column:{'' if row['m'] is None else row['m']}/{row['n']}"
401
+ return _spark_delta(frame)
402
+ if kind == "sql":
403
+ from prograph.plan.sql.dialect import get_dialect_instance
404
+ dialect = _graph(engine).sql_dialect()
405
+ d = get_dialect_instance(dialect)
406
+ if s.column is not None:
407
+ return _reader(engine).db.execute(column_sql(d, s)).fetchone()[0]
408
+ if s.reads: # a query: the declared columns of the tables it reads
409
+ return "column:" + ";".join(
410
+ _reader(engine).db.execute(column_sql(d, Source(t, t, c))).fetchone()[0][len("column:"):]
411
+ for t, c in s.reads)
412
+ if dialect in ("databricks_sql", "databricks"):
413
+ return _databricks_query_version(engine, s) if s.query else _databricks_version(engine, s)
414
+ return None
415
+ frame = (getattr(_graph(engine), "dataframes", None) or {}).get(s.name)
416
+ if frame is None or s.column is None:
417
+ return None
418
+ top = frame[s.column].max() if len(frame) else None
419
+ return f"column:{'' if top is None else top}/{len(frame)}"
420
+
421
+
422
+ def _checksum(engine: Any, s: Source) -> Optional[str]:
423
+ kind = _kind(engine)
424
+ if kind == "spark":
425
+ return _spark_checksum(_spark_frame(engine, s))
426
+ if kind == "sql":
427
+ from prograph.plan.sql.dialect import get_dialect_instance
428
+ dialect = _graph(engine).sql_dialect()
429
+ return _reader(engine).db.execute(checksum_sql(get_dialect_instance(dialect), dialect, s)).fetchone()[0]
430
+ # A pandas frame: its view on the engine's own DuckDB, summed there as a DuckDB table is.
431
+ from prograph.plan.sql.dialect import get_dialect_instance
432
+ graph = _graph(engine)
433
+ if (getattr(graph, "dataframes", None) or {}).get(s.name) is None or graph.sql_dialect() != "duckdb":
434
+ return None
435
+ d = get_dialect_instance("duckdb")
436
+ quoted = ".".join(d.quote(p) for p in s.name.split(".")) # its view, named as `register_frame` names it
437
+ return graph._run_local_sql(checksum_sql(d, "duckdb", Source(s.name, quoted, None))).fetchone()[0]
438
+
439
+
440
+ def _databricks_version(engine: Any, s: Source) -> Optional[str]:
441
+ """A Delta table's latest version on Databricks SQL, read now (never a cached one); None where the table
442
+ has no Delta history."""
443
+ try:
444
+ rows = _graph(engine)._execute_sql(f"DESCRIBE HISTORY {s.sql} LIMIT 1")
445
+ except Exception: # noqa: BLE001 - no history: not a Delta table
446
+ return None
447
+ if not rows or rows[0].get("version") is None:
448
+ return None
449
+ return f"delta:{s.sql}@{rows[0]['version']}"
450
+
451
+
452
+ def _databricks_query_version(engine: Any, s: Source) -> Optional[str]:
453
+ """An rr:sqlQuery on Databricks SQL: the Delta versions of the tables it reads, each read now. The tables
454
+ are its analyzed plan's leaves (`EXPLAIN EXTENDED`, which reads no row), views read through; None where a
455
+ leaf is not a table's `Relation` (rows of the query's own, a range, a function) or a table has no Delta
456
+ history: not proven."""
457
+ try:
458
+ rows = _graph(engine)._execute_sql(f"EXPLAIN EXTENDED SELECT * FROM {s.sql} AS _q")
459
+ except Exception: # noqa: BLE001 - no plan: not proven
460
+ return None
461
+ text = str(next(iter(rows[0].values()))) if rows else ""
462
+ tables = analyzed_relations(text)
463
+ if not tables:
464
+ return None
465
+ from prograph.plan.sql.dialect import get_dialect_instance
466
+ quote = get_dialect_instance("spark").quote
467
+ found = []
468
+ for name in tables:
469
+ token = _databricks_version(engine, Source(name, ".".join(quote(p) for p in name.split(".")), None))
470
+ if token is None:
471
+ return None
472
+ found.append(token[len("delta:"):])
473
+ return "delta:" + ";".join(sorted(found))
474
+
475
+
476
+ def analyzed_relations(plan: str) -> Optional[List[str]]:
477
+ """The tables a Spark SQL plan's text reads (its '== Analyzed Logical Plan ==' section): the name of each
478
+ leaf `Relation <name>[...]`; None where a leaf is anything else, or there is no such section."""
479
+ import re
480
+ if "== Analyzed Logical Plan ==" not in plan:
481
+ return None
482
+ section = plan.split("== Analyzed Logical Plan ==", 1)[1].split("\n== ", 1)[0]
483
+ lines = [x for x in section.splitlines() if x.strip()][1:] # the first line is the output's schema
484
+ nodes = []
485
+ for line in lines:
486
+ found = re.search(r"[A-Za-z'`]", line)
487
+ if found is not None:
488
+ nodes.append((found.start(), line[found.start():]))
489
+ out = set()
490
+ for i, (depth, text) in enumerate(nodes):
491
+ leaf = i + 1 == len(nodes) or nodes[i + 1][0] <= depth
492
+ if not leaf:
493
+ continue
494
+ match = re.match(r"Relation ([^\[\s]+)\[", text)
495
+ if match is None:
496
+ return None
497
+ out.add(match.group(1).replace("`", ""))
498
+ return sorted(out) or None
499
+
500
+
501
+ def _spark_frame(engine: Any, s: Source) -> Any:
502
+ frame = _graph(engine).dataframes.get(s.name)
503
+ if frame is None:
504
+ frame = engine._spark.sql(f"SELECT * FROM {s.sql}") if s.sql.startswith("(") else engine._spark.table(s.sql)
505
+ return frame
506
+
507
+
508
+ def _spark_checksum(frame: Any) -> str:
509
+ from pyspark.sql import functions as F
510
+ from prograph.plan.sql.dialect import get_dialect_instance
511
+ quote = get_dialect_instance("spark").quote
512
+ cols = [F.col(quote(c)) for c in frame.columns]
513
+ num = "decimal(38,0)"
514
+ row = frame.agg(F.count(F.lit(1)).alias("n"),
515
+ F.coalesce(F.sum(F.xxhash64(*cols).cast(num)), F.lit(0).cast(num)).alias("a"),
516
+ F.coalesce(F.sum(F.hash(*cols).cast(num)), F.lit(0).cast(num)).alias("b")).first()
517
+ return f"checksum:{row['n']}:{row['a']}:{row['b']}"
518
+
519
+
520
+ def _spark_delta(frame: Any) -> Optional[str]:
521
+ """The Delta versions of the tables a Spark frame reads, each read now (DESCRIBE HISTORY); None where a
522
+ leaf of its plan is not a Delta table (rows held in memory included: their frame is not named in a run's
523
+ record, so nothing proves it the same)."""
524
+ from prograph.plan.ir.key_checks import _delta_name, _spark_leaves
525
+ try:
526
+ leaves = _spark_leaves(frame)
527
+ except Exception: # noqa: BLE001 - a plan that cannot be read proves nothing
528
+ return None
529
+ if not leaves:
530
+ return None
531
+ found = []
532
+ for leaf in leaves:
533
+ name = _delta_name(leaf)
534
+ if name is None:
535
+ return None
536
+ row = frame.sparkSession.sql(f"DESCRIBE HISTORY {name} LIMIT 1").select("version").first()
537
+ if row is None:
538
+ return None
539
+ found.append(f"{name}@{row[0]}")
540
+ return "delta:" + ";".join(sorted(found))
541
+
542
+
543
+ # -- the record, and the comparison --------------------------------------------------------------------------
544
+
545
+ def recorded(text: Optional[str]) -> Dict[str, List[str]]:
546
+ """The source tokens a run's record holds (`owl_rl.RUN_SOURCES`), {} for none."""
547
+ if not text:
548
+ return {}
549
+ try:
550
+ held = json.loads(text)
551
+ except ValueError:
552
+ return {}
553
+ if not isinstance(held, dict):
554
+ return {}
555
+ return {k: (v if isinstance(v, list) else [v]) for k, v in held.items()}
556
+
557
+
558
+ def proof_name(token: Optional[str]) -> str:
559
+ """What proved a token's source unchanged, in words."""
560
+ kind = (token or "").split(":", 1)[0]
561
+ return {"column": "declared version column", "delta": "Delta version", "checksum": "checksum"}.get(kind, kind)
562
+
563
+
564
+ def as_record(tokens: Dict[str, List[str]]) -> str:
565
+ return json.dumps(tokens, sort_keys=True)
566
+
567
+
568
+ @dataclass(frozen=True)
569
+ class Verdict:
570
+ """A comparison of the sources now with a run's record: `changed` the sources (or 'the store') not as the
571
+ run read them, `unproven` those no token proves per plan (`read_now`)."""
572
+ changed: Tuple[str, ...] = ()
573
+ unproven: Tuple[str, ...] = ()
574
+
575
+ @property
576
+ def current(self) -> bool:
577
+ return not self.changed and not self.unproven
578
+
579
+
580
+ def compare(record: Dict[str, List[str]], now: Dict[str, Optional[str]], store_moved: bool) -> Verdict:
581
+ changed = [n for n, t in now.items() if t is not None and t not in record.get(n, ())]
582
+ if store_moved:
583
+ changed.append("the owned store")
584
+ unproven = [n for n, t in now.items() if t is None]
585
+ return Verdict(tuple(changed), tuple(unproven))
586
+
587
+
588
+ def refuse_unproven(srcs: Tuple[Source, ...]) -> None:
589
+ raise StoredFactsNotCurrentError(unproven_reason(srcs))
590
+
591
+
592
+ def unproven_reason(srcs: Tuple[Source, ...]) -> str:
593
+ """Why the stored facts cannot be proven current, naming each source as the user wrote it, and the ways out."""
594
+ listed = ", ".join(s.label or s.name for s in srcs)
595
+ one = len(srcs) == 1
596
+ tables = next((s for s in srcs if not s.query), None)
597
+ query = next((s for s in srcs if s.query), None)
598
+ ways = []
599
+ if tables is not None:
600
+ ways.append(f"declare a column your writes keep increasing (ProGraph(..., version_columns="
601
+ f"{{'{tables.name}': '<column>'}}))")
602
+ if query is not None:
603
+ ways.append("for an rr:sqlQuery, declare one on every table it reads (version_columns= takes those "
604
+ "tables' names)")
605
+ ways.append("or read the facts of the last run with freshness='last_run' (call materialize_entailments() "
606
+ "after your tables change)")
607
+ return (f"the stored OWL RL facts cannot be proven current for this query: {listed} {'has' if one else 'have'} "
608
+ f"no version a query can read (no declared version column, not a Delta table) and "
609
+ f"{'is' if one else 'are'} too big, or of unknown size, for a checksum per query. To fix: "
610
+ + "; ".join(ways))
611
+
612
+
613
+ def refresh(engine: Any, changed: Tuple[str, ...]) -> None:
614
+ """The stored facts made current: the closure computed again and its difference written
615
+ (`materialize_entailments`). A store that cannot be written refuses by name."""
616
+ from prograph.utils.exceptions import StoreReadOnlyError
617
+ try:
618
+ if engine._store.db.read_only():
619
+ raise StoreReadOnlyError("the connection is read-only")
620
+ engine._materialize_run()
621
+ except StoreReadOnlyError as exc:
622
+ raise StoredFactsNotCurrentError(
623
+ f"the stored OWL RL facts are not current ({', '.join(changed)} changed since the last run) and the "
624
+ f"owned store cannot be written to refresh them ({exc}). Run materialize_entailments() on a writable "
625
+ f"connection, or read the facts of the last run with freshness='last_run'") from exc