prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,39 @@
1
+ """The order a front end writes a group's joins in (the Cypher half follows the same rule).
2
+
3
+ The lowering fuses inner joins into one SELECT so the database orders them (`lower.region`), but under a
4
+ condition across the joins it keeps the written order (`lower._across`: DuckDB takes such a condition for a
5
+ join edge and may join on it first). So the written order must already be a good one: connected, each leaf
6
+ joined where a condition ties it to what is joined, a cross join only where the group is disconnected, and a
7
+ part started from its most selective leaf.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from typing import FrozenSet, List, Sequence, Set, Tuple
13
+
14
+ from prograph.rules import home
15
+
16
+
17
+ @home("join-order", "A group's joins written in connected order, from its most selective leaf",
18
+ terms=("join order", "connected order", "cross join", "pattern order", "selective leaf", "seed",
19
+ "most selective"))
20
+ def connected_order(leaves: Sequence[Tuple[str, FrozenSet[str], int]], have: Set[str],
21
+ conditions: Sequence[FrozenSet[str]]) -> List[str]:
22
+ """The names of `leaves` (name, the columns it gives, how selective it is: higher first) in the order to
23
+ join them after the columns `have`: next, a leaf a condition ties to what is joined (a condition reading
24
+ its columns and some joined ones, all present once it is), the most selective of those, then the first
25
+ written; with none, a new part from the most selective leaf left. Each condition is the column set it
26
+ reads."""
27
+ have = set(have)
28
+ left = list(leaves)
29
+ out: List[str] = []
30
+ while left:
31
+ def tied(leaf: Tuple[str, FrozenSet[str], int]) -> bool:
32
+ cols = leaf[1]
33
+ return any(c & have and c & cols and c <= have | cols for c in conditions)
34
+ candidates = [x for x in left if tied(x)] or left
35
+ best = max(candidates, key=lambda x: x[2]) # max keeps the first written of equals
36
+ left.remove(best)
37
+ have |= best[1]
38
+ out.append(best[0])
39
+ return out
@@ -0,0 +1,433 @@
1
+ """Reading a mapping's rows at a version of the data, and checking its key there (docs/architecture/keys-and-identity.md): a source's version (a pandas frame by object, a Spark
2
+ frame of fixed rows or a Delta table, a Databricks Delta table by its history), one key value two
3
+ of a map's rows share, read once per version (`KeyChecks`), and how a statement reads the version
4
+ checked (`PinnedRead`, `pinned_name`). `keys.checked_key` asks it; it moved here
5
+ from `prograph.node_keys`, whose Cypher policy stays there."""
6
+
7
+ from __future__ import annotations
8
+
9
+ import contextvars
10
+ import time
11
+ import weakref
12
+ from typing import Any, Dict, Optional, Tuple
13
+
14
+ from prograph.plan.ir.keys import rely_primary_key # noqa: F401 (the home)
15
+
16
+
17
+ # -- reading what was checked ----------------------------------------------
18
+
19
+ class PinnedRead:
20
+ """How a statement reads a checked table at the checked version: the
21
+ text that stands for the table in its SQL (``t VERSION AS OF n``), and
22
+ on Spark the frame that stands for it on the DataFrame path."""
23
+
24
+ def __init__(self, sql: str, frame: Any = None):
25
+ self.sql = sql
26
+ self.frame = frame
27
+
28
+
29
+ class Pins(dict):
30
+ """A statement's pinned reads, by table name."""
31
+
32
+
33
+ _PINNED: contextvars.ContextVar = contextvars.ContextVar("prograph_pinned_reads", default=None)
34
+
35
+
36
+ def pinned_name(table: str, written: str) -> str:
37
+ """A scan's table as the statement reads it: `written` (the table's name
38
+ as the dialect writes it), or the pinned read of a checked table."""
39
+ pins = _PINNED.get()
40
+ if pins:
41
+ pin = pins.get(table)
42
+ if pin is not None:
43
+ return pin.sql
44
+ return written
45
+
46
+
47
+ # -- the sources: a mapping's version, and a key value two rows share ------
48
+
49
+ def _example(values) -> Tuple[Any, ...]:
50
+ out = []
51
+ for v in values:
52
+ item = getattr(v, "item", None)
53
+ out.append(item() if callable(item) and type(v).__module__ == "numpy" else v)
54
+ return tuple(out)
55
+
56
+
57
+ class FrameSource:
58
+ """Pandas frames (``backend='duckdb'`` over frames).
59
+ A frame's version is the frame object and its arrays: a write that
60
+ replaces either is a new version."""
61
+
62
+ def __init__(self, graph: Any):
63
+ self.graph = graph
64
+
65
+ def _frame(self, mapping):
66
+ return self.graph.dataframes.get(mapping.source_table)
67
+
68
+ def version(self, mapping):
69
+ from prograph.graph.pandas_graph import _frame_state
70
+ frame = self._frame(mapping)
71
+ if frame is None:
72
+ return None
73
+ state = _frame_state(frame)
74
+ if state[2] is None:
75
+ return None
76
+ return (weakref.ref(frame),) + state
77
+
78
+ @staticmethod
79
+ def same(held, now) -> bool:
80
+ from prograph.graph.pandas_graph import _same_state
81
+ return held[0]() is not None and held[0]() is now[0]() and _same_state(held[1:], now[1:])
82
+
83
+ def describe(self, mapping):
84
+ return None
85
+
86
+ def pin(self, mapping, version):
87
+ return None
88
+
89
+ def duplicate(self, mapping, version=None):
90
+ from prograph.schema.rows import owned_rows
91
+ from prograph.schema.types import missing_mask
92
+ frame = owned_rows(self._frame(mapping), mapping)
93
+ if frame is None or len(frame) == 0:
94
+ return None
95
+ cols = list(mapping.id_columns)
96
+ if any(c not in frame.columns for c in cols):
97
+ return None
98
+ keys = frame[cols]
99
+ present = ~missing_mask(keys).any(axis=1) if len(cols) > 1 else ~missing_mask(keys[cols[0]])
100
+ keys = keys[present]
101
+ repeated = keys.duplicated()
102
+ if not repeated.any():
103
+ return None
104
+ return _example(keys[repeated].iloc[0].tolist())
105
+
106
+
107
+ def _spark_leaves(frame) -> Optional[list]:
108
+ jdf = getattr(frame, "_jdf", None)
109
+ if jdf is None:
110
+ return None
111
+ leaves, stack = [], [jdf.queryExecution().analyzed()]
112
+ while stack:
113
+ node = stack.pop()
114
+ children = node.children()
115
+ if children.isEmpty():
116
+ leaves.append(node)
117
+ continue
118
+ it = children.iterator()
119
+ while it.hasNext():
120
+ stack.append(it.next())
121
+ return leaves
122
+
123
+
124
+ # Leaves of a Spark plan whose rows are fixed with the frame: a frame over
125
+ # only these is one version for as long as it is the same object.
126
+ _FIXED_SPARK_LEAVES = ("LocalRelation", "LogicalRDD", "Range", "OneRowRelation")
127
+
128
+
129
+ class SparkFrameSource:
130
+ """Spark DataFrames. A frame over Delta tables is versioned by each
131
+ table's version (DESCRIBE HISTORY); a frame over rows held in memory by
132
+ the frame itself; one over anything else (a parquet table, a view of
133
+ another kind) has no version and is checked on each statement. A frame
134
+ that reads one Delta table as it is (`_plain_delta`) is checked and read
135
+ at the version looked up; a frame derived from Delta tables (a filter, a
136
+ join) cannot be pinned, since its plan is the caller's, and is checked at
137
+ the version looked up and read at the table's latest."""
138
+
139
+ def __init__(self, graph: Any):
140
+ self.graph = graph
141
+
142
+ def _frame(self, mapping):
143
+ return self.graph.dataframes.get(mapping.source_table)
144
+
145
+ def version(self, mapping):
146
+ frame = self._frame(mapping)
147
+ if frame is None:
148
+ return None
149
+ try:
150
+ leaves = _spark_leaves(frame)
151
+ except Exception: # noqa: BLE001 - a plan that cannot be read has no version
152
+ return None
153
+ if leaves is None:
154
+ return None
155
+ tables = []
156
+ for leaf in leaves:
157
+ kind = leaf.getClass().getSimpleName()
158
+ if kind in _FIXED_SPARK_LEAVES:
159
+ continue
160
+ name = _delta_name(leaf)
161
+ if name is None:
162
+ return None
163
+ tables.append((name, self._delta_version(frame, name)))
164
+ return (weakref.ref(frame), tuple(tables))
165
+
166
+ @staticmethod
167
+ def _delta_version(frame, name):
168
+ row = frame.sparkSession.sql(f"DESCRIBE HISTORY {name} LIMIT 1").select("version").first()
169
+ return None if row is None else row[0]
170
+
171
+ @staticmethod
172
+ def same(held, now) -> bool:
173
+ return held[0]() is not None and held[0]() is now[0]() and held[1] == now[1]
174
+
175
+ def describe(self, mapping):
176
+ return None
177
+
178
+ def _pinned(self, mapping, version):
179
+ """The Delta table `mapping`'s frame reads as it is, and the version
180
+ to read it at; None for a frame that is not one table read as it
181
+ is, or no version."""
182
+ if version is None or len(version[1]) != 1 or version[1][0][1] is None:
183
+ return None
184
+ frame = self._frame(mapping)
185
+ name = _plain_delta(frame)
186
+ if name is None or name != version[1][0][0]:
187
+ return None
188
+ return name, int(version[1][0][1])
189
+
190
+ def pin(self, mapping, version):
191
+ pinned = self._pinned(mapping, version)
192
+ if pinned is None:
193
+ return None
194
+ name, number = pinned
195
+ frame = self._frame(mapping)
196
+ return PinnedRead(f"{name} VERSION AS OF {number}",
197
+ frame.sparkSession.sql(f"SELECT * FROM {name} VERSION AS OF {number}"))
198
+
199
+ def duplicate(self, mapping, version=None):
200
+ from pyspark.sql import functions as F
201
+ from prograph.plan.sql.dialect import get_dialect_instance
202
+ from prograph.schema.rows import owned_rows
203
+ pinned = self.pin(mapping, version)
204
+ frame = owned_rows(pinned.frame if pinned is not None else self._frame(mapping), mapping)
205
+ if frame is None:
206
+ return None
207
+ quote = get_dialect_instance("spark").quote
208
+ cols = [F.col(quote(c)) for c in mapping.id_columns]
209
+ present = cols[0].isNotNull()
210
+ for c in cols[1:]:
211
+ present = present & c.isNotNull()
212
+ row = (frame.where(present).groupBy(*cols).count()
213
+ .where(F.col("count") > 1).limit(1).collect())
214
+ return tuple(row[0][:len(cols)]) if row else None
215
+
216
+
217
+ def _delta_name(leaf) -> Optional[str]:
218
+ """The name a Spark plan's leaf reads a Delta table by (its catalog
219
+ name, quoted, or ``delta.`path` ``), or None for a leaf of another
220
+ kind."""
221
+ if leaf.getClass().getSimpleName() != "LogicalRelation":
222
+ return None
223
+ relation = leaf.relation()
224
+ location = relation.location() if hasattr(relation, "location") else None
225
+ if location is None or "Tahoe" not in location.getClass().getName():
226
+ return None
227
+ table = leaf.catalogTable()
228
+ if table.isDefined():
229
+ return table.get().identifier().quotedString()
230
+ from prograph.plan.sql.dialect import get_dialect_instance
231
+ return "delta." + get_dialect_instance("spark").quote(location.rootPaths().head().toString())
232
+
233
+
234
+ def _plain_delta(frame) -> Optional[str]:
235
+ """The name of the Delta table `frame` reads as it is, every row and
236
+ column (``spark.table(name)``, or a ``select('*')`` of it, read directly
237
+ or through a temp view of one, as ``spark.table(name)`` is once an engine
238
+ has made a temp view under the table's name), or None."""
239
+ jdf = getattr(frame, "_jdf", None)
240
+ if jdf is None:
241
+ return None
242
+ try:
243
+ root = node = jdf.queryExecution().analyzed()
244
+ while node.getClass().getSimpleName() in ("SubqueryAlias", "Project", "View"):
245
+ if node.getClass().getSimpleName() == "Project" and \
246
+ node.output().toString() != node.child().output().toString():
247
+ return None
248
+ node = node.child()
249
+ if root.output().toString() != node.output().toString():
250
+ return None
251
+ return _delta_name(node)
252
+ except Exception: # noqa: BLE001 - a plan that cannot be read is not pinned
253
+ return None
254
+
255
+
256
+ class SQLSource:
257
+ """A database through a connection: a DuckDB connection, a Databricks SQL
258
+ warehouse, and the other SQL databases. A Databricks table is versioned
259
+ by its Delta version and trusted by a RELY primary key on its key; any
260
+ other source has no version and is checked on each statement."""
261
+
262
+ def __init__(self, graph: Any):
263
+ from prograph.plan.sql.dialect import get_dialect_instance
264
+ self.graph = graph
265
+ self.dialect_name = graph.sql_dialect()
266
+ self.dialect = get_dialect_instance(self.dialect_name)
267
+
268
+ @property
269
+ def _databricks(self) -> bool:
270
+ return self.dialect_name in ("databricks_sql", "databricks")
271
+
272
+ def _table(self, mapping) -> str:
273
+ return self.dialect.table_name(mapping.qualified_source_table)
274
+
275
+ def describe(self, mapping) -> Optional[Dict[str, Any]]:
276
+ """On Databricks SQL, what a mapping's source is, read once
277
+ (DESCRIBE TABLE EXTENDED): whether it has a Delta history to pin
278
+ (``history``), why not (``what``), and whether its key is a RELY
279
+ primary key (``rely``). None on another database."""
280
+ if not self._databricks:
281
+ return None
282
+ if getattr(mapping, "source_query", None):
283
+ return {"history": False, "what": "a SQL query", "rely": False}
284
+ try:
285
+ rows = self.graph._execute_sql(f"DESCRIBE TABLE EXTENDED {self._table(mapping)}")
286
+ except Exception: # noqa: BLE001 - no description, no history to pin
287
+ return {"history": False, "what": "a source that cannot be described", "rely": False}
288
+ fields = {str(r.get("col_name") or "").strip().lower(): str(r.get("data_type") or "").strip()
289
+ for r in rows}
290
+ kind, provider = fields.get("type", "").upper(), fields.get("provider", "").lower()
291
+ rely = any(rely_primary_key(" ".join(str(v) for v in row.values() if v is not None),
292
+ mapping.id_columns)
293
+ for row in rows)
294
+ if kind not in ("MANAGED", "EXTERNAL") or provider != "delta":
295
+ what = f"a {kind.lower().replace('_', ' ')}" if kind not in ("MANAGED", "EXTERNAL", "") \
296
+ else f"a {provider or 'non-Delta'} table"
297
+ return {"history": False, "what": what, "rely": rely}
298
+ return {"history": True, "what": "a Delta table", "rely": rely}
299
+
300
+ # How long a table's version is taken as current without asking again: a DESCRIBE HISTORY is a
301
+ # round trip per statement on a warehouse. A write within the window is read at the version checked
302
+ # (the statement pins it), so answers stay consistent; the newer rows show after the window.
303
+ VERSION_TTL_S = 60.0
304
+
305
+ def version(self, mapping):
306
+ if not self._databricks or getattr(mapping, "source_query", None):
307
+ return None
308
+ import time
309
+ table = self._table(mapping)
310
+ cache = self.graph.__dict__.setdefault("_table_versions", {})
311
+ held = cache.get(table)
312
+ if held is not None and time.monotonic() - held[0] < self.VERSION_TTL_S:
313
+ return held[1]
314
+ try:
315
+ rows = self.graph._execute_sql(f"DESCRIBE HISTORY {table} LIMIT 1")
316
+ except Exception: # noqa: BLE001 - no history read: checked on each statement
317
+ return None
318
+ if not rows:
319
+ return None
320
+ found = ("delta", table, rows[0].get("version"))
321
+ cache[table] = (time.monotonic(), found)
322
+ return found
323
+
324
+ @staticmethod
325
+ def same(held, now) -> bool:
326
+ return held == now
327
+
328
+ def pin(self, mapping, version):
329
+ if not self._databricks or version is None or version[2] is None:
330
+ return None
331
+ return PinnedRead(f"{self._table(mapping)} VERSION AS OF {int(version[2])}")
332
+
333
+ def duplicate(self, mapping, version=None):
334
+ from prograph.schema.rows import owned_conditions
335
+ q = self.dialect.quote
336
+ cols = ", ".join(q(c) for c in mapping.id_columns)
337
+ conditions = [f"{q(c)} IS NOT NULL" for c in mapping.id_columns]
338
+ conditions += owned_conditions(mapping, self.dialect)
339
+ pinned = self.pin(mapping, version)
340
+ table = pinned.sql if pinned is not None else self._table(mapping)
341
+ from prograph.plan.sql import serializer_for
342
+ sql = serializer_for(self.dialect_name).write_select(
343
+ table, "_k", [cols], where=conditions, group_by=[cols],
344
+ having=["COUNT(*) > 1"], limit=1)
345
+ try:
346
+ rows = self.graph._execute_sql(sql)
347
+ except Exception as exc: # noqa: BLE001 - the driver's own error types
348
+ # Generated SQL the database rejected: a runtime error that carries the SQL,
349
+ # as for the statement itself.
350
+ from prograph.utils.exceptions import SQLRuntimeError
351
+ raise SQLRuntimeError(f"SQL execution failed: {exc}",
352
+ reason="execution_error", sql=sql) from exc
353
+ if not rows:
354
+ return None
355
+ row = rows[0]
356
+ return tuple(row[k] for k in list(row)[:len(mapping.id_columns)])
357
+
358
+
359
+ def source_of(graph: Any) -> Optional[Any]:
360
+ """The source a graph reads its mappings' rows from, or None for a graph
361
+ that maps no table (the memory backend's own graph)."""
362
+ from prograph.graph.pandas_graph import PandasGraph
363
+ from prograph.graph.sql_graph import SQLGraph
364
+ if isinstance(graph, PandasGraph):
365
+ return FrameSource(graph)
366
+ if isinstance(graph, SQLGraph):
367
+ return SQLSource(graph)
368
+ try:
369
+ from prograph.graph.spark_graph import SparkGraph
370
+ except ImportError: # pragma: no cover - no pyspark
371
+ return None
372
+ if isinstance(graph, SparkGraph):
373
+ return SparkFrameSource(graph)
374
+ return None
375
+
376
+
377
+ # -- once per version -------------------------------------------------------
378
+
379
+ class KeyChecks:
380
+ """One engine's verdicts on its mappings' keys, each kept for the
381
+ version of the data it was made on. ``scans`` counts the times the data
382
+ was read to check a key."""
383
+
384
+ def __init__(self):
385
+ self._verdicts: Dict[Any, Tuple[Any, Any]] = {}
386
+ self._descriptions: Dict[Any, Optional[Dict[str, Any]]] = {}
387
+ self.scans = 0
388
+
389
+ @staticmethod
390
+ def _identity(mapping) -> Tuple[Any, ...]:
391
+ return (id(mapping), mapping.source_table, getattr(mapping, "source_query", None),
392
+ tuple(mapping.id_columns),
393
+ tuple(sorted((mapping.filter_predicate or {}).items(), key=lambda kv: kv[0])))
394
+
395
+ def _described(self, source: Any, mapping: Any) -> Optional[Dict[str, Any]]:
396
+ identity = self._identity(mapping)
397
+ if identity not in self._descriptions:
398
+ described = source.describe(mapping)
399
+ self._descriptions[identity] = described
400
+ if described is not None and not described["history"] and not described["rely"]:
401
+ from prograph.logging import get_plan_logger
402
+ get_plan_logger().warning(
403
+ "node_key_trusted",
404
+ message=(f"The key of node mapping {mapping.label!r} ({mapping.source_table}, "
405
+ f"key {', '.join(mapping.id_columns)}) is trusted, not checked: its "
406
+ f"source is {described['what']}, which has no version to read the "
407
+ f"answer at. If two rows share a key value, the answer is undefined."),
408
+ label=mapping.label, table=mapping.source_table, source=described["what"])
409
+ return self._descriptions[identity]
410
+
411
+ def verdict(self, source: Any, mapping: Any) -> Tuple[Optional[Tuple[Any, ...]], Optional[PinnedRead]]:
412
+ """One key value two of `mapping`'s rows share, or None, read from
413
+ the data once per version; and how the statement reads the version
414
+ checked (None: as the source reads now)."""
415
+ from prograph.logging import get_plan_logger
416
+ described = self._described(source, mapping)
417
+ if described is not None and (described["rely"] or not described["history"]):
418
+ return None, None
419
+ identity = self._identity(mapping)
420
+ version = source.version(mapping)
421
+ held = self._verdicts.get(identity)
422
+ if version is not None and held is not None and held[0] is not None \
423
+ and source.same(held[0], version):
424
+ return held[1], source.pin(mapping, version)
425
+ started = time.perf_counter()
426
+ found = source.duplicate(mapping, version)
427
+ self.scans += 1
428
+ get_plan_logger().debug("node_key_check", label=mapping.label,
429
+ table=mapping.source_table, versioned=version is not None,
430
+ duplicate=found is not None,
431
+ elapsed_ms=round((time.perf_counter() - started) * 1000, 3))
432
+ self._verdicts[identity] = (version, found)
433
+ return found, source.pin(mapping, version)