prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,454 @@
1
+ """Which keys a plan may rely on as unique (one-plan.md 2.4, decided with
2
+ decided 2026-11-08): a key is trusted only when asserted or checked. An
3
+ enforced constraint, a Databricks RELY primary key, or a mapping's
4
+ `keys: trusted`; `keys: check`, verified for the data version, is not built
5
+ yet and relies on nothing until it is. Otherwise the plan keeps its
6
+ Distincts. A stated key is identity, not uniqueness; an R2RML subject
7
+ template's columns state no key; `on_duplicate` is Cypher's policy and says
8
+ nothing to a plan.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import re
14
+ from typing import Any, Dict, Iterable, List, Optional, Tuple
15
+
16
+ from prograph.rules import home
17
+
18
+ KEY_TRUST = ("trusted", "check")
19
+
20
+
21
+ def key_trust_name(value: Optional[str], where: str) -> Optional[str]:
22
+ """A mapping's `keys` setting, checked: 'trusted', 'check' or None."""
23
+ if value is None:
24
+ return None
25
+ if value not in KEY_TRUST:
26
+ raise ValueError(f"{where}: {value!r} is not one of {', '.join(KEY_TRUST)}")
27
+ return value
28
+
29
+
30
+ @home("trusted-keys", "When a plan may rely on a key being unique",
31
+ terms=("trusted key", "unique_keys", "RELY", "PRIMARY KEY", "UNIQUE", "keys: trusted"))
32
+ def trusted_keys(graph: Any, source_map: Any, maps: Any = None) -> Tuple[Tuple[str, ...], ...]:
33
+ """The column sets of `source_map`'s source a plan may rely on as unique:
34
+ the mapping's own key where it says `keys: trusted`; every PRIMARY KEY
35
+ and UNIQUE constraint the database enforces (DuckDB, PostgreSQL); a
36
+ Databricks primary key declared RELY on exactly the key; and, for an edge
37
+ map, a node map's trusted key on the same table that is exactly the edge's
38
+ start columns (`maps`: a parent pointer declared `keys: trusted` on its
39
+ node). Read once per graph and table, cached on the graph."""
40
+ return tuple(columns for columns, _ in key_sources(graph, source_map, maps))
41
+
42
+
43
+ ASSERTED = "asserted by the mapping (keys: trusted), not checked"
44
+ ASSERTED_ID = "asserted by the mapping (a relationship's id_column), not checked"
45
+ CHECKED = "checked for this version of the data (keys: check)"
46
+ ENFORCED = "a constraint the database enforces"
47
+ RELY = "a Databricks primary key declared RELY"
48
+ STORE_VIEW = "one row per created element, by the owned store's view"
49
+ SAME_TABLE = "the key of node map {node} on the same table ({why})"
50
+
51
+
52
+ def key_sources(graph: Any, source_map: Any, maps: Any = None) -> Tuple[Tuple[Tuple[str, ...], str], ...]:
53
+ """`trusted_keys`, each with what makes it trusted (`ASSERTED`,
54
+ `ENFORCED`, `RELY`), for EXPLAIN: an asserted key is never shown as a
55
+ checked one (decided 2026-11-09)."""
56
+ source = source_map.source
57
+ from prograph.schema.store_view import is_store_map
58
+ if is_store_map(source_map):
59
+ # The owned store's generic map: one row per created element (its query groups by it).
60
+ key = getattr(source_map, "key", None)
61
+ return ((tuple(key.columns) if key is not None else tuple(source_map.row_key), STORE_VIEW),)
62
+ if source.query is not None:
63
+ return query_keys(graph, source)
64
+ key = getattr(source_map, "key", None)
65
+ out: List[Tuple[Tuple[str, ...], str]] = []
66
+ if key is not None and key.stated and key.trust == "trusted":
67
+ out.append((tuple(key.columns), ASSERTED))
68
+ if key is not None and key.stated and key.trust == "check":
69
+ if checked_key(graph, source_map):
70
+ out.append((tuple(key.columns), CHECKED))
71
+ row_key = getattr(source_map, "row_key", None)
72
+ if row_key:
73
+ # An edge map's id_column: a key the mapping states, not checked (as `keys: trusted`).
74
+ out.append((tuple(row_key), ASSERTED_ID))
75
+ if getattr(source_map, "trust", None) == "trusted" and hasattr(source_map, "start"):
76
+ # An edge map's asserted key: its start and end columns.
77
+ edge_key = tuple(dict.fromkeys(tuple(source_map.start.columns) + tuple(source_map.end.columns)))
78
+ if edge_key:
79
+ out.append((edge_key, ASSERTED))
80
+ if maps is not None and hasattr(source_map, "start") and source.query is None \
81
+ and getattr(source_map, "join", None) is None:
82
+ # An edge map reading a node map's table, its start columns that node map's trusted key (a parent
83
+ # pointer: `unit.parent` on the unit table, the node keyed `keys: trusted` on `id`): the edge has one
84
+ # row per table row, so it too is unique on its start (rule trusted-keys).
85
+ start = tuple(source_map.start.columns)
86
+ for node in getattr(maps, "nodes", ()):
87
+ # The node's key holds over its filtered rows only: the edge must read no row outside them (its
88
+ # own filter holds every (column, value) of the node's).
89
+ if source.rows_within(node.source):
90
+ for columns, why in key_sources(graph, node):
91
+ if set(columns) == set(start) and columns not in [c for c, _ in out]:
92
+ out.append((columns, SAME_TABLE.format(node=node.id, why=why)))
93
+ rely = _dialect(graph) in ("databricks_sql", "databricks")
94
+ # On a Databricks warehouse, where the mapping asserts its key, no catalog read is needed to trust it:
95
+ # the constraint probe (a DESCRIBE TABLE EXTENDED, a round trip of about a second) is skipped.
96
+ # Only on a warehouse, where the read is a round trip; elsewhere the database's own keys are read too
97
+ # (DuckDB and PostgreSQL enforce them, and a plan may rely on one beside the asserted key).
98
+ asserted = rely and [c for c, why in out if why == ASSERTED]
99
+ for k in (() if asserted else _constraint_keys(graph, source.qualified, key)):
100
+ if k not in [c for c, _ in out]:
101
+ out.append((k, RELY if rely else ENFORCED))
102
+ for k, why in out:
103
+ if why in (ASSERTED, ASSERTED_ID, RELY):
104
+ warn_if_repeating(graph, source.qualified, k, why)
105
+ return tuple(out)
106
+
107
+
108
+ QUERY_TABLE = "{why} of the table a plain rr:sqlQuery selects from"
109
+
110
+
111
+ def query_keys(graph: Any, source: Any) -> Tuple[Tuple[Tuple[str, ...], str], ...]:
112
+ """The keys an rr:sqlQuery source holds: where the query is a plain select of one table (`simple_select`:
113
+ `SELECT * | t.* | columns FROM table WHERE ...`, no join, grouping, DISTINCT, window, LIMIT, set operation,
114
+ expression or renamed column), its rows are rows of that table, each once, so a key the database
115
+ enforces (or Databricks RELY) on the table holds on them where the query projects all its columns. A
116
+ mapping cannot assert a key on a query; any other query has none (rule trusted-keys)."""
117
+ from prograph.schema.source_map import simple_select
118
+ shape = simple_select(source.query)
119
+ if shape is None:
120
+ return ()
121
+ rely = _dialect(graph) in ("databricks_sql", "databricks")
122
+ out: List[Tuple[Tuple[str, ...], str]] = []
123
+ for k in _constraint_keys(graph, shape.table, None):
124
+ if (shape.columns is None or set(k) <= set(shape.columns)) and k not in [c for c, _ in out]:
125
+ out.append((k, QUERY_TABLE.format(why=RELY if rely else ENFORCED)))
126
+ return tuple(out)
127
+
128
+
129
+ def _dialect(graph: Any) -> Optional[str]:
130
+ try:
131
+ return graph.sql_dialect() if graph is not None and hasattr(graph, "sql_dialect") else None
132
+ except Exception: # noqa: BLE001
133
+ return None
134
+
135
+
136
+ class AssertedKeyWarning(UserWarning):
137
+ """A mapping's key is asserted (`keys: trusted`) or asked to be checked
138
+ (`keys: check`), not enforced or checked: see `warn_asserted_keys`."""
139
+
140
+
141
+ def warn_asserted_keys(schema: Any) -> None:
142
+ """At mapping load, one warning per mapping whose keys are asserted
143
+ (`keys: trusted`) or asked to be checked (`keys: check`, not built: it
144
+ relies on nothing yet). An asserted key is the user's promise, an
145
+ escape hatch where the database cannot enforce one; plans rely on it,
146
+ and a key the data breaks gives wrong answers (decided 2026-11-09:
147
+ m2-performance.md 2.4)."""
148
+ import warnings
149
+ from prograph.plan.ir.entry import source_maps
150
+ try:
151
+ maps = source_maps(schema)
152
+ except Exception: # noqa: BLE001 - a schema the one plan does not read asserts no key here
153
+ return
154
+ # The source maps, not the authoring objects: a mapping generated as source maps (no YAML) warns
155
+ # alike, one warning per map (one per YAML mapping: the compiler makes one map of each).
156
+ for kind, found in (("node", [(m, m.key.trust, m.label or m.id, m.key.columns) for m in maps.nodes]),
157
+ ("relationship", [(m, m.trust, m.rel_type or m.id, m.start.columns + m.end.columns)
158
+ for m in maps.edges])):
159
+ for m, trust, name, cols in found:
160
+ if trust is None:
161
+ continue
162
+ columns = ", ".join(str(c) for c in cols if c)
163
+ if trust == "trusted":
164
+ message = (f"The key of {kind} mapping {name!r} ({m.source.table}: {columns}) is "
165
+ f"asserted (keys: trusted), not checked. Plans rely on it to drop "
166
+ f"duplicate removal and joins; if two rows share a key value, SPARQL "
167
+ f"answers are wrong. Where the database can enforce the key (a PRIMARY "
168
+ f"KEY, UNIQUE, or a Databricks PRIMARY KEY ... RELY), declare it there.")
169
+ else:
170
+ continue # checked per data version (`checked_key`), refused without one
171
+ warnings.warn(message, AssertedKeyWarning, stacklevel=3)
172
+
173
+
174
+ def _constraint_keys(graph: Any, qualified: str, key: Any) -> Tuple[Tuple[str, ...], ...]:
175
+ if graph is None or not hasattr(graph, "sql_dialect") or not hasattr(graph, "_execute_sql"):
176
+ return ()
177
+ cache = graph.__dict__.setdefault("_trusted_key_cache", {})
178
+ if qualified not in cache:
179
+ try:
180
+ cache[qualified] = _read_constraints(graph, qualified, key)
181
+ except Exception: # noqa: BLE001 - a catalog that cannot be read trusts nothing
182
+ cache[qualified] = ()
183
+ return cache[qualified]
184
+
185
+
186
+ def _read_constraints(graph: Any, qualified: str, key: Any) -> Tuple[Tuple[str, ...], ...]:
187
+ from prograph.plan.sql.dialect import get_dialect_instance
188
+ name = graph.sql_dialect()
189
+ lit = get_dialect_instance(name).literal
190
+ parts = [p.strip('`"') for p in qualified.split(".")]
191
+ table, schema = parts[-1], (parts[-2] if len(parts) > 1 else None)
192
+ database = parts[-3] if len(parts) > 2 else None
193
+ if name == "duckdb":
194
+ # The table the name resolves to: its database and schema the
195
+ # current ones where the name leaves them out (fresh review,
196
+ # 2026-11-09: a same-named table elsewhere is another table).
197
+ sql = ("SELECT constraint_column_names AS cols FROM duckdb_constraints() "
198
+ f"WHERE constraint_type IN ('PRIMARY KEY', 'UNIQUE') AND table_name = {lit(table)}"
199
+ f" AND schema_name = {lit(schema) if schema else 'current_schema()'}"
200
+ f" AND database_name = {lit(database) if database else 'current_database()'}")
201
+ return tuple(tuple(r["cols"]) for r in graph._execute_sql(sql))
202
+ if name in ("postgresql", "postgres"):
203
+ sql = ("SELECT tc.constraint_name AS name, kcu.column_name AS col "
204
+ "FROM information_schema.table_constraints tc "
205
+ "JOIN information_schema.key_column_usage kcu "
206
+ "ON kcu.constraint_schema = tc.constraint_schema "
207
+ "AND kcu.constraint_name = tc.constraint_name "
208
+ f"WHERE tc.constraint_type IN ('PRIMARY KEY', 'UNIQUE') AND tc.table_name = {lit(table)}"
209
+ " AND tc.is_deferrable = 'NO'")
210
+ sql += f" AND tc.table_schema = {lit(schema) if schema else 'current_schema()'}"
211
+ sql += " ORDER BY tc.constraint_name, kcu.ordinal_position"
212
+ by_name: Dict[str, List[str]] = {}
213
+ for r in graph._execute_sql(sql):
214
+ by_name.setdefault(r["name"], []).append(r["col"])
215
+ return tuple(tuple(v) for v in by_name.values())
216
+ if name in ("databricks_sql", "databricks"):
217
+ # Every RELY primary key the table declares, as PostgreSQL's and DuckDB's are read
218
+ # whether or not the mapping states the key (LDBC's edge tables). A table the engine's
219
+ # catalog read found no primary key for has none to read.
220
+ if graph.__dict__.get("_warehouse_keyed", {}).get(qualified) is False:
221
+ return ()
222
+ rows = _described_extended(graph, qualified)
223
+ found = (rely_primary_key_columns(" ".join(str(v) for v in row.values() if v is not None))
224
+ for row in rows)
225
+ return tuple(dict.fromkeys(cols for cols in found if cols))
226
+ return ()
227
+
228
+
229
+ def rely_primary_key_columns(text: str) -> Optional[Tuple[str, ...]]:
230
+ """The columns of a RELY primary key a constraint's text declares
231
+ (``PRIMARY KEY (`Forum.id`, `Person.id`) RELY``, any case, names quoted
232
+ or not), or None."""
233
+ import re
234
+ match = re.search(r"PRIMARY\s+KEY\s*\(([^)]*)\)(.*)", text, re.IGNORECASE)
235
+ if match is None or not re.search(r"\bRELY\b", match.group(2), re.IGNORECASE) \
236
+ or re.search(r"\bNORELY\b", match.group(2), re.IGNORECASE):
237
+ return None
238
+ names = re.findall(r"`((?:[^`]|``)*)`|\"((?:[^\"]|\"\")*)\"|([^,\s]+)", match.group(1))
239
+ return tuple((a.replace("``", "`") or b.replace('""', '"') or c) for a, b, c in names) or None
240
+
241
+
242
+ def rely_primary_key(text: str, columns: Iterable[str]) -> bool:
243
+ """Whether a constraint's text declares a RELY primary key on exactly `columns`."""
244
+ found = rely_primary_key_columns(text)
245
+ return found is not None and {c.lower() for c in found} == {c.lower() for c in columns}
246
+
247
+
248
+ # -- keys: check (docs/architecture/keys-and-identity.md) -------------------------------
249
+
250
+ import contextvars # noqa: E402
251
+
252
+ # The pinned reads (`key_checks.Pins`) the keys checked while a query is planned ask for: the
253
+ # statement reads each checked table at the version checked (`key_checks.pinned_name`, `lower._pinned`).
254
+ PLANNING_PINS: contextvars.ContextVar = contextvars.ContextVar("prograph_planning_pins", default=None)
255
+
256
+
257
+ class KeyCheckUnavailableError(ValueError):
258
+ """`keys: check` on a map whose source has no version to check it at (a PostgreSQL or DuckDB
259
+ table, a Spark frame that is not Delta): the check and the query would be two statements over
260
+ data that may change between them, so the setting is refused rather than quietly not acted on
261
+ (decided 2026-11-10)."""
262
+
263
+
264
+ class RepeatedKeyWarning(UserWarning):
265
+ """A `keys: check` key that two of the map's rows share: not relied on, the plan keeps its
266
+ Distincts (the answers stay right)."""
267
+
268
+
269
+ def _checked_mapping(graph: Any, source_map: Any) -> Any:
270
+ """What `key_checks.KeyChecks` reads of a map, one object per map (its verdicts are kept by
271
+ the object): the map's rows (its source, row filter or query) and its key."""
272
+ from types import SimpleNamespace
273
+ held = graph.__dict__.setdefault("_key_check_maps", {})
274
+ source = source_map.source
275
+ ident = (source_map.id, source.table, source.qualified, source.query, source.filter,
276
+ tuple(source_map.key.columns))
277
+ if ident not in held:
278
+ held[ident] = SimpleNamespace(
279
+ label=getattr(source_map, "label", None) or source_map.id, source_table=source.table,
280
+ qualified_source_table=source.qualified, source_query=source.query,
281
+ id_columns=list(source_map.key.columns), filter_predicate=dict(source.filter or ()))
282
+ return held[ident]
283
+
284
+
285
+ def _key_source(graph: Any) -> Any:
286
+ held = graph.__dict__.get("_key_check_source")
287
+ if held is None:
288
+ from prograph.plan.ir.key_checks import KeyChecks, source_of
289
+ held = (source_of(graph), KeyChecks())
290
+ graph.__dict__["_key_check_source"] = held
291
+ return held
292
+
293
+
294
+ @home("keys-check", "keys: check, a key verified for the data's version",
295
+ terms=("keys: check", "KeyCheckUnavailableError", "RepeatedKeyWarning", "VERSION AS OF",
296
+ "data version", "checked key"))
297
+ def checked_key(graph: Any, source_map: Any) -> bool:
298
+ """Whether a `keys: check` map's key is unique over the map's rows (no two rows share it, a
299
+ row with a missing key part left out) at the data's current version, read once per version
300
+ (`key_checks.KeyChecks`); the statement then reads the version checked (`PLANNING_PINS`). A
301
+ source with no version is refused when the engine starts (`refuse_unversioned_checks`); a key
302
+ two rows share is not relied on, with a warning naming one of its values."""
303
+ if graph is None:
304
+ return False
305
+ source, checks = _key_source(graph)
306
+ if source is None:
307
+ return False
308
+ mapping = _checked_mapping(graph, source_map)
309
+ if source.version(mapping) is None:
310
+ return False
311
+ found, pin = checks.verdict(source, mapping)
312
+ if found is not None:
313
+ import warnings
314
+ warned = graph.__dict__.setdefault("_key_check_warned", set())
315
+ if (id(mapping), found) not in warned:
316
+ warned.add((id(mapping), found))
317
+ warnings.warn(f"keys: check on {mapping.label} ({mapping.source_table}): two rows share the key "
318
+ f"{', '.join(mapping.id_columns)} = {found!r}, so the plan does not rely on it",
319
+ RepeatedKeyWarning, stacklevel=4)
320
+ return False
321
+ from prograph.plan.ir.key_checks import FrameSource
322
+ if pin is None and not isinstance(source, FrameSource):
323
+ # Checked but not pinned: the statement would read the source as it is then, not the
324
+ # version checked (a Spark frame derived from Delta tables; a table with no history, or
325
+ # one that cannot be described). Not relied on (M2 review R2, R3); a pandas frame's
326
+ # version is the frame itself, which ProGraph's own writes replace.
327
+ return False
328
+ pins = PLANNING_PINS.get()
329
+ if pins is not None and pin is not None:
330
+ pins[mapping.qualified_source_table] = pin
331
+ return True
332
+
333
+
334
+ def refuse_unversioned_checks(engine: Any) -> None:
335
+ """At engine start: `keys: check` on a map whose source gives no version is refused, naming the
336
+ ways out for that source."""
337
+ schema = getattr(engine, "_schema", None)
338
+ graph = getattr(getattr(engine, "executor", None), "graph", None)
339
+ if schema is None or graph is None:
340
+ return
341
+ try:
342
+ from prograph.plan.ir.entry import source_maps
343
+ maps = source_maps(schema)
344
+ except Exception: # noqa: BLE001 - a schema the one plan does not read checks nothing here
345
+ return
346
+ checked = [m for m in maps.nodes if m.key is not None and m.key.stated and m.key.trust == "check"]
347
+ if not checked:
348
+ return
349
+ source, _ = _key_source(graph)
350
+ backend = getattr(engine, "backend", None)
351
+ for m in checked:
352
+ mapping = _checked_mapping(graph, m)
353
+ if source is not None and source.version(mapping) is not None:
354
+ continue
355
+ if backend == "spark":
356
+ ways = "make the table a Delta table, write keys: trusted, or remove keys: check"
357
+ what = "a Spark frame that is not a Delta table"
358
+ elif backend in ("postgresql", "duckdb"):
359
+ ways = ("declare a PRIMARY KEY or UNIQUE constraint on the key (the database enforces it and "
360
+ "the plan relies on it), write keys: trusted, or remove keys: check (the plan keeps "
361
+ "its Distincts: the same answers, slower)")
362
+ what = f"a {backend} table, which has no version"
363
+ else:
364
+ ways = "write keys: trusted, or remove keys: check"
365
+ what = "a source with no version"
366
+ raise KeyCheckUnavailableError(
367
+ f"keys: check on {mapping.label} ({m.source.table}) cannot be kept: its source is {what}, so "
368
+ f"the check and the query would read data that may change between them. To fix: {ways}.")
369
+
370
+
371
+ # -- the load warning (one-plan.md 2.4; keys-and-identity.md, point 3) ----------------------------
372
+
373
+ class RepeatingKeyWarning(UserWarning):
374
+ """A key the plan relies on because it is asserted (`keys: trusted`, a Databricks RELY), whose
375
+ table's statistics count fewer distinct values than 0.9 of its rows: a repeat gives wrong
376
+ answers. 0.9 catches gross repeats only; for a relied-on key, a 1% repeat is wrong answers too."""
377
+
378
+
379
+ @home("key-load-warning", "Warning that an asserted key repeats, from the engine's statistics",
380
+ terms=("RepeatingKeyWarning", "n_distinct", "distinct_count", "statistics", "0.9"))
381
+ def warn_if_repeating(graph: Any, qualified: str, columns: Tuple[str, ...], why: str) -> None:
382
+ """Once per table and key: warn where the engine's own statistics (PostgreSQL `pg_stats`, a
383
+ Databricks or Spark table's column statistics) give the key fewer distinct values than 0.9 of
384
+ the rows. Never a scan: no statistics, no warning. A single-column key only."""
385
+ if graph is None or len(columns) != 1:
386
+ return
387
+ seen = graph.__dict__.setdefault("_repeating_key_warned", set())
388
+ if (qualified, columns) in seen:
389
+ return
390
+ seen.add((qualified, columns))
391
+ try:
392
+ counted = _statistics(graph, qualified, columns[0])
393
+ except Exception: # noqa: BLE001 - statistics that cannot be read warn nothing
394
+ return
395
+ if counted is None:
396
+ return
397
+ distinct, rows = counted
398
+ if rows > 0 and distinct < 0.9 * rows:
399
+ import warnings
400
+ warnings.warn(
401
+ f"the key {columns[0]} of {qualified} is relied on ({why}), but the table's statistics count "
402
+ f"about {int(distinct)} distinct values in {int(rows)} rows: where two rows share a key value, "
403
+ f"SPARQL answers are wrong. Use a fuller key, a current-rows filter, or keys: check.",
404
+ RepeatingKeyWarning, stacklevel=4)
405
+
406
+
407
+ def _described_extended(graph: Any, qualified: str) -> List[Dict[str, Any]]:
408
+ """`DESCRIBE TABLE EXTENDED` of a table, read once per graph: the RELY constraints and the table's
409
+ statistics come from one statement."""
410
+ cache = graph.__dict__.setdefault("_described_extended", {})
411
+ if qualified not in cache:
412
+ cache[qualified] = list(graph._execute_sql(f"DESCRIBE TABLE EXTENDED {qualified}"))
413
+ return cache[qualified]
414
+
415
+
416
+ def _statistics(graph: Any, qualified: str, column: str) -> Optional[Tuple[float, float]]:
417
+ """(distinct values, rows) from the engine's statistics, or None where it keeps none."""
418
+ if not hasattr(graph, "_execute_sql"):
419
+ return None
420
+ name = _dialect(graph)
421
+ parts = [p.strip('`"') for p in qualified.split(".")]
422
+ if name in ("postgresql", "postgres"):
423
+ from prograph.plan.sql.dialect import get_dialect_instance
424
+ lit = get_dialect_instance(name).literal
425
+ table, schema = parts[-1], (parts[-2] if len(parts) > 1 else None)
426
+ where = f"tablename = {lit(table)} AND attname = {lit(column)}" + \
427
+ (f" AND schemaname = {lit(schema)}" if schema else " AND schemaname = current_schema()")
428
+ rows = graph._execute_sql(
429
+ f"SELECT s.n_distinct AS d, c.reltuples AS r FROM pg_stats s JOIN pg_class c ON c.relname = s.tablename "
430
+ f"JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = s.schemaname WHERE {where}")
431
+ if not rows or rows[0]["r"] is None or float(rows[0]["r"]) <= 0:
432
+ return None
433
+ d, r = float(rows[0]["d"]), float(rows[0]["r"])
434
+ return (-d * r if d < 0 else d), r
435
+ if name == "databricks_sql":
436
+ # On a SQL warehouse each statistics read is a round trip of about a second before the first answer;
437
+ # the warning is a hint, not a check, so it reads nothing there.
438
+ return None
439
+ if name in ("databricks", "spark"):
440
+ import re
441
+ table_rows = None
442
+ for row in _described_extended(graph, qualified):
443
+ text = " ".join(str(v) for v in row.values() if v is not None)
444
+ m = re.search(r"Statistics\s+[\d,]+ bytes,\s*([\d,]+) rows", text)
445
+ if m:
446
+ table_rows = float(m.group(1).replace(",", ""))
447
+ if not table_rows:
448
+ return None
449
+ for row in graph._execute_sql(f"DESCRIBE TABLE EXTENDED {qualified} {column}"):
450
+ values = [str(v) for v in row.values() if v is not None]
451
+ if values and values[0] == "distinct_count" and len(values) > 1 and values[1] not in ("NULL", ""):
452
+ return float(values[1]), table_rows
453
+ return None
454
+ return None