prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,992 @@
1
+ """The engine's query path: a query's text, through its language's front
2
+ end, to one plan over the source maps, lowered to SQL and run on the
3
+ engine's backend (docs/architecture/one-plan.md).
4
+
5
+ What is not built yet raises NotImplementedError whose message starts
6
+ "one-plan:".
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import contextvars
12
+ from typing import Any, Dict, List, Optional, Tuple
13
+
14
+
15
+ def source_maps(schema: Any) -> Any:
16
+ """The schema's source maps: an R2RML mapping's compiled straight from
17
+ it (one-plan.md 2.6), kept on the schema; else the schema's own."""
18
+ r2rml = schema.__dict__.get("r2rml_source")
19
+ if r2rml is None:
20
+ return schema.source_maps()
21
+ from prograph.r2rml.schema import store_version, with_included
22
+ cached = schema.__dict__.get("_one_plan_maps")
23
+ version = (store_version(schema), getattr(schema, "decimal_text", None))
24
+ if cached is None or cached[0] is not r2rml or cached[1] != version:
25
+ from prograph.r2rml.compile import compile_r2rml
26
+ cached = (r2rml, version, with_included(schema, compile_r2rml(r2rml[0], r2rml[1])))
27
+ schema.__dict__["_one_plan_maps"] = cached
28
+ return cached[2]
29
+
30
+
31
+ def saturated(maps: Any, onto: Any) -> Any:
32
+ """`maps` saturated by `onto` (T-mappings), kept on the maps for that ontology: the same maps
33
+ and ontology give the same saturated maps, and so the same property-graph view (`lpg_view.view_of`)."""
34
+ held = maps.__dict__.get("_saturated")
35
+ if held is None or held[0] is not onto:
36
+ from prograph.plan.ir.saturate import saturate
37
+ held = (onto, saturate(maps, onto))
38
+ maps.__dict__["_saturated"] = held
39
+ return held[1]
40
+
41
+
42
+ def mapped_tables(schema: Any) -> Any:
43
+ """The logical tables the schema's source maps read (`SourceMaps.tables`): what an engine
44
+ loads and types, so they are the tables a plan reads."""
45
+ return source_maps(schema).tables()
46
+
47
+
48
+ def _cypher_identity(engine: Any) -> Any:
49
+ """The owl:sameAs components a Cypher query reads (one node per component), under profile rl; None else."""
50
+ if getattr(engine, "_profile", None) != "rl":
51
+ return None
52
+ from prograph.plan.ir.identity import for_engine
53
+ return for_engine(engine)
54
+
55
+
56
+ def _cypher_stored(engine: Any) -> Any:
57
+ """The engine's warnings of the node properties stored OWL RL facts give, under profile rl; None else."""
58
+ if getattr(engine, "_profile", None) != "rl":
59
+ return None
60
+ from prograph.plan.ir.cypher_ontology import stored_values_of
61
+ return stored_values_of(engine)
62
+
63
+
64
+ # The keys a running query (`run`) read beside stored OWL RL facts, gathered as it is planned (`_planned`): None
65
+ # outside `run`, so to_sql and explain gather none.
66
+ _STORED_READS: Any = contextvars.ContextVar("prograph_stored_reads", default=None)
67
+
68
+
69
+ def _check_stored(engine: Any, reads: Any) -> None:
70
+ """After a read-only Cypher query ran: whether the keys it read beside stored OWL RL facts have values that
71
+ differ, warned once per engine (`cypher_ontology.StoredValues.check`). Never from to_sql, explain or a write
72
+ (a write's translation gathers no keys)."""
73
+ checker = _cypher_stored(engine) if reads else None
74
+ if checker is not None:
75
+ for key, label, predicates in dict.fromkeys(reads):
76
+ checker.check(label, key, predicates)
77
+
78
+
79
+ def ontology_of(engine: Any) -> Any:
80
+ """The engine's ontology as the plan reads it (`saturate.Ontology`), read once per ontology
81
+ content; None without one. The axioms its profile does not answer are refused, or dropped
82
+ (`profile_report`). The ontology's triples are read whole: a dropped axiom is not reasoned with (no OWL 2 RL
83
+ rule is compiled from it, `ProGraph._rl_rules`), and its triples stay in the default graph as stated."""
84
+ layer = getattr(engine, "_ontology", None)
85
+ if layer is None:
86
+ return None
87
+ from prograph.plan.ir.saturate import read_ontology
88
+ key, _, kept = profile_report(engine)
89
+ graph = layer.rdf_graph if not callable(layer.rdf_graph) else layer.rdf_graph()
90
+ cached = getattr(engine, "_saturation", None)
91
+ if cached is None or cached[0] != key:
92
+ # Under profile="rl" the axioms OWL RL's rules apply are applied, not ignored.
93
+ rl = getattr(engine, "_profile", None) == "rl"
94
+ # Under "ql" the QL axioms are read from the graph less those on_unsupported="ignore" dropped.
95
+ onto = read_ontology(graph, engine._rl_rules().consumed if rl else frozenset(), rl=rl, kept=kept)
96
+ cached = (key, onto)
97
+ engine._saturation = cached
98
+ return cached[1]
99
+
100
+
101
+ def profile_report(engine: Any) -> Any:
102
+ """(content key, `profiles.OntologyReport`, the graph OWL 2 RL's rules are compiled from) for the engine's ontology,
103
+ once per ontology content: the axioms its profile does not answer raise `OWLProfileError`, or under
104
+ `on_unsupported="ignore"` are dropped from the graph (`profiles.kept_graph`)."""
105
+ layer = engine._ontology
106
+ graph = layer.rdf_graph if not callable(layer.rdf_graph) else layer.rdf_graph()
107
+ key = (len(graph), hash(frozenset(graph)))
108
+ held = engine.__dict__.get("_profile_checked")
109
+ if held is None or held[0] != key:
110
+ from prograph.plan.ir import profiles
111
+ got = profiles.check(graph, engine._profile, getattr(layer, "on_unsupported", "error"))
112
+ held = (key, got, profiles.kept_graph(graph, got))
113
+ engine.__dict__["_profile_checked"] = held
114
+ return held
115
+
116
+
117
+ def _planned(engine: Any, query: str, language: str, hints: Any,
118
+ parameters: Optional[dict]):
119
+ """The query's plan and form, and whether its answer is in full terms."""
120
+ if getattr(engine, "_profile", None) == "ql":
121
+ # The consistency verdict re-proven for this query, SPARQL or Cypher (`consistency.recheck`).
122
+ from prograph.plan.ir.consistency import recheck
123
+ from prograph.plan.ir.dataset import EXPLAINING
124
+ recheck(engine, explaining=EXPLAINING.get())
125
+ if language == "cypher":
126
+ # Cypher reads labels: refused over an R2RML mapping whatever the front end.
127
+ from prograph.r2rml.schema import require_property_graph
128
+ require_property_graph(engine._schema, "Cypher")
129
+ if language == "cypher":
130
+ from prograph.plan.ir import cypher_front
131
+ from prograph.plan.ir.keys import trusted_keys
132
+ from prograph.parameters import cypher_values
133
+ graph = getattr(getattr(engine, "executor", None), "graph", None)
134
+ # The ontology saturates the maps for Cypher (T-mappings): inferred labels,
135
+ # relationship types and property keys (one-plan 2.5). The owned store is read once for the planning
136
+ # (`store_now`: what another writer appended since, its labels, keys and types).
137
+ from prograph.plan.ir.dataset import maps_seen
138
+ onto = ontology_of(engine)
139
+ maps = maps_seen(engine, onto)
140
+ if onto is not None:
141
+ from prograph.plan.ir.cypher_ontology import warn_once
142
+ layer = engine._ontology
143
+ warn_once(engine, layer.rdf_graph if not callable(layer.rdf_graph) else layer.rdf_graph(),
144
+ [e.predicate for e in maps.edges if e.predicate],
145
+ cypher_front._view(maps, onto).typing_lost, cypher_front._view(maps, onto).derived_values)
146
+ translated = cypher_front.translate(query, maps, _held(engine._schema),
147
+ lambda m: trusted_keys(graph, m, maps),
148
+ procedures=engine._read_procedures(),
149
+ step=engine.__dict__.get("_write_step"),
150
+ dialect=graph.sql_dialect() if hasattr(graph, "sql_dialect") else None,
151
+ parameters=cypher_values(parameters),
152
+ superclasses=getattr(onto, "superclasses", None),
153
+ ontology=onto,
154
+ identity=_cypher_identity(engine),
155
+ stored=_cypher_stored(engine))
156
+ _warn_untrusted_keys(engine, translated)
157
+ gathering = _STORED_READS.get()
158
+ if gathering is not None: # a query `run` runs: its keys checked once it has answered
159
+ gathering.extend(getattr(translated, "stored_reads", ()))
160
+ # Cypher plans skip the plan rewrites (below, for SPARQL): lowered as translated
161
+ # (tests/plan/test_cypher_plans_skip_rewrites.py).
162
+ return translated, False, maps.settings.base_iri
163
+ if language != "sparql":
164
+ raise NotImplementedError(f"one-plan: the {language} front end")
165
+ from prograph.plan.ir import sparql_front
166
+ from prograph.parameters import bindings as bound_terms
167
+ schema = engine._schema
168
+ bindings = bound_terms(parameters, getattr(schema, "decimal_text", None))
169
+ graph = getattr(getattr(engine, "executor", None), "graph", None)
170
+ held = _held(schema, graph.sql_dialect() if hasattr(graph, "sql_dialect") else None, True)
171
+ # The dataset the query sees: the mapped data, the store's (read once for the planning, `store_now`: what
172
+ # another writer appended since, the record of the last run, the alias generation), saturated by the
173
+ # ontology (T-mappings).
174
+ from prograph.plan.ir.dataset import maps_seen, store_now
175
+ onto = ontology_of(engine)
176
+ now = store_now(engine, onto)
177
+ maps = maps_seen(engine, onto, now)
178
+ from prograph.plan.ir.keys import trusted_keys
179
+ graph = getattr(getattr(engine, "executor", None), "graph", None)
180
+ from prograph.plan.ir.identity import for_engine
181
+ translated = sparql_front.translate(query, maps, held, lambda m: trusted_keys(graph, m, maps),
182
+ union_default=getattr(engine, "default_graph", "union") == "union",
183
+ ontology=onto, bindings=bindings,
184
+ dataset=getattr(hints, "dataset", None), identity=for_engine(engine, now),
185
+ implied_individuals=getattr(engine, "_implied_individuals", False),
186
+ datatypes=getattr(engine, "_datatypes", "term"))
187
+ from dataclasses import replace
188
+ from prograph.plan.ir.rewrite import rewrite
189
+ engine_dialect = graph.sql_dialect() if hasattr(graph, "sql_dialect") else None
190
+ plan, certificates = rewrite(translated.plan, engine_dialect)
191
+ check_sources(engine, plan)
192
+ translated = replace(translated, plan=plan, certificates=certificates)
193
+ terms_all = getattr(hints, "terms", None) == "all"
194
+ return translated, terms_all, maps.settings.base_iri
195
+
196
+
197
+ class UntrustedKeyWarning(UserWarning):
198
+ """A node mapping whose key is not declared unique: Cypher merges its duplicate rows; a relationship
199
+ mapping with no key: Cypher numbers the rows of one start and end to tell them apart."""
200
+
201
+
202
+ def _warn_untrusted_keys(engine: Any, translated: Any) -> None:
203
+ """Each node mapping a Cypher query grouped because its key is neither trusted nor checked, once per
204
+ engine, in property-graph words (its rows are grouped so a repeated key is one node; declaring the key
205
+ removes that work). Read off the plan: no catalog read."""
206
+ import warnings
207
+ warned = engine.__dict__.setdefault("_untrusted_keys_warned", set())
208
+ for label, columns, table in getattr(translated, "untrusted", ()):
209
+ if label in warned:
210
+ continue
211
+ warned.add(label)
212
+ warnings.warn(f"{label}'s key ({', '.join(columns)}) isn't declared unique, so queries merge duplicate "
213
+ f"rows of {table}. Declare keys: trusted, or keys: check on Delta.",
214
+ UntrustedKeyWarning, stacklevel=5)
215
+ for rel_type, table in getattr(translated, "untrusted_rels", ()):
216
+ if ("rel", rel_type) in warned:
217
+ continue
218
+ warned.add(("rel", rel_type))
219
+ warnings.warn(f"{rel_type}'s rows have no declared key, so queries number repeated rows of {table}. Declare "
220
+ f"id_column, or keys: trusted if {table} holds one row per start and end.",
221
+ UntrustedKeyWarning, stacklevel=5)
222
+ def check_sources(engine: Any, plan: Any) -> None:
223
+ """Refuse a query that reads a source the engine could not read, naming the source, its table and the
224
+ database's message (`ProGraph._record_column_types`), or a column the mapping names that the table it
225
+ described lacks; else the cause would surface as a query that cannot be built."""
226
+ from prograph.plan.ir import ops as O
227
+ from prograph.utils.exceptions import SourceNotReadableError
228
+ unreadable = getattr(engine, "_unreadable_sources", None) or {}
229
+ schema = engine._schema
230
+ seen = set()
231
+ stack = [plan]
232
+ while stack:
233
+ op = stack.pop()
234
+ if id(op) in seen:
235
+ continue
236
+ seen.add(id(op))
237
+ stack.extend(op.children())
238
+ if not isinstance(op, O.MapScan):
239
+ continue
240
+ source = op.map.source
241
+ who = getattr(op.map, "label", None) or getattr(op.map, "rel_type", None) or op.map.id
242
+ join = getattr(op.map, "join", None)
243
+ named = [source] + ([join.parent] if join is not None else []) # a joint query's parent too
244
+ bad = next((x for x in named if x.table in unreadable), None)
245
+ if bad is not None:
246
+ source = bad
247
+ qualified, message = unreadable[source.table]
248
+ raise SourceNotReadableError(
249
+ f"the source {source.table!r} ({who}) could not be read{_where(engine)}: {message}. Name the table "
250
+ f"as the database knows it ({qualified!r} was read): qualify it (catalog.schema.table), or give "
251
+ f"the engine default_catalog= and default_schema=; a YAML mapping for a SQL backend takes the "
252
+ f"sources' tables with to_schema(by_table=True). A table created after the engine is read once "
253
+ f"update_schema() is called or the engine is built again")
254
+ described = (getattr(schema, "column_types", {}) or {}).get(source.table)
255
+ if described:
256
+ known = {str(c).lower() for c in described}
257
+ for r in op.reads:
258
+ if not r.parent and str(r.column).lower() not in known:
259
+ raise SourceNotReadableError(
260
+ f"the mapping {who} reads the column {r.column!r} of {source.table!r}, which the table "
261
+ f"does not have; its columns are {sorted(map(str, described))}")
262
+
263
+
264
+ def _where(engine: Any) -> str:
265
+ """Where a bare table name resolves on the engine's connection (`on the warehouse, in main.default`), for a
266
+ message; empty where it cannot be read."""
267
+ conn = getattr(engine, "_connection", None)
268
+ backend = getattr(engine, "backend", None)
269
+ if conn is None:
270
+ return ""
271
+ sql = {"databricks_sql": "SELECT current_catalog(), current_schema()",
272
+ "duckdb": "SELECT current_database(), current_schema()",
273
+ "postgresql": "SELECT current_database(), current_schema()"}.get(backend)
274
+ if sql is None:
275
+ return f" on {backend}"
276
+ try:
277
+ from prograph.graph.drivers import fetch_rows
278
+ row = fetch_rows(conn, sql)[1][0]
279
+ return f" on {backend}, where a bare name is read in {row[0]}.{row[1]}"
280
+ except Exception: # noqa: BLE001 - the message stands without it
281
+ return f" on {backend}"
282
+
283
+
284
+ def _held(schema: Any, dialect: Optional[str] = None, text_of_untyped: bool = False):
285
+ """How a table's column is held: (its datatype, zoned, digits, reader, bytewise), from its recorded type
286
+ (bytewise: a string that compares byte for byte, `collations.compares_bytes`);
287
+ `held.kind_of(table, column)` the recorded type itself (Cypher reads a TIME column as a local time).
288
+ On PostgreSQL (`dialect`) a NUMERIC column is read through `NUMERIC_VALUE`: a NaN or infinity, no
289
+ xsd:decimal, is a data error naming the column wherever the column is read, never a number PostgreSQL
290
+ ranks above every other (`lower._read`).
291
+ `text_of_untyped` (SPARQL): a column of a type with no natural datatype is read as the database's text
292
+ (rule `no-natural-datatype-text`); Cypher keeps its own values (lists, maps)."""
293
+ from prograph.schema import xsd
294
+
295
+ def held(table: str, column: str):
296
+ kind = schema.table_column_type(table, column)
297
+ dt = xsd.datatype_of_type(kind)
298
+ textual = text_of_untyped and (getattr(kind, "textual", False) or (dt is None and getattr(kind, "kind", None) in ("map", "list")))
299
+ if textual:
300
+ # R2RML 10.2: a value of a type with no natural datatype is a plain literal of the database's
301
+ # own text (a struct, a map, an array, jsonb, inet, interval, money, an enum, xml ...).
302
+ return xsd.STRING, False, None, "CAST_TEXT", False
303
+ digits = getattr(kind, "digits", None) if dt == xsd.DECIMAL else None
304
+ uuid = getattr(kind, "uuid", False)
305
+ reader = ("UUID_TEXT_UPPER" if dialect == "sqlserver" else "UUID_TEXT") if uuid else \
306
+ "CHAR_TEXT" if getattr(kind, "padded", False) else \
307
+ "FLOAT32_VALUE" if getattr(kind, "single", False) else \
308
+ "BINARY_TEXT" if getattr(kind, "collated", False) else \
309
+ "NUMERIC_VALUE" if dialect == "postgresql" and dt == xsd.DECIMAL else None
310
+ return dt, bool(getattr(kind, "zoned", False)), digits, reader, bool(getattr(kind, "bytewise", False)) or uuid
311
+ held.kind_of = schema.table_column_type
312
+ return held
313
+
314
+
315
+ def node_keys(engine: Any, iri: str) -> List[Tuple[Any, Any, List[str]]]:
316
+ """(node map, the database's answer, its key columns' names in it) for each node map, the
317
+ answer at most one row whose subject is `iri` (`ProGraph.resolve_iri`, which
318
+ reads it): over the source maps, so R2RML and YAML alike, each map's own rows (its filter or
319
+ query kept), the subject compared as a term, the key's values as its columns hold them."""
320
+ from prograph.plan.ir import expr as E, lower, ops as O, sparql_front
321
+ from prograph.plan.ir.terms import IRITerm
322
+ from prograph.schema.source_map import column_value
323
+ schema = engine._schema
324
+ maps = source_maps(schema)
325
+ translator = sparql_front.Translator(maps, _held(schema, None, True), lambda m: (), True, None)
326
+ run_sql, dialect = _runner(getattr(getattr(engine, "executor", None), "graph", None))
327
+ found = []
328
+ for m in maps.nodes:
329
+ key = list(m.key.columns)
330
+ values = [column_value(c) for c in key]
331
+ try:
332
+ scan, names = translator.scan(m, [m.subject] + values)
333
+ except Exception: # noqa: BLE001 (a map this cannot read: no node of it)
334
+ continue
335
+ x: O.Op = O.Extend(scan, "#s", E.MakeTerm(m.subject, tuple((c, E.Col(names[c]))
336
+ for c in m.subject.columns())))
337
+ x = O.Filter(x, E.Compare("=", E.Col("#s"), E.Const(IRITerm(iri)), E.TERM))
338
+ out = []
339
+ for i, (c, tm) in enumerate(zip(key, values)):
340
+ x = O.Extend(x, f"k{i}", E.MakeTerm(tm, ((c, E.Col(names[c])),)))
341
+ out.append(f"k{i}")
342
+ plan = O.Slice(O.Project(x, tuple(out)), 0, 1)
343
+ try:
344
+ sql_text = lower.to_sql(plan, dialect, base_iri=maps.settings.base_iri)
345
+ except NotImplementedError:
346
+ continue
347
+ found.append((m, run_sql(sql_text), [lower.sql_name(n) for n in out]))
348
+ return found
349
+
350
+
351
+ def warn_written_constants(schema: Any) -> None:
352
+ """At mapping load: a map's constant integer or decimal literal not in
353
+ canonical form (`rr:constant "5"^^xsd:decimal`) is a term of its own,
354
+ which no column's value of that datatype is (ProGraph writes those in
355
+ canonical form, D3), so a join or pattern meeting both never matches
356
+ (m2-functions.md 5)."""
357
+ import warnings
358
+ from prograph.plan.ir.slot import _canonical_text
359
+ from prograph.plan.ir.sparql_front import PatternNeverMatchesWarning
360
+ from prograph.schema import xsd
361
+ try:
362
+ maps = source_maps(schema)
363
+ except Exception: # noqa: BLE001 - a mapping that does not compile says so when planned
364
+ return
365
+ for m in getattr(maps, "nodes", ()):
366
+ for po in getattr(m, "predicate_objects", ()):
367
+ t = po.object
368
+ if t.constant is None or t.datatype not in (xsd.DECIMAL, xsd.INTEGER) \
369
+ or _canonical_text(t.constant, t.datatype):
370
+ continue
371
+ warnings.warn(
372
+ f"The mapping's constant \"{t.constant}\"^^{t.datatype.replace(xsd.XSD, 'xsd:')} for "
373
+ f"<{po.predicate}> is not in canonical form: it is a term of its own, which a "
374
+ f"column's value of that datatype never is, so a join between them never matches; "
375
+ f"FILTER(=) compares values.", PatternNeverMatchesWarning, stacklevel=3)
376
+
377
+
378
+ def run(engine: Any, query: str, language: str, **kw: Any) -> Any:
379
+ """`_run` under the query's statement expectation (`execution.expect_long`); then, once it has answered, the
380
+ check of the stored OWL RL values its plan read (`_check_stored`)."""
381
+ from prograph.execution import statement_expectation
382
+ reads: List[Any] = []
383
+ token = _STORED_READS.set(reads)
384
+ try:
385
+ with statement_expectation():
386
+ out = _run(engine, query, language, **kw)
387
+ finally:
388
+ _STORED_READS.reset(token)
389
+ _check_stored(engine, reads)
390
+ return out
391
+
392
+
393
+ def expects_long(plan: Any) -> bool:
394
+ """Whether a plan is expected to run long: a property path's closure, or a shape the cost hints name
395
+ (`cost.SHAPES`: an unpinned join, an unseeded closure, the whole graph, a whole-IRI merge). Read off the
396
+ plan, no data: what decides how a statement is submitted on a SQL warehouse."""
397
+ from prograph.plan.ir import ops as O
398
+ from prograph.plan.ir.cost import SHAPES, cost_hints
399
+ stack, seen = [plan], set()
400
+ while stack:
401
+ op = stack.pop()
402
+ if id(op) in seen:
403
+ continue # a shared subtree is read once
404
+ seen.add(id(op))
405
+ if isinstance(op, O.Closure):
406
+ return True
407
+ stack.extend(op.children())
408
+ return any(h.shape in SHAPES for h in cost_hints(plan))
409
+
410
+
411
+ def _run(engine: Any, query: str, language: str, *, hints: Any = None,
412
+ parameters: Optional[dict] = None, streaming: bool = False, step_key: Optional[str] = None) -> Any:
413
+ """Answer `query`: front end, the plan, lowering to SQL, execution on
414
+ `engine`'s backend. Returns a pyarrow.Table of solutions (an ASK's: one
415
+ row, its column `boolean`), which `execute` converts to the asked
416
+ return type, or a CONSTRUCT's graph (an rdflib.Graph,
417
+ `construct.instantiate`). `query` is the text without its hint block, `hints` the
418
+ block's settings (`prograph.hints`)."""
419
+ graph = getattr(getattr(engine, "executor", None), "graph", None)
420
+ run_sql, dialect = _runner(graph)
421
+ from prograph.plan.ir import lower
422
+ # Spark: with no path_depth hint a closure walks first to the level form's depth; a
423
+ # longer walk raises in the statement, and it runs again to the default depth by the recursive form.
424
+ # Never a short answer. Which engines, and the depth: `lower.first_depth`, which to_sql shows too.
425
+ ladder = lower.closure_ladder(dialect, hints)
426
+ first = ladder[0] if ladder else None
427
+ retry = bool(ladder)
428
+ sql_text, names, translated, _pins, digits = _lowered(engine, query, language, dialect, hints,
429
+ parameters, full=True, depth=first,
430
+ step_key=step_key)
431
+ # A walk of a functional step (a parent pointer) skips the ladder's middle rungs (`closure_ladder`).
432
+ ladder = lower.closure_ladder(dialect, hints, translated.plan) if ladder else ladder
433
+ import time
434
+ from prograph.logging import get_plan_logger
435
+ # Each step below (a statement, a rung of the ladder, the RT form) starts only while the query is not
436
+ # cancelled (`execution.checkpoint`); a cancel between two never starts the next.
437
+ from prograph.execution import checkpoint, expect_long
438
+ checkpoint()
439
+ expect_long(expects_long(translated.plan))
440
+ # Streamed (return_type="batches"; decided 2026-11-11): a SELECT whose statement raises
441
+ # nowhere in the query, through the graph's own batch reader; a statement that may raise (a depth
442
+ # guard, a data error) is read whole, so no error follows rows already handed over.
443
+ # A write's answer, read in its step from the rows it wrote (`cypher_write.apply`); None for any other.
444
+ written = getattr(translated, "answer", None)
445
+ from prograph.plan.ir.cypher_mixed import holds_mixed, unwrap_table
446
+ # A column of values of several types answers as Arrow's dense union, rebuilt from the tagged
447
+ # struct (`cypher_mixed.unwrap_table`), so it is read whole, not streamed.
448
+ mixed_answer = lower.TAGGED_PROPERTIES.get() or any(holds_mixed(slot) for slot in translated.plan.slots.values())
449
+ if streaming and written is None and translated.form == "select" and names != ["__one"] \
450
+ and not mixed_answer and not lower.wrote_a_raise():
451
+ batches = _batch_runner(graph)
452
+ if batches is not None:
453
+ try:
454
+ opened = _with_digits(batches, digits)(sql_text, getattr(engine, "_batch_size", None) or 100_000)
455
+ except Exception as exc:
456
+ typed = _typed(exc, sql_text)
457
+ if typed is exc:
458
+ raise
459
+ raise typed from exc
460
+ reader = _renamed(opened, names, sql_text)
461
+ # The event every executed statement logs; its rows are not counted until they are read.
462
+ get_plan_logger().info("compositional_backend_executed", sql=sql_text, exec_time_ms=None,
463
+ result_count=None, sql_generator="one-plan", streamed=True)
464
+ return reader
465
+ started = time.time()
466
+ try:
467
+ try:
468
+ table = written if written is not None else _with_digits(run_sql, digits)(sql_text)
469
+ rung = first
470
+ except Exception as exc:
471
+ if not (retry and lower.depth_guard(str(exc))):
472
+ raise
473
+ # The ladder's next rungs (`lower.closure_ladder`), then the recursive form at the default
474
+ # depth: each runs only where the one before found a longer walk. Never a short answer.
475
+ table = None
476
+ for rung in tuple(ladder[1:]) + (None,):
477
+ checkpoint()
478
+ sql_text, names, translated, _pins, digits = _lowered(engine, query, language, dialect, hints,
479
+ parameters, full=True, costed=False,
480
+ depth=rung)
481
+ try:
482
+ table = _with_digits(run_sql, digits)(sql_text)
483
+ break
484
+ except Exception as again:
485
+ if rung is None or not lower.depth_guard(str(again)):
486
+ raise
487
+ except Exception as exc:
488
+ if lower.recursion_refused(str(exc)):
489
+ checkpoint()
490
+ # A warehouse that refuses WITH RECURSIVE (Lakehouse RT): the statement by levels to RT_LEVELS
491
+ # (or the path_depth hint, where smaller); a longer walk is a typed error naming the warehouse
492
+ # and path_depth. Never a short answer.
493
+ asked = getattr(hints, "path_depth", None) or lower.DEFAULT_PATH_DEPTH
494
+ depth = min(asked, lower.RT_LEVELS)
495
+ limit = lower.LEVEL_LIMIT.set(depth)
496
+ try:
497
+ sql_text, names, translated, _pins, digits = _lowered(engine, query, language, dialect, hints,
498
+ parameters, full=True, costed=False,
499
+ depth=depth)
500
+ finally:
501
+ lower.LEVEL_LIMIT.reset(limit)
502
+ rung = depth
503
+ try:
504
+ table = _with_digits(run_sql, digits)(sql_text)
505
+ except Exception as deeper:
506
+ if lower.depth_guard(str(deeper)):
507
+ from prograph.utils.exceptions import SQLRuntimeError
508
+ raise SQLRuntimeError(
509
+ f"A property path walks past {depth} steps, and this warehouse (a Databricks Lakehouse RT "
510
+ f"warehouse) refuses the recursive SQL that walks further: ProGraph walks it by levels to "
511
+ f"{lower.RT_LEVELS} steps there. Bound the path, or run the query on a SQL warehouse, "
512
+ f"where path_depth (up to 1000) sets the depth.", reason="path_too_deep",
513
+ sql=sql_text) from deeper
514
+ typed = _typed(deeper, sql_text)
515
+ if typed is deeper:
516
+ raise
517
+ if isinstance(typed, TwoValuesError):
518
+ typed = _two_values(engine, run_sql, dialect, typed)
519
+ raise typed from deeper
520
+ else:
521
+ typed = _typed(exc, sql_text)
522
+ if typed is exc:
523
+ raise
524
+ if isinstance(typed, TwoValuesError):
525
+ typed = _two_values(engine, run_sql, dialect, typed)
526
+ raise typed from exc
527
+ # The event every executed statement logs (`prograph.plan.run_record`).
528
+ get_plan_logger().info("compositional_backend_executed", sql=sql_text,
529
+ exec_time_ms=round((time.time() - started) * 1000, 2),
530
+ result_count=table.num_rows, sql_generator="one-plan",
531
+ closure_rung=(rung if rung is not None else "recursive") if retry else None)
532
+ # The answer's columns by the query's own names (`lower.sql_name`); a
533
+ # solution of no variable is a row of no column.
534
+ table = table.drop_columns(["__one"]) if names == ["__one"] else table.rename_columns(names)
535
+ if mixed_answer:
536
+ table = unwrap_table(table)
537
+ if translated.form == "construct":
538
+ from prograph.plan.ir.construct import instantiate
539
+ from prograph.plan.ir.sparql_front import HIDDEN
540
+ return instantiate(translated.template, table.to_pylist(), translated.prefixes, HIDDEN)
541
+ return table
542
+
543
+
544
+
545
+ def _written(translated: Any, dialect: str, base: Optional[str], terms_all: bool) -> str:
546
+ """The statement for a plan (`answer_sql`); a write's own answer is lowered in its step (`cypher_write.apply`)."""
547
+ return answer_sql(translated.plan, dialect, base, terms_all or translated.form == "construct",
548
+ parts=translated.form == "select" and getattr(translated, "write", None) is None)
549
+
550
+
551
+ def answer_sql(plan: Any, dialect: str, base: Optional[str], terms_all: bool = False, parts: bool = True) -> str:
552
+ """The statement of an answer: a read's, or a write's RETURN read in its step. On PostgreSQL it writes a whole
553
+ node, relationship, path or map, and a list of them, as flat typed columns the reader makes the Arrow struct of
554
+ (rule cypher-result-values). Where a step cannot write them and no answer column holds one (a quantifier over a
555
+ trail's nodes, a count), the statement is written without them, as it always was; where an answer column holds
556
+ one, the refusal stands."""
557
+ from prograph.plan.ir import lower
558
+ from prograph.plan.ir.cypher_answer import PartsRefused, answers_elements
559
+ try:
560
+ return lower.to_sql(plan, dialect, base_iri=base, terms_all=terms_all, split_terms=True, element_parts=parts)
561
+ except PartsRefused:
562
+ if not parts or any(answers_elements(plan.slots[c]) for c in plan.columns if c in plan.slots):
563
+ raise
564
+ return lower.to_sql(plan, dialect, base_iri=base, terms_all=terms_all, split_terms=True, element_parts=False)
565
+
566
+
567
+ def _typed(exc: Exception, sql_text: str) -> Exception:
568
+ """The database refused the statement (an overflow past D7's 38 digits, a limit): ProGraph's typed
569
+ error, to raise with the database's as its cause. ProGraph's own errors pass as they are."""
570
+ from prograph.execution import INTERRUPTS
571
+ if type(exc).__module__.split(".")[0] == "prograph" or isinstance(exc, INTERRUPTS):
572
+ # A cancel or the server's timeout (prograph.execution) is no refusal: it passes as it is.
573
+ return exc
574
+ from prograph.plan.ir.cypher_write import TWO_TYPES
575
+ if TWO_TYPES in str(exc):
576
+ # A read after a query's writes changed a property's type, where an element still holds the old one
577
+ # (`store_view._value`): named, never the database's raw error.
578
+ from prograph.plan.ir.facts import NotBuilt
579
+ return NotBuilt("a read of a property after this query's writes changed its type, where some element "
580
+ "still holds the old type (a column holds one type): give every element one type, or "
581
+ "read in a query of its own")
582
+ from prograph.plan.ir.lower import DATA_ERROR_MARK, TWO_VALUES, TWO_VALUES_SEP
583
+ if TWO_VALUES + TWO_VALUES_SEP in str(exc) and not _analysis_error(exc):
584
+ # TWO_VALUES ␟ label ␟ property ␟ one ␟ other ␟: the closing separator ends the payload, so what an
585
+ # engine wraps it in (PostgreSQL's quote, a newline and context) is not read as part of a value.
586
+ parts = str(exc)[str(exc).index(TWO_VALUES):].split(TWO_VALUES_SEP)[1:]
587
+ if len(parts) >= 5:
588
+ return TwoValuesError(*parts[:4])
589
+ if DATA_ERROR_MARK in str(exc).split("Query was:")[0] and not _analysis_error(exc):
590
+ # A row whose term R2RML cannot make (R2RML 11), raised by the statement; not the statement's
591
+ # own text, which a driver quotes after "Query was:" or an engine in an analysis error.
592
+ from prograph.r2rml.compile import R2RMLDataError
593
+ at = str(exc).index(DATA_ERROR_MARK)
594
+ message = str(exc)[at:].split("\n")[0]
595
+ if str(exc)[at - 1:at] == '"' and message.endswith('"'):
596
+ message = message[:-1] # PostgreSQL's quotes around the text its cast refused
597
+ return R2RMLDataError(message)
598
+ from prograph.plan.ir.cypher_runtime_errors import RUNTIME_ERROR_MARK
599
+ if RUNTIME_ERROR_MARK in str(exc).split("Query was:")[0] and not _analysis_error(exc):
600
+ # A Cypher TypeError a row raised (rule cypher-runtime-errors): its text from the mark, the first line,
601
+ # without the quote PostgreSQL puts around it.
602
+ from prograph.utils.exceptions import CypherExecutionError
603
+ message = str(exc)[str(exc).index(RUNTIME_ERROR_MARK) + len(RUNTIME_ERROR_MARK):].split("\n")[0]
604
+ return CypherExecutionError(message.rstrip('"'))
605
+ from prograph.utils.exceptions import SQLRuntimeError
606
+ return SQLRuntimeError(f"The database refused the statement: {exc}", reason="execution_error",
607
+ sql=sql_text)
608
+
609
+ def _analysis_error(exc: Exception) -> bool:
610
+ """Whether the engine refused the statement before running it (Spark's analysis errors, a syntax
611
+ error): its message quotes the statement's text, the guard's raise among it, which is no raise."""
612
+ text = str(exc)
613
+ return "AnalysisException" in type(exc).__name__ or any(
614
+ mark in text for mark in ("DATATYPE_MISMATCH", "cannot resolve", "UNRESOLVED_", "syntax error at or near"))
615
+
616
+
617
+ class TwoValuesError(ValueError):
618
+ """A node whose rows give one property two values, worded for a property-graph
619
+ user: its label, the property, both values and, where a diagnostic read finds them, the tables and keys
620
+ of the rows that hold them (rule cypher-node-scan). `detail` holds the node ids."""
621
+
622
+ def __init__(self, label: str, key: str, one: str, other: str, message: str = "", detail: str = ""):
623
+ self.label, self.key, self.one, self.other, self.detail = label, key, one, other, detail
624
+ super().__init__(message or f"{label or 'A'} node has two values of {key}: {one!r} and {other!r}")
625
+
626
+
627
+ def _two_values(engine: Any, run_sql: Any, dialect: str, err: "TwoValuesError") -> "TwoValuesError":
628
+ """The error worded from the rows that hold the two values: for each mapping of the label giving the
629
+ property, the rows whose property is one of the two (a small read, run only when the error fired); the
630
+ message names a node whose rows hold both, its table and key values ("Person id=7 has two values of
631
+ name: 'Ann' (people) and 'Anne' (staff)"). Any failure: the plain message."""
632
+ try:
633
+ from prograph.plan.sql.dialect import get_dialect_instance
634
+ from prograph.schema.lpg_view import view_of
635
+ d = get_dialect_instance(dialect)
636
+ quote = d.quote
637
+ view = view_of(source_maps(engine._schema))
638
+ found = [] # (node id, table, key text, value)
639
+
640
+ def literal(v: str) -> str:
641
+ """A value's text from the error as the dialect's string literal (`dialect.literal`), which the
642
+ engines read as the column's type (a number, a date; a text column keeps '007' as it is)."""
643
+ return d.literal(v)
644
+ for m in view.node_maps(None): # every mapping giving the property: the node's rows may be any
645
+ props = view.property_of(m, err.key)
646
+ if len(props) != 1 or props[0].object.column is None:
647
+ continue
648
+ col, keys = props[0].object.column, list(m.subject.columns())
649
+ sql = (f"SELECT {', '.join(quote(c) for c in keys)}, {quote(col)} FROM {m.source.qualified} "
650
+ f"WHERE {quote(col)} IN ({literal(err.one)}, {literal(err.other)})")
651
+ # The property's own digits (a NUMERIC(10,2) shows 1.50), where its reader needs them.
652
+ digits = (None,) * len(keys) + (_held(engine._schema)(m.source.table, col)[2],)
653
+ for row in _with_digits(run_sql, digits)(sql).to_pylist():
654
+ key = ", ".join(f"{c}={row[c]}" for c in keys)
655
+ # The node's id as its mapping writes it: a template's text, or a column of whole ids' value.
656
+ node = "".join(str(row[t]) if is_col else t for t, is_col in m.subject.template) \
657
+ if m.subject.template is not None else \
658
+ str(row[m.subject.column]) if m.subject.column is not None else ""
659
+ found.append((node or key, m.source.table, key, str(row[col])))
660
+ by_node: Dict[str, List[Tuple[str, str, str]]] = {}
661
+ for node, table, key, value in found:
662
+ by_node.setdefault(node, []).append((table, key, value))
663
+ for node, held in by_node.items():
664
+ values = {v for _, _, v in held}
665
+ if len(values) > 1:
666
+ key = held[0][1]
667
+ held = [(t, v) for t, _, v in held]
668
+ shown = " and ".join(f"{v!r} ({t})" for t, v in sorted(set(held), key=lambda x: x[1]))
669
+ return TwoValuesError(err.label, err.key, err.one, err.other,
670
+ f"{err.label or 'A node'} {key} has two values of {err.key}: {shown}",
671
+ detail=f"node id {node}")
672
+ except Exception: # noqa: BLE001 - the diagnostic is a courtesy; the plain error stands
673
+ pass
674
+ return err
675
+
676
+
677
+ def _with_digits(reader: Any, digits: Tuple[Any, ...]) -> Any:
678
+ """`reader` told the answer columns' SQL decimal digits (`lower.ANSWER_DIGITS`), where it reads the
679
+ database's decimals as text and so needs them (`reads_answer_digits`: PostgreSQL); else as it is."""
680
+ if getattr(getattr(reader, "__self__", None), "reads_answer_digits", False):
681
+ import functools
682
+ return functools.partial(reader, digits=digits)
683
+ return reader
684
+
685
+
686
+ def _batch_runner(graph: Any):
687
+ """The graph's streaming read (sql, batch_size) -> RecordBatchReader: a database's cursor kept open
688
+ until the reader is read (`_execute_sql_batches`), DuckDB over frames on a cursor of its own; None
689
+ where the graph has none (Spark: its answer is read whole and sliced)."""
690
+ from prograph.graph.sql_graph import SQLGraph
691
+ if isinstance(graph, SQLGraph) and hasattr(graph, "_execute_sql_batches"):
692
+ return graph._execute_sql_batches
693
+ if hasattr(graph, "enable_local_sql") and hasattr(graph, "execute_local_sql_batches"):
694
+ if getattr(graph, "_local_sql", None) is None:
695
+ graph.enable_local_sql()
696
+ return graph.execute_local_sql_batches
697
+ return None
698
+
699
+
700
+ def _renamed(reader: Any, names: List[str], sql_text: str = "") -> Any:
701
+ """`reader` under the answer's names (`lower.sql_name`'s inverse), schema only: each batch's
702
+ columns as they come, no cell touched. A database's error while the batches are read is
703
+ ProGraph's typed one, as a statement read whole gives it (`SQLRuntimeError`, the cause attached)."""
704
+ import pyarrow as pa
705
+ from prograph.utils.exceptions import SQLRuntimeError
706
+ schema = pa.schema([f.with_name(n) for f, n in zip(reader.schema, names)])
707
+
708
+ def batches():
709
+ try:
710
+ try:
711
+ for batch in reader:
712
+ yield pa.RecordBatch.from_arrays(batch.columns, schema=schema)
713
+ except Exception as exc:
714
+ from prograph.execution import INTERRUPTS
715
+ if type(exc).__module__.split(".")[0] == "prograph" or isinstance(exc, INTERRUPTS):
716
+ raise
717
+ raise SQLRuntimeError(f"The database refused the statement: {exc}",
718
+ reason="execution_error", sql=sql_text) from exc
719
+ finally:
720
+ # The driver's reader closes its cursor when it is closed: read through or abandoned, the
721
+ # statement does not outlive the stream (ADBC refuses to close a connection with one open).
722
+ reader.close()
723
+ return pa.RecordBatchReader.from_batches(schema, batches())
724
+
725
+
726
+ def _runner(graph: Any):
727
+ """How the engine's graph runs SQL, and in which dialect: a database's
728
+ own connection; DuckDB over frames, the pandas backend's included
729
+ (one-plan.md 2.7: `memory` and `pandas` run on DuckDB)."""
730
+ from prograph.graph.sql_graph import SQLGraph
731
+ if isinstance(graph, SQLGraph) and hasattr(graph, "_execute_sql_arrow"):
732
+ return graph._execute_sql_arrow, graph.sql_dialect()
733
+ if hasattr(graph, "enable_local_sql") and hasattr(graph, "execute_local_sql_arrow"):
734
+ if getattr(graph, "_local_sql", None) is None:
735
+ graph.enable_local_sql()
736
+ return graph.execute_local_sql_arrow, "duckdb"
737
+ if type(graph).__name__ == "SparkGraph":
738
+ def run_on_spark(sql_text: str):
739
+ # The frames are temporary views under their table names; a
740
+ # statement routed to a SQL warehouse comes back as rows.
741
+ import pyarrow as pa
742
+ from pyspark.sql import SparkSession
743
+ graph._ensure_temp_views()
744
+ session = next((f.sparkSession for f in graph.dataframes.values()
745
+ if getattr(f, "sparkSession", None) is not None),
746
+ None) or SparkSession.getActiveSession()
747
+ result = graph._execute_sql(sql_text, session)
748
+ if isinstance(result, pa.Table):
749
+ return result
750
+ from prograph.results import rows_to_arrow
751
+ return rows_to_arrow(list(result or []))
752
+ return run_on_spark, graph.sql_dialect()
753
+ raise NotImplementedError(f"one-plan: running SQL on {type(graph).__name__}")
754
+
755
+
756
+ def sql(engine: Any, query: str, language: str, dialect: Optional[str], *, hints: Any = None,
757
+ parameters: Optional[dict] = None) -> str:
758
+ """The SQL `run` runs for `query`, in `dialect` (None: the backend's),
759
+ lowered and serialized, not executed."""
760
+ if dialect is None:
761
+ dialect = _runner(getattr(getattr(engine, "executor", None), "graph", None))[1]
762
+ from prograph.plan.ir import lower
763
+ return _lowered(engine, query, language, dialect, hints, parameters,
764
+ depth=lower.first_depth(dialect, hints))[0]
765
+
766
+
767
+ def _lowered(engine: Any, query: str, language: str, dialect: str, hints: Any,
768
+ parameters: Optional[dict], full: bool = False, depth: Optional[int] = None,
769
+ costed: bool = True, step_key: Optional[str] = None):
770
+ """`_lowered_once`, planned again where the query's write step lost its race on Delta (`StepLostRace`: the
771
+ store changed after it was read, and a plan made on the old store may not see what the other writer wrote:
772
+ a MERGE of a label it created). Each time under the one step key; `Store.REDO_LIMIT` times, then
773
+ `StoreWriteConflictError`, nothing written."""
774
+ if full and costed and not step_key:
775
+ import uuid
776
+ step_key = uuid.uuid4().hex
777
+ from prograph.store import StepLostRace
778
+ tries = 0
779
+ while True:
780
+ try:
781
+ return _lowered_once(engine, query, language, dialect, hints, parameters, full, depth, costed, step_key)
782
+ except StepLostRace:
783
+ store = engine.__dict__.get("_store")
784
+ tries += 1
785
+ if tries >= getattr(store, "REDO_LIMIT", 20):
786
+ from prograph.utils.exceptions import StoreWriteConflictError
787
+ raise StoreWriteConflictError(
788
+ f"write contention: the write lost its race to other writers {tries} times (each redo plans the query again "
789
+ f"on the store as it is and runs the step whole); nothing was written. Call the query "
790
+ f"again, or write from fewer sessions at once") from None
791
+ import random
792
+ import time
793
+ time.sleep(random.uniform(0, 0.05 * tries))
794
+
795
+
796
+ def _lowered_once(engine: Any, query: str, language: str, dialect: str, hints: Any,
797
+ parameters: Optional[dict], full: bool = False, depth: Optional[int] = None,
798
+ costed: bool = True, step_key: Optional[str] = None):
799
+ """The SQL for `query` in `dialect`, and its answer's column names (and,
800
+ when `full`, the translated query). A CONSTRUCT's solutions are in full
801
+ terms: its template needs each term's kind. When `full` (the statement is to run) and `costed`,
802
+ its costly shapes are warned of and an opted-in one refused before any SQL runs;
803
+ `explain` and `to_sql` only show them."""
804
+ from prograph.plan.ir import semantics
805
+ with semantics.planning_under(semantics.of_query(engine, hints)):
806
+ return _lowered_under(engine, query, language, dialect, hints, parameters, full, depth, costed, step_key)
807
+
808
+
809
+ def _lowered_under(engine: Any, query: str, language: str, dialect: str, hints: Any,
810
+ parameters: Optional[dict], full: bool, depth: Optional[int], costed: bool,
811
+ step_key: Optional[str]):
812
+ """`_lowered_once` under the query's semantics profile."""
813
+ from prograph.plan.ir import lower
814
+ from prograph.plan.ir.keys import PLANNING_PINS
815
+ from prograph.plan.ir import key_checks as node_keys
816
+ # The keys checked while planning (keys: check) pin the tables they were checked at; the SQL
817
+ # reads them so (`key_checks.pinned_name` in the scans, `lower._pinned`).
818
+ pins = node_keys.Pins()
819
+ asked = PLANNING_PINS.set(pins)
820
+ if full and costed:
821
+ # A query that writes is one write step (Cypher's CREATE, one-plan 2.10): its key, the caller's or
822
+ # the run's; planned and run in the store's `writing` block (its reads after a write see the step's
823
+ # own rows, `Store.read_own`).
824
+ import uuid
825
+ engine.__dict__["_write_step"] = step_key or uuid.uuid4().hex
826
+ store = getattr(getattr(engine, "_schema", None), "__dict__", {}).get("owned_store") if full and costed else None
827
+ import contextlib
828
+ with store.writing() if store is not None else contextlib.nullcontext():
829
+ return _lowered_written(engine, query, language, dialect, hints, parameters, full, depth, costed, pins,
830
+ asked)
831
+
832
+
833
+ def _lowered_written(engine: Any, query: str, language: str, dialect: str, hints: Any,
834
+ parameters: Optional[dict], full: bool, depth: Optional[int], costed: bool, pins: Any,
835
+ asked: Any):
836
+ """`_lowered_once` from planning on."""
837
+ from prograph.plan.ir import lower
838
+ from prograph.plan.ir import key_checks as node_keys
839
+ from prograph.plan.ir.keys import PLANNING_PINS
840
+ try:
841
+ translated, terms_all, base = _planned(engine, query, language, hints, parameters)
842
+ finally:
843
+ PLANNING_PINS.reset(asked)
844
+ engine.__dict__.pop("_write_step", None)
845
+ if full and costed:
846
+ from prograph.plan.ir import cost
847
+ refuse = tuple(getattr(engine, "refuse_costly", ())) + (
848
+ (hints.refuse,) if getattr(hints, "refuse", None) else ())
849
+ cost.check(translated.plan, refuse)
850
+ pinned = node_keys._PINNED.set(pins or None)
851
+ depth = lower.PATH_DEPTH.set(depth or getattr(hints, "path_depth", None) or lower.DEFAULT_PATH_DEPTH)
852
+ from prograph.plan.ir import fresh
853
+ nonce = fresh.NONCE.set(fresh.new_nonce()) # one per lowering: each execution's values are new
854
+ try:
855
+ digits: Tuple[Any, ...] = ()
856
+ if translated.form == "ask":
857
+ sql_text, names = lower.to_sql_ask(translated.plan, dialect, base), ["boolean"]
858
+ else:
859
+ names = list(translated.plan.columns) or ["__one"]
860
+ sql_text = _written(translated, dialect, base, terms_all)
861
+ digits = lower.ANSWER_DIGITS.get()
862
+ write = getattr(translated, "write", None)
863
+ if full and costed and write is not None:
864
+ # A query that writes: its step appended once everything is lowered (a refused answer writes
865
+ # nothing), its rows read once, its answer read from them in the step's transaction (one that
866
+ # fails rolls it back) and kept for `run` (`cypher_write.apply`).
867
+ from prograph.plan.ir.cypher_write import apply
868
+ from prograph.store import Store
869
+ translated.answer = apply(engine, write, dialect, translated.plan if write.answers else None, base,
870
+ digest=Store.digest(query, parameters))
871
+ finally:
872
+ lower.PATH_DEPTH.reset(depth)
873
+ node_keys._PINNED.reset(pinned)
874
+ fresh.NONCE.reset(nonce)
875
+ return (sql_text, names, translated, pins, digits) if full else (sql_text, names)
876
+
877
+
878
+ def explain(engine: Any, query: str, language: str, dialect: Optional[str], *, hints: Any = None,
879
+ parameters: Optional[dict] = None) -> str:
880
+ """What `run` would do for `query`, in words: the SQL, each rewrite of
881
+ the plan with what it relied on (its certificate), and each key the plan
882
+ may rely on with what makes it trusted. An asserted key (`keys:
883
+ trusted`) is shown as asserted, never as checked (m2-performance.md
884
+ 2.4, decided 2026-11-09). Under profile='rl' it reports the stored facts' freshness check (how each table
885
+ was proven, its time; a refresh it would run, a refusal it would raise) and acts on none of it."""
886
+ if dialect is None:
887
+ dialect = _runner(getattr(getattr(engine, "executor", None), "graph", None))[1]
888
+ from prograph.plan.ir.dataset import EXPLAINING
889
+ explaining = EXPLAINING.set(True) # the freshness check reported, never acted on: nothing written or raised
890
+ try:
891
+ return _explained(engine, query, language, dialect, hints, parameters)
892
+ finally:
893
+ EXPLAINING.reset(explaining)
894
+
895
+
896
+ def _explained(engine: Any, query: str, language: str, dialect: Optional[str], hints: Any,
897
+ parameters: Optional[dict]) -> str:
898
+ from prograph.plan.ir import ops as O
899
+ from prograph.plan.ir.keys import key_sources
900
+ import warnings
901
+ with warnings.catch_warnings(record=True) as said:
902
+ warnings.simplefilter("always")
903
+ from prograph.plan.ir import lower
904
+ first = lower.first_depth(dialect, hints)
905
+ text = _lowered(engine, query, language, dialect, hints, parameters, depth=first)[0]
906
+ from prograph.plan.ir import semantics
907
+ profile = semantics.of_query(engine, hints)
908
+ with semantics.planning_under(profile):
909
+ translated, _, _ = _planned(engine, query, language, hints, parameters)
910
+ notes = list(dict.fromkeys(str(w.message) for w in said))
911
+ if first is not None and "RECURSIVE" not in text and lower.DEPTH_MARK in text:
912
+ rungs = " then ".join(str(d) for d in lower.closure_ladder(dialect, hints, translated.plan))
913
+ notes.append(f"A property path is walked by levels to {rungs} steps in turn, each run only where the "
914
+ f"one before found a longer walk, then by the recursive form to path_depth "
915
+ f"{lower.DEFAULT_PATH_DEPTH} (the query hint path_depth sets one depth); the run's log "
916
+ f"names the rung that answered (closure_rung).")
917
+ lines = [f"SQL ({dialect}):", text, ""]
918
+ lines += [f"Semantics: {profile.name} (comparisons: {profile.comparisons})", ""]
919
+ if notes:
920
+ lines += ["Notes:"] + [f" {n}" for n in notes] + [""]
921
+ lines += ["Rewrites:"]
922
+ lines += [f" {c.rule}: {c.relied_on}" for c in translated.certificates] or [" none"]
923
+ from prograph.plan.ir.cost import WARNED_BY_DEFAULT, cost_hints
924
+ lines += ["", "Cost hints (refuse_costly / the refuse hint refuses a shape; a note is never refused):"]
925
+ lines += [f" {h}{' (warned)' if h.shape in WARNED_BY_DEFAULT else ''}"
926
+ for h in cost_hints(translated.plan)] or [" none"]
927
+ graph = getattr(getattr(engine, "executor", None), "graph", None)
928
+ seen, keys = set(), []
929
+ stack = [translated.plan]
930
+ while stack:
931
+ op = stack.pop()
932
+ stack.extend(op.children())
933
+ if isinstance(op, O.MapScan) and op.unique_keys and op.map.id not in seen:
934
+ seen.add(op.map.id)
935
+ for columns, how in key_sources(graph, op.map, source_maps(engine._schema)):
936
+ keys.append(f" {op.map.id} ({op.map.source.table}): "
937
+ f"({', '.join(columns)}), {how}")
938
+ lines += ["", "Keys the plan may rely on:"] + (sorted(keys) or [" none"])
939
+ onto = ontology_of(engine)
940
+ if getattr(engine, "_profile", None) == "rl" and getattr(engine, "_store", None) is not None:
941
+ from prograph.plan.ir.dataset import engine_key, store_now
942
+ now = store_now(engine, onto)
943
+ if now is not None and now.current:
944
+ said = (f" the facts of the last run, as of {now.ran_at}, under key {engine_key(engine, onto)}: "
945
+ f"{now.how}")
946
+ else:
947
+ said = " none: the store holds no run under this engine's key"
948
+ lines += ["", "Stored OWL RL facts:", said]
949
+ held = getattr(engine, "__dict__", {}).get("_consistency")
950
+ if onto is not None and held is not None:
951
+ # When the consistency verdict was computed (`consistency`): under 'ql' when the engine was built, under 'rl'
952
+ # by which run, and whether this engine reused it.
953
+ lines += ["", "Consistency verdict:", f" {held.summary()}"]
954
+ proofs = getattr(engine, "__dict__", {}).get("_ql_verdict")
955
+ if proofs is not None:
956
+ # Under 'ql': how each table the checks read was proven for this query (`consistency.recheck`).
957
+ for name, how in sorted(proofs.get("how", {}).items()):
958
+ lines.append(f" {name}: {how}")
959
+ if onto is not None:
960
+ # What the ontology added to the maps the plan reads, and from which axiom.
961
+ from prograph.plan.ir.saturate import origins
962
+ scanned, stack = set(), [translated.plan]
963
+ while stack:
964
+ op = stack.pop()
965
+ stack.extend(op.children())
966
+ if isinstance(op, O.MapScan):
967
+ scanned.add(op.map.id.split(" via ")[0])
968
+ notes = origins(source_maps(engine._schema), onto, scanned)
969
+ lines += ["", "Ontology (T-mappings):"] + ([f" {n}" for n in notes] or [" nothing the plan reads"])
970
+ return "\n".join(lines)
971
+
972
+
973
+ def check(engine: Any, query: str, language: str = "sparql") -> None:
974
+ """Read `query` as `execute` would, up to its plan, and run nothing
975
+ (`engine` None: over an empty mapping, for the query alone):
976
+ a syntax error or a static error of the query (SPARQL 1.1 19.8, 18.2.1)
977
+ raises QueryError; an update is refused as read-only; what the plan does
978
+ not build raises NotImplementedError('one-plan: ...')."""
979
+ from prograph.hints import parse_hints
980
+ text, hints = parse_hints(query)
981
+ if language == "sparql":
982
+ from prograph.plan.ir.sparql_front import query_form
983
+ if query_form(text) == "update":
984
+ from prograph.read_only import refuse
985
+ refuse("a SPARQL Update")
986
+ if engine is None:
987
+ from types import SimpleNamespace
988
+ from prograph.schema import GraphSchema
989
+ engine = SimpleNamespace(_schema=GraphSchema())
990
+ from prograph.plan.ir import semantics
991
+ with semantics.planning_under(semantics.of_query(engine, hints)):
992
+ _planned(engine, text, language, hints, None)