prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/engine.py ADDED
@@ -0,0 +1,2718 @@
1
+ """ProGraph: SPARQL (and Cypher) over existing tables, as SQL.
2
+
3
+ A query is translated into one relational plan over the mapping's source
4
+ maps, lowered to SQL and run by the backend's database (DuckDB over
5
+ DataFrames, Spark, or a SQL connection); see docs/architecture/one-plan.md.
6
+ """
7
+
8
+ from typing import Any, Dict, List, Literal, Optional, Sequence, Set, Tuple, Union, TYPE_CHECKING
9
+ from dataclasses import dataclass
10
+ import logging
11
+ import re
12
+ import threading
13
+ import time
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+
18
+
19
+ def _one_at_a_time(method):
20
+ """Run an engine method under the engine's lock (see _execute_lock):
21
+ planning uses shared, stateful plan builders and schema changes alter
22
+ what a running query reads."""
23
+ import functools
24
+
25
+ @functools.wraps(method)
26
+ def locked(self, *args, **kwargs):
27
+ lock = getattr(self, "_execute_lock", None)
28
+ if lock is None: # during construction
29
+ return method(self, *args, **kwargs)
30
+ with lock:
31
+ return method(self, *args, **kwargs)
32
+ return locked
33
+
34
+ from prograph.executors.base import BaseExecutor
35
+ from prograph.hints import parse_hints, QueryHints
36
+ from prograph import read_only
37
+ from prograph.rules import home
38
+ from prograph.schema import GraphSchema, NodeMapping, RelationshipMapping, SchemaChangeSet
39
+
40
+ if TYPE_CHECKING:
41
+ from pathlib import Path
42
+ import rdflib
43
+ from prograph.ontology import OntologyLayer
44
+ from prograph.execution import QueryContext
45
+ from prograph.shacl import ValidationReport
46
+ from prograph.plan.ir.semantics import Semantics
47
+
48
+
49
+ BackendType = Literal["spark", "databricks_sql", "duckdb", "mysql", "postgresql", "sqlserver", "oracle"]
50
+ LanguageType = Literal["auto", "cypher", "sparql"]
51
+ _UNSUPPORTED_LANGUAGES = {"gremlin": "Gremlin", "sqlpgq": "SQL/PGQ", "gql": "GQL"}
52
+ ReturnType = Literal["arrow", "batches", "list", "dataframe"]
53
+
54
+ # Backends that carry schema-mapped DataFrames the OWL materializer can
55
+ # write `_inferred` rows into.
56
+ def _require_duckdb() -> None:
57
+ """DuckDB is the default engine for DataFrames; say how to get it."""
58
+ try:
59
+ import duckdb # noqa: F401
60
+ except ImportError as exc:
61
+ raise ImportError(
62
+ "Querying DataFrames runs SQL through DuckDB by default: "
63
+ "pip install 'prograph[duckdb]'."
64
+ ) from exc
65
+
66
+
67
+ def _is_a_duckdb_connection(connection: Any) -> bool:
68
+ try:
69
+ import duckdb
70
+ except ImportError:
71
+ return False
72
+ return isinstance(connection, duckdb.DuckDBPyConnection)
73
+
74
+
75
+ def _backend_when_none_is_named(dataframes: Any, spark: Any, connection: Any) -> str:
76
+ """The backend ProGraph() uses when its caller names none.
77
+
78
+ A SparkSession means Spark (over the DataFrames given
79
+ beside it, if any); DataFrames alone mean DuckDB over them, SQL first; a
80
+ DuckDB connection means that database, SQL only, like a warehouse. A
81
+ connection of another kind, or a connection beside DataFrames or a
82
+ SparkSession, does not say which engine reads what, and the caller names
83
+ the backend. Nothing given (a schema alone, or not even that) is an
84
+ embedded DuckDB with no table: a query over the empty dataset (VALUES,
85
+ BIND) answers, and a triple pattern matches nothing.
86
+ """
87
+ if connection is not None:
88
+ if dataframes is not None or spark is not None:
89
+ given = "spark=" if spark is not None else "dataframes="
90
+ raise ValueError(
91
+ f"ProGraph was given both connection= and {given} with no backend "
92
+ f"named, and reads one source of data: name the backend "
93
+ f"(backend='duckdb' with connection= alone runs SQL in that "
94
+ f"database; dataframes= alone runs SQL through an embedded DuckDB "
95
+ f"over them)."
96
+ )
97
+ if _is_a_duckdb_connection(connection):
98
+ return "duckdb"
99
+ raise ValueError(
100
+ f"ProGraph was given a connection of type {type(connection).__name__!r} "
101
+ f"with no backend named: name the database's backend "
102
+ f"('databricks_sql', 'postgresql', 'mysql', 'sqlserver' or 'oracle'). "
103
+ f"Only a DuckDB connection needs no backend name."
104
+ )
105
+ if spark is not None:
106
+ return "spark"
107
+ if dataframes is not None:
108
+ return "duckdb"
109
+ return "duckdb"
110
+
111
+
112
+ @home("yaml-declared-table", "Where a YAML source's declared table is read",
113
+ terms=("sources.table", "declared table", "to_schema", "by_table", "table_qualifications", "source name"))
114
+ def apply_declared_tables(schema: Any, dataframes: Optional[Dict[str, Any]]) -> None:
115
+ """A YAML schema's declared tables (`GraphSchema.declared_tables`, `sources.<name>.table`) as the
116
+ engine's table qualifications: each source name read at its declared table (an unqualified one under the
117
+ default catalog and schema, as the source's name was), except a name the engine is given a frame of (the
118
+ frame is that source's rows) or one the user qualified (`table_qualifications`). The schema is shared by
119
+ the engines built over it, so a qualification a declared table gave an earlier engine is taken back for
120
+ an engine given that frame (`_declared_applied` marks them; the user's own are never touched)."""
121
+ declared = getattr(schema, "declared_tables", None) or {}
122
+ if not declared:
123
+ return
124
+ frames = set(dataframes or ())
125
+ held = schema.table_qualifications
126
+ applied = schema.__dict__.setdefault("_declared_applied", {})
127
+ before = dict(held)
128
+ for name, table in declared.items():
129
+ mine = name in applied and held.get(name) == applied[name]
130
+ if name in frames:
131
+ if mine:
132
+ del held[name]
133
+ del applied[name]
134
+ continue
135
+ if name in held and not mine:
136
+ continue # the user's own qualification decides
137
+ if "." not in table and schema.default_catalog and schema.default_schema:
138
+ table = f"{schema.default_catalog}.{schema.default_schema}.{table}"
139
+ elif "." not in table and schema.default_schema:
140
+ table = f"{schema.default_schema}.{table}"
141
+ held[name] = applied[name] = table
142
+ if held != before:
143
+ schema._schema_changed() # a new version only where a qualification changed (the maps recompile)
144
+
145
+
146
+ def _shape_datatype(kind):
147
+ """A column's `sh:datatype`: its type's datatype; xsd:string for one with no natural datatype (a map,
148
+ a list, a text-read type: rule `no-natural-datatype-text`)."""
149
+ from prograph.schema import xsd
150
+ return xsd.datatype_of_type(kind) or (xsd.STRING if getattr(kind, "kind", None) in ("map", "list") else None)
151
+
152
+
153
+ class ProGraph:
154
+ """Main entry point for executing graph queries.
155
+
156
+ ProGraph provides a high-level interface for parsing and executing
157
+ graph queries against various backends. Supports multiple graph
158
+ query languages with automatic language detection.
159
+
160
+ Threads: an engine is safe to share, and runs one query at a time
161
+ (execute and schema updates take the engine's lock). The database still parallelises each query. For
162
+ queries running side by side, give each thread or worker its own
163
+ engine; on a SQL warehouse, each with its own connection.
164
+
165
+ Backends (the full list is in docs/guides/backends.md):
166
+ - 'pandas': pandas DataFrames, run as SQL on an embedded DuckDB.
167
+ - 'spark': Spark DataFrames, run as Spark SQL.
168
+ Every backend is read-only over the tables it maps (prograph/read_only.py), but for ``set_properties`` on a node
169
+ mapping marked ``mutable``.
170
+ - 'duckdb': The default when dataframes= is given with no backend (an
171
+ embedded DuckDB over the frames), and when connection= is a DuckDB connection (that
172
+ database, SQL only). See _backend_when_none_is_named.
173
+ - 'databricks_sql', 'postgresql', 'mysql', 'sqlserver', 'oracle': a SQL
174
+ database, reached through connection=.
175
+
176
+ Example (Pandas backend):
177
+ >>> import pandas as pd
178
+ >>> from prograph.schema import GraphSchema, NodeMapping
179
+ >>>
180
+ >>> movies_df = pd.DataFrame({'id': [1], 'title': ['The Matrix']})
181
+ >>> schema = GraphSchema()
182
+ >>> schema.add_node_mapping(NodeMapping(
183
+ ... source_table='movies', label='Movie',
184
+ ... id_column='id', property_columns={'title': 'title'}
185
+ ... ))
186
+ >>>
187
+ >>> engine = ProGraph(
188
+ ... schema=schema,
189
+ ... dataframes={'movies': movies_df}
190
+ ... )
191
+ >>> results = engine.execute("MATCH (m:Movie) RETURN m.title")
192
+
193
+ Example (Spark backend):
194
+ >>> from pyspark.sql import SparkSession
195
+ >>> from prograph.schema import GraphSchema, NodeMapping
196
+ >>>
197
+ >>> spark = SparkSession.builder.getOrCreate()
198
+ >>> movies_df = spark.createDataFrame([(1, 'The Matrix')], ['id', 'title'])
199
+ >>> schema = GraphSchema()
200
+ >>> schema.add_node_mapping(NodeMapping(
201
+ ... source_table='movies', label='Movie',
202
+ ... id_column='id', property_columns={'title': 'title'}
203
+ ... ))
204
+ >>>
205
+ >>> engine = ProGraph(
206
+ ... backend='spark',
207
+ ... schema=schema,
208
+ ... dataframes={'movies': movies_df}
209
+ ... )
210
+ >>> results = engine.execute("MATCH (m:Movie) RETURN m.title")
211
+ """
212
+
213
+ def __init__(
214
+ self,
215
+ backend: Optional[BackendType] = None,
216
+ schema: Optional[GraphSchema] = None,
217
+ dataframes: Optional[Dict[str, Any]] = None,
218
+ spark: Optional[Any] = None,
219
+ default_language: LanguageType = "auto",
220
+ default_return_type: ReturnType = "arrow",
221
+ ontology: Optional['OntologyLayer'] = None,
222
+ profile: Optional[str] = None,
223
+ default_shapes: Optional[str] = None,
224
+ sql_warehouse_id: Optional[str] = None,
225
+ warehouse_timeout: Optional[float] = None,
226
+ default_catalog: Optional[str] = None,
227
+ default_schema: Optional[str] = None,
228
+ table_qualifications: Optional[Dict[str, str]] = None,
229
+ connection: Optional[Any] = None,
230
+ default_graph: str = "unnamed",
231
+ ansi_session: bool = True,
232
+ refuse_costly: Sequence[str] = (),
233
+ store: Optional[Any] = None,
234
+ semantics: Optional[Union[str, 'Semantics']] = None,
235
+ sameas: str = "all",
236
+ implied_individuals: bool = False,
237
+ datatypes: Optional[str] = None,
238
+ freshness: Optional[str] = None,
239
+ version_columns: Optional[Dict[str, str]] = None,
240
+ inconsistency: str = "error",
241
+ ):
242
+ """Initialize the ProGraph engine.
243
+
244
+ Args:
245
+ backend: The execution backend to use. When omitted: 'spark'
246
+ if a SparkSession is given, 'duckdb' if dataframes are or
247
+ the connection is a DuckDB connection; with none of
248
+ them, ValueError. A connection of another kind, or one beside dataframes or a
249
+ SparkSession, raises ValueError: name the backend.
250
+ - 'duckdb': with dataframes, SQL through an embedded DuckDB
251
+ over them (the default local engine); with a connection,
252
+ that DuckDB database
253
+ - 'pandas': pandas DataFrames, on an embedded DuckDB
254
+ - 'spark': Spark DataFrames
255
+ - 'databricks_sql', 'postgresql', ...: a SQL database, SQL only
256
+ (requires schema and connection)
257
+ schema: GraphSchema defining node/relationship mappings
258
+ (required with dataframes or spark)
259
+ dataframes: Dictionary mapping table names to DataFrames.
260
+ If not provided but spark and schema are, tables are
261
+ loaded automatically from Unity Catalog using schema mappings.
262
+ spark: Optional SparkSession. If provided with schema but without
263
+ dataframes, tables are loaded automatically from Unity Catalog.
264
+ default_language: Default query language for execute() calls.
265
+ - 'auto': Detect language from query syntax (default)
266
+ - 'cypher': Parse all queries as Cypher
267
+ - 'sparql': Parse all queries as SPARQL
268
+ Can be overridden per-call via the language parameter.
269
+ default_return_type: Default return type for execute() calls.
270
+ - 'arrow': a pyarrow.Table (default): what the database
271
+ returned, with no per-row Python on the SQL path
272
+ - 'list': List[Dict], the table's to_pylist()
273
+ - 'dataframe': Return native DataFrame (Spark→Spark DF, else pandas)
274
+ Can be overridden per-call via the return_type parameter.
275
+ ontology: Optional OntologyLayer. Passing one turns reasoning
276
+ on: its triples are in the default graph and the RDFS core
277
+ (subClassOf, subPropertyOf, domain, range) is always
278
+ honoured, under ``profile``. None: a query matches simply.
279
+ profile: The OWL profile the ontology is reasoned under (needs
280
+ ``ontology=``; passing it without one raises ValueError):
281
+ ``'ql'`` (default with an ontology) answers by rewriting
282
+ alone, always current, nothing stored; ``'rl'`` also
283
+ materializes OWL 2 RL's facts into the owned store when the
284
+ engine is built (needs ``store=``), under the stored facts'
285
+ cache key, and refreshes them where the tables changed
286
+ (``freshness``). Any other value raises ValueError.
287
+ sql_warehouse_id: Optional Databricks SQL Warehouse ID for optimized
288
+ SQL execution. When provided (Spark backend only), generated
289
+ queries are routed to the SQL Warehouse for faster execution
290
+ via Photon. Falls back to Spark compute if unavailable.
291
+ warehouse_timeout: Optional timeout in seconds for SQL Warehouse
292
+ queries. When set, queries that exceed this time are cancelled
293
+ on the warehouse and raise TimeoutError. None (default) means
294
+ poll indefinitely until completion.
295
+ default_catalog: Optional Unity Catalog name for qualifying table
296
+ references. Sets schema.default_catalog for SQL generation.
297
+ default_schema: Optional Unity Catalog schema name. Sets
298
+ schema.default_schema for SQL generation.
299
+ table_qualifications: Optional explicit mapping of short table names
300
+ to fully qualified names. Takes precedence over default_catalog/
301
+ default_schema. Example: {"accounts": "bg.demo.my_accounts"}
302
+ default_graph: SPARQL's default graph, what a pattern outside
303
+ GRAPH reads. 'unnamed' (default, the standard: SPARQL 13,
304
+ R2RML 11): only the triples with no named graph (no graph_iri,
305
+ no graph map but rr:defaultGraph), so a named graph is read
306
+ through GRAPH alone. 'union': every triple, named graphs
307
+ included (Jena's unionDefaultGraph practice).
308
+ ansi_session: On Spark and Databricks, run the engine's SQL
309
+ with ANSI mode on, so that an integer overflow is an
310
+ error and not a wrapped number (default True). The session
311
+ or connection is the caller's: on a classic SparkSession
312
+ each statement runs in a clone and the session's own
313
+ setting is never changed; on Spark Connect the setting is
314
+ changed for the length of each statement; on a Databricks
315
+ SQL connection it is set once and put back by
316
+ :meth:`close`. On a classic SparkSession whose parser
317
+ drops :meth:`execute_sql`'s parameters (Delta's, on Spark
318
+ 4.1) they are bound by Spark's analyzer for that statement.
319
+ False leaves every setting as it is. No effect on the
320
+ other backends.
321
+ store: An owned store (`prograph.store.Store`) on the engine's
322
+ own connection: the statements ProGraph writes, read beside
323
+ the mapped tables (SPARQL through its generated mapping,
324
+ Cypher through its property-graph view) and written by
325
+ Cypher's writes. None: the engine writes nothing. On DuckDB
326
+ and PostgreSQL (the ADBC connection the engine reads on).
327
+ semantics: The semantics profile queries are planned under: ``"default"`` (None) reads
328
+ 17.3's operators plus XPath's for the other XSD types and the W3C optional features
329
+ KnownTypesDefault2Neq and LangTagAwareness; ``"core"`` is strict SPARQL 1.1 (only
330
+ 17.3's operators, every other pair of literals RDFterm-equal's type error, YEAR,
331
+ MONTH and DAY of an xsd:date an error). A name or a
332
+ :class:`~prograph.plan.ir.semantics.Semantics`; a query's ``/*+ semantics=core */``
333
+ hint wins. An unknown name raises ValueError. Not entailment: it chooses operator
334
+ behaviour only.
335
+ sameas: Under profile='rl', how a pattern answers an
336
+ entity owl:sameAs gives several IRIs: 'all' (default, the
337
+ entailed answer) one solution per IRI of the entity;
338
+ 'representative' one per entity, under its representative
339
+ (the least IRI by code point). FILTER, VALUES, BIND,
340
+ DISTINCT and GROUP BY compare RDF terms in both.
341
+ implied_individuals: Under profile='ql', whether a SPARQL query's blank
342
+ nodes and hidden variables (a variable not projected and read nowhere
343
+ outside its triples block) may be satisfied by individuals the ontology
344
+ implies but no data names (``B rdfs:subClassOf [owl:onProperty p;
345
+ owl:someValuesFrom C]``): the certain answers, by tree-witness
346
+ rewriting. False (default): the W3C reading, every term of a pattern a
347
+ named one. The answers with it on hold those with it off. Under
348
+ profile='rl' it raises ValueError (OWL 2 RL implies no unnamed
349
+ individual).
350
+ datatypes: How a SPARQL basic graph pattern matches literals: 'term'
351
+ (RDF term identity: `1` matches "1"^^xsd:integer only) or 'value'
352
+ (D-entailment and OWL 2 RL's dt-eq: a literal matches every literal
353
+ of the same value, so `1` also matches "1.00"^^xsd:decimal and
354
+ "1"^^xsd:short; xsd:float and xsd:double are value spaces of their
355
+ own). None (default): 'value' under profile='rl', else 'term'.
356
+ FILTER, sameTerm, STR, ORDER BY and BIND compare as before.
357
+ freshness: Under profile='ql', how the consistency verdict is kept
358
+ current: None (default) each table the checks read proven
359
+ unchanged per query as below, the checks run again where one
360
+ changed, and a table nothing proves keeps the build's verdict
361
+ with a ``ConsistencyVerdictWarning`` (once); ``'strict'``
362
+ refuses such a query (``ConsistencyVerdictNotCurrentError``);
363
+ ``'last_run'`` keeps the build's verdict, nothing read.
364
+ Under profile='rl', when a query reads the stored
365
+ facts. None (default): only where they are provably
366
+ current: every mapped table unchanged since the run, by its
367
+ declared version column (``version_columns``), its Delta
368
+ version, or, for a table under its backend's size
369
+ threshold (``freshness.SMALL``), a checksum of all its
370
+ rows, read once per query, and the owned store unchanged;
371
+ where one changed, the engine refreshes them first (as
372
+ ``materialize_entailments()``). A table none of them
373
+ proves is refused with ``StoredFactsNotCurrentError``, as
374
+ is a change the store cannot be written to refresh.
375
+ ``explain()`` shows how each table was proven, and the time.
376
+ ``'last_run'``: the facts of the last run, not checked;
377
+ ``explain()`` says when that run was and under which key.
378
+ version_columns: Under profile='rl' or 'ql', a version column per mapped
379
+ table (``{table: column}``, the table as the mapping names
380
+ it): a column your writes keep increasing (each insert and
381
+ update sets a value above every one before). Its maximum and
382
+ the row count, compared with the run's, prove the table
383
+ unchanged. A table no map reads raises ValueError.
384
+ inconsistency: What an ontology and data that are inconsistent
385
+ under the profile do (an individual of two disjoint
386
+ classes, an irreflexive property's loop, two values of a
387
+ functional property that are not one value ...).
388
+ ``'error'`` (default): ``InconsistentOntologyError``,
389
+ naming each axiom and a few of the facts, raised when the
390
+ engine is built or a materialization finds it.
391
+ ``'report'``: queries answer as usual and
392
+ :meth:`consistency_report` lists what matched. Checked under
393
+ profile='rl' once per materialization (its verdict stored
394
+ with the facts), under profile='ql' once when the engine is
395
+ built. Any other value raises ValueError.
396
+ """
397
+
398
+ if backend is None:
399
+ backend = _backend_when_none_is_named(dataframes, spark, connection)
400
+ if backend == "duckdb" and connection is None and dataframes is None:
401
+ # The empty dataset: no table, and a schema mapping none.
402
+ schema = schema if schema is not None else GraphSchema()
403
+ dataframes = {}
404
+ self.backend = backend
405
+ # True when queries run over in-memory frames (pandas, Spark, or
406
+ # DuckDB over DataFrames), which decides the W3C paths: reasoning
407
+ # writes `_inferred` rows back into frames rather than issuing
408
+ # INSERTs, which DuckDB's views over DataFrames would refuse. Set
409
+ # here for spark; the DuckDB-over-frames branch of the
410
+ # executor setup sets it True once it has built the frames' DuckDB.
411
+ self._frame_backed = backend == "spark"
412
+ # One query at a time per engine: a DuckDB connection is not safe
413
+ # across threads (four threads on one engine deadlocked).
414
+ self._execute_lock = threading.RLock()
415
+ self._connection = connection
416
+
417
+ # Set default catalog/schema/table_qualifications on the schema if provided
418
+ if schema is not None:
419
+ if default_catalog is not None:
420
+ schema.default_catalog = default_catalog
421
+ if default_schema is not None:
422
+ schema.default_schema = default_schema
423
+ if table_qualifications is not None:
424
+ schema.table_qualifications.update(table_qualifications)
425
+ apply_declared_tables(schema, dataframes)
426
+ self.default_language = default_language
427
+ self.default_return_type = default_return_type
428
+ self._schema = schema
429
+ self._spark = spark
430
+ self._store = store
431
+ if store is not None:
432
+ if schema is None:
433
+ raise ValueError("an owned store reads beside a schema: give the engine a schema too")
434
+ store.attach(schema)
435
+ if schema is not None:
436
+ # A schema whose node IRIs cannot name one node each is refused
437
+ # here, by name.
438
+ from prograph.schema.iri import check_identities, check_meetings
439
+ check_identities(schema)
440
+ check_meetings(schema)
441
+ # An asserted key is said so at load (m2-performance.md 2.4).
442
+ from prograph.plan.ir.keys import warn_asserted_keys
443
+ warn_asserted_keys(schema)
444
+ from prograph.plan.ir.entry import warn_written_constants
445
+ warn_written_constants(schema)
446
+
447
+ # Auto-load dataframes from Unity Catalog if spark provided without dataframes
448
+ if dataframes is None and spark is not None and schema is not None:
449
+ dataframes = self._load_dataframes_from_schema(spark, schema)
450
+ self._dataframes = dataframes
451
+ # An ontology saturates the source maps on the SPARQL path, over YAML and R2RML alike
452
+ # (prograph.plan.ir.saturate).
453
+ self._ontology = ontology
454
+ self._sql_warehouse_id = sql_warehouse_id
455
+ self._warehouse_timeout = warehouse_timeout
456
+ if ontology is not None and schema is not None and getattr(schema, "property_graph", True) is not False:
457
+ # Bind the ontology to the schema's labels (its class hierarchy is read by IRI); Cypher and SPARQL
458
+ # both read it as the saturated maps (`entry.ontology_of`).
459
+ ontology.bind_to_schema(schema, strict=False)
460
+
461
+ # The OWL profile (`dataset.engine_profile`), checked up front so misconfiguration surfaces at
462
+ # construction time rather than during materialization.
463
+ from prograph.plan.ir.dataset import engine_profile
464
+ profile = engine_profile(ontology, profile)
465
+ if profile == "rl" and store is None:
466
+ raise ValueError("profile='rl' materializes the ontology's OWL 2 RL entailments into the "
467
+ "owned store: give the engine a store= too")
468
+ self._profile = profile
469
+ if ontology is not None:
470
+ # The axioms the profile does not answer are refused here, by name, or dropped
471
+ # (on_unsupported="ignore"; `ontology_report`).
472
+ from prograph.plan.ir.entry import profile_report
473
+ profile_report(self)
474
+ # How the stored facts are trusted (`freshness`): checked before this engine's first run.
475
+ from prograph.plan.ir.freshness import engine_freshness
476
+ self._freshness = engine_freshness(freshness, profile)
477
+ if version_columns and profile not in ("rl", "ql"):
478
+ raise ValueError("version_columns= proves the stored OWL RL facts (profile='rl') or the consistency "
479
+ "verdict (profile='ql') current: build the engine with one of those profiles")
480
+ self._version_columns = dict(version_columns or {})
481
+ from prograph.plan.ir.consistency import engine_inconsistency
482
+ self._inconsistency = engine_inconsistency(inconsistency)
483
+ from prograph.plan.ir.identity import MODES as _SAMEAS_MODES
484
+ if sameas not in _SAMEAS_MODES:
485
+ raise ValueError(f"sameas is 'all' or 'representative', not {sameas!r}")
486
+ self._sameas = sameas
487
+ if implied_individuals and getattr(self, "_profile", None) == "rl":
488
+ raise ValueError("implied_individuals is a profile='ql' feature")
489
+ self._implied_individuals = bool(implied_individuals)
490
+ if datatypes not in (None, "term", "value"):
491
+ raise ValueError(f"datatypes is 'term' or 'value', not {datatypes!r}")
492
+ self._datatypes = datatypes or ("value" if getattr(self, "_profile", None) == "rl" else "term")
493
+ # Costly query shapes refused before they run (`prograph.plan.ir.cost`): opt-in.
494
+ from prograph.plan.ir.cost import SHAPES
495
+ unknown = set(refuse_costly) - set(SHAPES) - {"all"}
496
+ if unknown:
497
+ raise ValueError(f"refuse_costly takes {', '.join(SHAPES)} or all, not {sorted(unknown)}")
498
+ self.refuse_costly = tuple(refuse_costly)
499
+ from prograph.plan.ir.semantics import named
500
+ self.semantics = named(semantics)
501
+ # Operator-configured SHACL shapes for the runtime guardrail. When set,
502
+ # the MCP server exposes a zero-argument data-quality tool that agents'
503
+ # context passes auto-invoke, surfacing the conformance verdict into
504
+ # the agent's reasoning without it having to author shapes itself.
505
+ self._default_shapes = default_shapes
506
+
507
+ self._ansi_session = ansi_session
508
+ self.executor = self._create_executor(
509
+ backend, schema, dataframes, sql_warehouse_id,
510
+ warehouse_timeout,
511
+ connection=connection,
512
+ )
513
+ self._record_column_types()
514
+ if schema is not None:
515
+ schema.check_language_columns()
516
+ # keys: check on a source with no version is refused now, not at a first query.
517
+ from prograph.plan.ir.keys import refuse_unversioned_checks
518
+ refuse_unversioned_checks(self)
519
+
520
+ # Store config if created from YAML
521
+ self._config = None
522
+
523
+ # SPARQL's default graph (read by the plan's front end).
524
+ self.default_graph = default_graph
525
+ if profile == "rl":
526
+ # The ontology's kinds RL does not apply warn (or raise) now; the entailments are materialized
527
+ # once, where the store holds none for this engine's cache key (`dataset.cache_key`).
528
+ self._materialize_on_open()
529
+ elif profile == "ql":
530
+ # The negative axioms' checks, once (`consistency`): raised here under inconsistency='error'.
531
+ from prograph.plan.ir.consistency import check_on_open
532
+ check_on_open(self)
533
+
534
+ def close(self) -> None:
535
+ """Give back what the engine changed on a session it was handed:
536
+ a Databricks SQL connection's ANSI mode is put back as it was
537
+ (``ansi_session``). The connection and the SparkSession are the
538
+ caller's and stay open. An engine never closed never restores."""
539
+ graph = getattr(getattr(self, 'executor', None), 'graph', None)
540
+ close = getattr(graph, 'close', None)
541
+ if callable(close):
542
+ close()
543
+
544
+ def __enter__(self) -> 'ProGraph':
545
+ return self
546
+
547
+ def __exit__(self, exc_type, exc, tb) -> None:
548
+ self.close()
549
+
550
+ @property
551
+ def spark(self) -> Optional[Any]:
552
+ """The SparkSession this engine runs on, or None on another
553
+ backend: the one given at construction, else the session of the
554
+ DataFrames it was given."""
555
+ if self._spark is not None:
556
+ return self._spark
557
+ if self.backend != "spark":
558
+ return None
559
+ graph = getattr(self.executor, 'graph', None)
560
+ for frame in (getattr(graph, 'dataframes', None) or {}).values():
561
+ session = getattr(frame, 'sparkSession', None)
562
+ if session is not None:
563
+ return session
564
+ return None
565
+
566
+ def declare_procedure(self, name: str, inputs: Any, outputs: Any, rows: Any = None,
567
+ source: Optional[str] = None, columns: Optional[Dict[str, str]] = None) -> None:
568
+ """Declare a Cypher procedure as data (rule cypher-procedures).
569
+ `inputs` and `outputs` are (name, type) pairs. Give `rows` (literal, each the inputs' values then
570
+ the outputs') or `source`, a table or view this engine reads (its columns named as the inputs then
571
+ outputs, or `columns` maps an input or output name to its column); a source's rows are read when a
572
+ query calling the procedure is planned. A mapping's `procedures:` section declares the same."""
573
+ from prograph.plan.ir.cypher_procedures import declare
574
+ proc = declare(name, inputs, outputs, rows, source, columns)
575
+ if not hasattr(self, "_procedures"):
576
+ self._procedures = {}
577
+ self._procedures[proc.name] = proc
578
+
579
+ def _declare_config_procedures(self, config: Any) -> None:
580
+ for proc in config.procedures:
581
+ self.declare_procedure(proc.name, proc.inputs, proc.outputs, proc.rows if proc.source is None else None,
582
+ proc.source, dict(proc.columns))
583
+
584
+ def _read_procedures(self) -> Optional[Any]:
585
+ """The declared procedures for one query: a source's rows are read when a call looks it up."""
586
+ declared = getattr(self, "_procedures", None)
587
+ if not declared or all(p.source is None for p in declared.values()):
588
+ return declared
589
+ from prograph.plan.ir.cypher_procedures import Lazy
590
+ from prograph.plan.sql.dialect import get_dialect_instance
591
+ graph = getattr(self.executor, "graph", None)
592
+ dialect = get_dialect_instance((graph.sql_dialect() if hasattr(graph, "sql_dialect") else None)
593
+ or "spark")
594
+
595
+ def read(sql):
596
+ return [tuple(row.values()) for row in self.execute_sql(sql, return_type="list")]
597
+ return Lazy(declared, read, dialect)
598
+
599
+ def execute_sql(
600
+ self,
601
+ sql: str,
602
+ parameters: Optional[Dict[str, Any]] = None,
603
+ return_type: Optional[ReturnType] = None,
604
+ read_only: bool = False,
605
+ ) -> Any:
606
+ """Run SQL text on the backend's own connection, in its dialect,
607
+ and return the result as :meth:`execute` does.
608
+
609
+ For a layer above that needs the database the graph lives in: a
610
+ scoring query over the same tables, a table of its own. The SQL is
611
+ the backend's, not translated: Spark SQL on Spark, the database's
612
+ dialect on a SQL backend, DuckDB over DataFrames (each frame a
613
+ table under its name). The result is the database's own Arrow
614
+ result, converted to the return type and nothing else.
615
+ ``parameters`` are bound by the backend, in its own placeholder
616
+ syntax (``:name`` on Spark, ``$name`` on DuckDB), never put into
617
+ the text. On a Spark session whose parser drops them (Delta's, on
618
+ Spark 4.1) they are bound in Spark's analyzer, where a value goes
619
+ where a constant goes (``prograph/graph/ansi_session.py``).
620
+
621
+ ``read_only=True`` is for SQL the caller did not write (a policy
622
+ file's query): the text must be one query, and the backend itself
623
+ says so, not an inspection of the text. On DuckDB its own parser
624
+ must find exactly one statement, a SELECT; on Spark the text runs
625
+ as the subquery of one SELECT, and Spark takes one statement per
626
+ call. Text that is not one query raises ``NotAQueryError`` or the
627
+ backend's parse error, and nothing runs. The other SQL backends do
628
+ not enforce it yet and refuse the call.
629
+
630
+ Raises ``ValueError`` for ``parameters`` on a SQL backend other than DuckDB, whose
631
+ connection is given SQL text only, and for ``read_only`` on a
632
+ backend that cannot enforce it.
633
+ """
634
+ import pyarrow as pa
635
+
636
+ wanted = return_type or self.default_return_type
637
+ graph = getattr(self.executor, 'graph', None)
638
+
639
+ if self.backend == "spark":
640
+ session = self.spark
641
+ if session is None:
642
+ raise ValueError("execute_sql needs a SparkSession, and this engine has none")
643
+ if read_only:
644
+ # One statement per call is Spark's own rule, so as the
645
+ # subquery of this SELECT the text can only be a query. A
646
+ # trailing semicolon ends a query everywhere else, and is
647
+ # dropped here so it does not end the wrapper early.
648
+ sql = f"SELECT * FROM (\n{sql.rstrip().rstrip(';').rstrip()}\n) AS _read_only"
649
+ if parameters:
650
+ from prograph.graph.ansi_session import parameter_session
651
+ with parameter_session(session, self._ansi_session) as bound:
652
+ frame = bound.sql(sql, args=dict(parameters))
653
+ else:
654
+ frame = session.sql(sql)
655
+ return frame if wanted == "dataframe" else self._convert_results(frame.toArrow(), wanted)
656
+
657
+ local = getattr(graph, '_local_sql', None) is not None # DuckDB over DataFrames
658
+ if not local and getattr(graph, '_connection', None) is None:
659
+ raise ValueError(
660
+ f"execute_sql needs a SQL connection, and the {self.backend} backend has none")
661
+ if read_only:
662
+ if self.backend != "duckdb":
663
+ raise ValueError(
664
+ f"execute_sql(read_only=True) is enforced on DuckDB and Spark; the "
665
+ f"{self.backend} backend does not enforce it yet")
666
+ from prograph.writes import require_one_query
667
+ require_one_query(graph._local_sql if local else graph._connection, sql)
668
+ if not parameters:
669
+ table = graph.execute_local_sql_arrow(sql) if local else graph._execute_sql_arrow(sql)
670
+ elif self.backend == "duckdb":
671
+ from prograph.graph.drivers import duckdb_arrow_table
672
+ connection = graph._prepare_local_sql() if local else graph._connection
673
+ table = duckdb_arrow_table(connection, sql, parameters)
674
+ else:
675
+ raise ValueError(
676
+ f"execute_sql binds parameters on Spark and DuckDB; the {self.backend} "
677
+ f"backend's connection is given SQL text only")
678
+ return self._convert_results(table, wanted)
679
+
680
+ @property
681
+ def schema(self) -> Optional[GraphSchema]:
682
+ """The graph schema defining node and relationship mappings.
683
+
684
+ Returns the GraphSchema that maps tabular data sources to graph
685
+ concepts (nodes, relationships, predicates).
686
+
687
+ Returns:
688
+ The GraphSchema if one was provided, None otherwise.
689
+ """
690
+ return self._schema
691
+
692
+ def _load_dataframes_from_schema(
693
+ self,
694
+ spark: Any,
695
+ schema: GraphSchema
696
+ ) -> Dict[str, Any]:
697
+ """Load DataFrames from Unity Catalog using schema mappings.
698
+
699
+ Args:
700
+ spark: SparkSession to use for loading tables
701
+ schema: GraphSchema (or an R2RML mapping's) whose source maps name the tables
702
+
703
+ Returns:
704
+ Dictionary mapping table names to Spark DataFrames.
705
+ Keys are the original (possibly unqualified) names from mappings.
706
+ """
707
+ dataframes = {}
708
+ # The tables the source maps read (`entry.mapped_tables`), each by its name: a query
709
+ # source's rows are its query's.
710
+ from prograph.plan.ir.entry import mapped_tables
711
+ for source in mapped_tables(schema):
712
+ if source.table in dataframes:
713
+ continue
714
+ try:
715
+ # The name as Spark reads it (an R2RML delimited "Production"."Product" as
716
+ # `Production`.`Product`: rule source-table-names).
717
+ from prograph.plan.sql.dialect import get_dialect_instance
718
+ dataframes[source.table] = (spark.sql(source.query) if source.query
719
+ else spark.table(get_dialect_instance("spark").table_name(source.qualified)))
720
+ except Exception as e:
721
+ raise ValueError(f"Failed to load table '{source.qualified}' from Unity Catalog: {e}")
722
+
723
+ return dataframes
724
+
725
+
726
+ def _create_executor(
727
+ self,
728
+ backend: BackendType,
729
+ schema: Optional[GraphSchema],
730
+ dataframes: Optional[Dict[str, Any]],
731
+ sql_warehouse_id: Optional[str] = None,
732
+ warehouse_timeout: Optional[float] = None,
733
+ connection: Optional[Any] = None,
734
+ ) -> BaseExecutor:
735
+ """Create the appropriate executor for the backend."""
736
+ if backend == "spark":
737
+ if schema is None:
738
+ raise ValueError("Spark backend requires a schema parameter")
739
+ if dataframes is None:
740
+ raise ValueError(
741
+ "Spark backend requires either dataframes or spark parameter. "
742
+ "Provide dataframes dict directly, or pass spark to auto-load from Unity Catalog."
743
+ )
744
+ from prograph.executors.spark_executor import SparkExecutor
745
+ return SparkExecutor(
746
+ schema, dataframes,
747
+ sql_warehouse_id=sql_warehouse_id,
748
+ warehouse_timeout=warehouse_timeout,
749
+ ansi_session=self._ansi_session,
750
+ )
751
+ elif backend == "databricks_sql":
752
+ if schema is None:
753
+ raise ValueError("Databricks SQL backend requires a schema parameter")
754
+ if connection is None:
755
+ raise ValueError(
756
+ "Databricks SQL backend requires a connection parameter. "
757
+ "Pass a databricks.sql.connect() connection."
758
+ )
759
+ from prograph.executors.databricks_sql_executor import DatabricksSQLExecutor
760
+ return DatabricksSQLExecutor(
761
+ schema, connection,
762
+ catalog=self._schema.default_catalog if self._schema else None,
763
+ schema_name=self._schema.default_schema if self._schema else None,
764
+ ansi_session=self._ansi_session,
765
+ )
766
+ elif backend == "duckdb":
767
+ if schema is None:
768
+ raise ValueError("DuckDB backend requires a schema parameter")
769
+ frames = connection is None
770
+ if self._store is not None:
771
+ from prograph.store import engine_connection
772
+ # An owned store's connection is the one the frames are read on (rule `store-connection`).
773
+ shared = engine_connection(self._store, connection, frames)
774
+ if frames:
775
+ # DataFrames: SQL through an embedded DuckDB over the frames,
776
+ # the default local engine. Given a connection instead,
777
+ # DuckDB is a database like any warehouse.
778
+ # Neither: an engine over the empty dataset.
779
+ _require_duckdb()
780
+ from prograph.executors.pandas_executor import PandasExecutor
781
+ executor = PandasExecutor(schema, dataframes)
782
+ executor.graph.enable_local_sql(shared if self._store is not None else None)
783
+ self._connection = executor.graph._local_sql
784
+ self._frame_backed = True
785
+ return executor
786
+ from prograph.executors.duckdb_executor import DuckDBExecutor
787
+ return DuckDBExecutor(schema, connection)
788
+ elif backend == "mysql":
789
+ if schema is None:
790
+ raise ValueError("MySQL backend requires a schema parameter")
791
+ if connection is None:
792
+ raise ValueError(
793
+ "MySQL backend requires a connection parameter. "
794
+ "Pass a mysql.connector.connect() connection."
795
+ )
796
+ from prograph.executors.mysql_executor import MySQLExecutor
797
+ return MySQLExecutor(schema, connection)
798
+ elif backend == "postgresql":
799
+ if schema is None:
800
+ raise ValueError("PostgreSQL backend requires a schema parameter")
801
+ if connection is None:
802
+ raise ValueError("PostgreSQL backend requires a connection parameter")
803
+ from prograph.executors.postgresql_executor import PostgreSQLExecutor
804
+ executor = PostgreSQLExecutor(schema, connection)
805
+ # Given a URI, the connection is the one ProGraph opened.
806
+ self._connection = executor.graph._connection
807
+ return executor
808
+ elif backend == "sqlserver":
809
+ if schema is None:
810
+ raise ValueError("SQL Server backend requires a schema parameter")
811
+ if connection is None:
812
+ raise ValueError("SQL Server backend requires a connection parameter")
813
+ from prograph.executors.sqlserver_executor import SQLServerExecutor
814
+ return SQLServerExecutor(schema, connection)
815
+ elif backend == "oracle":
816
+ if schema is None:
817
+ raise ValueError("Oracle backend requires a schema parameter")
818
+ if connection is None:
819
+ raise ValueError("Oracle backend requires a connection parameter")
820
+ from prograph.executors.oracle_executor import OracleExecutor
821
+ return OracleExecutor(schema, connection)
822
+ else:
823
+ raise ValueError(f"Unknown backend: {backend}")
824
+
825
+ @classmethod
826
+ def _from_source_maps(cls, maps: Any, connection: Any = None, backend: str = "duckdb",
827
+ **kwargs: Any) -> "ProGraph":
828
+ """An engine over source maps compiled elsewhere. Provisional and internal: the Cypher TCK
829
+ harness uses it for each scenario's typed tables, and a generated mapping will;
830
+ its public form is Brice's decision. The engine reads no authoring object:
831
+ column types come from the connection, as for an R2RML mapping."""
832
+ from prograph.schema.maps_schema import SourceMapsSchema
833
+ return cls(schema=SourceMapsSchema(maps), backend=backend, connection=connection, **kwargs)
834
+
835
+ @classmethod
836
+ def from_config(
837
+ cls,
838
+ config_path: str,
839
+ spark: Optional[Any] = None,
840
+ backend: Optional[BackendType] = None,
841
+ **kwargs
842
+ ) -> "ProGraph":
843
+ """Create a ProGraph engine from a YAML configuration file.
844
+
845
+ This factory method loads graph schema and data sources from a YAML
846
+ configuration file, following patterns from Databricks Metric Views
847
+ and Asset Bundles (DABs).
848
+
849
+ Args:
850
+ config_path: Path to YAML configuration file
851
+ spark: SparkSession (for the Spark backend; or pass connection= for table sources)
852
+ backend: Override backend type (default: Spark with spark=, the connection's with connection=,
853
+ DuckDB over the files otherwise)
854
+ **kwargs: Additional arguments passed to ProGraph constructor
855
+
856
+ Returns:
857
+ Configured ProGraph engine
858
+
859
+ Example YAML configuration:
860
+
861
+ version: "1.0"
862
+
863
+ graph:
864
+ name: social_network
865
+ comment: "LDBC SNB social network schema"
866
+
867
+ sources:
868
+ person:
869
+ table: catalog.schema.person
870
+ knows:
871
+ file: ./data/person_knows_person.csv
872
+ options:
873
+ delimiter: "|"
874
+
875
+ nodes:
876
+ - label: Person
877
+ source: person
878
+ id_column: id
879
+ properties:
880
+ firstName: firstName
881
+ lastName: lastName
882
+
883
+ relationships:
884
+ - type: KNOWS
885
+ source: knows
886
+ start_node:
887
+ label: Person
888
+ column: Person.id
889
+ end_node:
890
+ label: Person
891
+ column: Person.id.1
892
+
893
+ Example usage:
894
+
895
+ # With Spark (for table sources or large files)
896
+ engine = ProGraph.from_config('graph_schema.yaml', spark=spark)
897
+
898
+ # With pandas (for local files only)
899
+ engine = ProGraph.from_config('graph_schema.yaml')
900
+
901
+ # Over a database's tables (DuckDB, PostgreSQL, ...)
902
+ engine = ProGraph.from_config('graph_schema.yaml', connection=duckdb.connect('my.db'))
903
+
904
+ # Override backend explicitly
905
+ engine = ProGraph.from_config(
906
+ 'graph_schema.yaml',
907
+ spark=spark,
908
+ backend='spark'
909
+ )
910
+ """
911
+ from pathlib import Path
912
+ from prograph.config import load_config, validate_config, load_dataframes
913
+
914
+ # Load and validate configuration
915
+ config = load_config(config_path)
916
+ errors = validate_config(config)
917
+ if errors:
918
+ raise ValueError(f"Configuration errors: {'; '.join(errors)}")
919
+
920
+ return cls._from_graph_config(config, spark, backend, Path(config_path).parent, "from_config", kwargs)
921
+
922
+ @classmethod
923
+ def _from_graph_config(cls, config, spark, backend, base_path, factory: str, kwargs) -> "ProGraph":
924
+ """The engine of a loaded YAML configuration (`from_config`, `from_yaml`). With `connection=` (a DuckDB,
925
+ PostgreSQL ... connection) every source is a table it holds, read where it is (a query or file source is
926
+ refused, by name); with `spark=` tables, queries and files are read by Spark; with neither, file sources
927
+ are read as frames by DuckDB."""
928
+ from prograph.config import load_dataframes
929
+ connection = kwargs.get("connection")
930
+ if spark is None and connection is not None:
931
+ # Over a database's tables: each source a table the connection holds, read where it is.
932
+ queries = [n for n, s in config.sources.items() if s.query]
933
+ files = [n for n, s in config.sources.items() if s.file]
934
+ if queries:
935
+ raise ValueError(
936
+ f"query sources with connection=: {', '.join(queries)}. Create a view of each query in the "
937
+ f"database and name it as a table source, or pass spark= to {factory}().")
938
+ if files:
939
+ raise ValueError(
940
+ f"file sources with connection=: {', '.join(files)}. Load the files into the database and "
941
+ f"name them as table sources, or leave out connection= to read the files with DuckDB.")
942
+ engine = cls(backend=backend, schema=config.to_schema(), spark=None, **kwargs)
943
+ else:
944
+ in_database = [n for n, s in config.sources.items() if s.table or s.query]
945
+ if in_database and spark is None:
946
+ raise ValueError(
947
+ f"these sources are tables or queries: {', '.join(in_database)}. Pass connection= (a "
948
+ f"database connection that holds the tables, e.g. duckdb.connect('my.db')) or spark= to "
949
+ f"{factory}().")
950
+ if backend is None:
951
+ backend = 'spark' if spark is not None else 'duckdb' # local files: DuckDB over the frames
952
+ dataframes = load_dataframes(config, spark=spark, base_path=base_path)
953
+ engine = cls(backend=backend, schema=config.to_schema(), dataframes=dataframes, spark=spark,
954
+ **kwargs)
955
+ engine._config = config # the configuration, for debugging
956
+ engine._declare_config_procedures(config)
957
+ return engine
958
+
959
+ @classmethod
960
+ def from_direct_mapping(
961
+ cls,
962
+ connection: Any,
963
+ base_iri: str,
964
+ schema: Optional[str] = None,
965
+ tables: Optional[Sequence[str]] = None,
966
+ backend: Optional[BackendType] = None,
967
+ **kwargs
968
+ ) -> "ProGraph":
969
+ """Create a ProGraph engine over a database's tables as the W3C Direct Mapping
970
+ (https://www.w3.org/TR/rdb-direct-mapping/) gives them as RDF, with no mapping to write.
971
+
972
+ The catalog (tables, columns, declared primary and foreign keys) is read once and the mapping
973
+ generated from it (`prograph.r2rml.direct`): a row of a table with a primary key is
974
+ `<base_iri>Table/pk=value`, a row of a table with none a blank node, a column a literal
975
+ `<base_iri>Table#Column`, a foreign key a reference `<base_iri>Table#ref-column`. SPARQL then
976
+ reads the live tables; nothing is copied. `ProGraph.direct_mapping` gives the same mapping as
977
+ R2RML Turtle, to edit.
978
+
979
+ Args:
980
+ connection: A DuckDB connection, or a PostgreSQL or Databricks SQL one with `backend` named.
981
+ base_iri: The absolute IRI the generated IRIs are relative to; required.
982
+ schema: The schema to map (`catalog.schema` on Databricks); the current one by default.
983
+ tables: Map only these tables; a foreign key to another is refused.
984
+ backend: 'duckdb' (the default for a DuckDB connection), 'postgresql' or 'databricks_sql'.
985
+ **kwargs: Passed to the ProGraph constructor.
986
+
987
+ Raises:
988
+ prograph.r2rml.catalog.DirectMappingError: A database with no catalog reader, a relative
989
+ base IRI, a table not there, or a foreign key to a table not mapped.
990
+ """
991
+ from prograph.r2rml.direct import direct_mapping_r2rml
992
+ from prograph.r2rml.schema import R2RMLSchema
993
+
994
+ mapping = direct_mapping_r2rml(connection, base_iri, schema, tables, backend=backend)
995
+ return cls(backend=backend or 'duckdb', schema=R2RMLSchema(mapping, base_iri=base_iri),
996
+ connection=connection, **kwargs)
997
+
998
+ @classmethod
999
+ def direct_mapping(
1000
+ cls,
1001
+ connection: Any,
1002
+ base_iri: str,
1003
+ schema: Optional[str] = None,
1004
+ tables: Optional[Sequence[str]] = None,
1005
+ backend: Optional[BackendType] = None,
1006
+ format: str = "turtle",
1007
+ ) -> Any:
1008
+ """The Direct Mapping `from_direct_mapping` reads, as a mapping to edit and load (on the engine it was
1009
+ exported from: a keyless table's rows are numbered by a query in that engine's SQL).
1010
+
1011
+ Args:
1012
+ connection, base_iri, schema, tables, backend: As `from_direct_mapping`.
1013
+ format: "turtle" gives the R2RML Turtle text (write it to a file, edit it, load it with
1014
+ `from_r2rml`); "r2rml" gives the `R2RMLMapping` model. A YAML export is not offered:
1015
+ YAML source maps cannot state a keyless table's numbered blank nodes.
1016
+
1017
+ Raises:
1018
+ prograph.r2rml.catalog.DirectMappingError: As `from_direct_mapping`.
1019
+ ValueError: A format other than "turtle" or "r2rml".
1020
+ """
1021
+ from prograph.r2rml.direct import direct_mapping_r2rml
1022
+ from prograph.r2rml.export import to_turtle
1023
+
1024
+ if format not in ("turtle", "r2rml"):
1025
+ raise ValueError(f"format must be 'turtle' or 'r2rml'; got {format!r}")
1026
+ mapping = direct_mapping_r2rml(connection, base_iri, schema, tables, backend=backend)
1027
+ return to_turtle(mapping) if format == "turtle" else mapping
1028
+
1029
+ @classmethod
1030
+ def from_r2rml(
1031
+ cls,
1032
+ r2rml_path: str,
1033
+ dataframes: Optional[Dict[str, Any]] = None,
1034
+ spark: Optional[Any] = None,
1035
+ backend: Optional[BackendType] = None,
1036
+ **kwargs
1037
+ ) -> "ProGraph":
1038
+ """Create a ProGraph engine from a W3C R2RML mapping file.
1039
+
1040
+ Parses the R2RML Turtle document, compiles it to a GraphSchema,
1041
+ and creates an engine ready for queries.
1042
+
1043
+ A TriplesMap whose logical table is an rr:sqlQuery (R2RML 5.2) is a
1044
+ mapping whose source is that query: SQL reads it as a derived table,
1045
+ on every backend, and a DataFrame backend computes its frame from
1046
+ the given frames (Spark: from the session's tables).
1047
+
1048
+ Args:
1049
+ r2rml_path: Path to R2RML Turtle file
1050
+ dataframes: Optional dict of source name -> DataFrame
1051
+ spark: SparkSession
1052
+ backend: Backend type (inferred from spark if not given)
1053
+ **kwargs: Additional arguments passed to ProGraph constructor
1054
+
1055
+ Returns:
1056
+ Configured ProGraph engine
1057
+
1058
+ Raises:
1059
+ FileNotFoundError: If the R2RML file does not exist
1060
+ ImportError: If rdflib is not installed
1061
+
1062
+ Example::
1063
+
1064
+ engine = ProGraph.from_r2rml(
1065
+ "mapping.ttl",
1066
+ spark=spark
1067
+ )
1068
+ results = engine.execute("MATCH (e:Employee) RETURN e.name")
1069
+ """
1070
+ from prograph.r2rml.parser import R2RMLParser
1071
+ from prograph.r2rml.schema import R2RMLSchema
1072
+
1073
+ # The mapping's source maps, no property-graph view.
1074
+ schema = R2RMLSchema(R2RMLParser().parse_file(r2rml_path))
1075
+
1076
+ # Spark when a session is given; a connection alone is left to the
1077
+ # constructor (a DuckDB connection is DuckDB, SQL only; another kind
1078
+ # needs its backend named); otherwise DuckDB over the dataframes.
1079
+ resolved_backend = backend or (
1080
+ 'spark' if spark else None if kwargs.get('connection') is not None
1081
+ and dataframes is None else 'duckdb')
1082
+
1083
+ return cls(
1084
+ backend=resolved_backend,
1085
+ schema=schema,
1086
+ dataframes=dataframes,
1087
+ spark=spark,
1088
+ **kwargs,
1089
+ )
1090
+
1091
+ @classmethod
1092
+ def from_yaml(
1093
+ cls,
1094
+ yaml_string: str,
1095
+ spark: Optional[Any] = None,
1096
+ backend: Optional[BackendType] = None,
1097
+ base_path: Optional[str] = None,
1098
+ **kwargs
1099
+ ) -> "ProGraph":
1100
+ """Create a ProGraph engine from a YAML string.
1101
+
1102
+ Similar to from_config() but accepts a YAML string directly.
1103
+ Useful for embedded configurations or testing.
1104
+
1105
+ Args:
1106
+ yaml_string: YAML configuration as string
1107
+ spark: SparkSession (for the Spark backend; or pass connection= for table sources)
1108
+ backend: Override backend type
1109
+ base_path: Base path for resolving relative file paths
1110
+ **kwargs: Additional arguments passed to ProGraph constructor
1111
+
1112
+ Returns:
1113
+ Configured ProGraph engine
1114
+ """
1115
+ from pathlib import Path
1116
+ from prograph.config import load_config_from_string, validate_config, load_dataframes
1117
+
1118
+ # Load and validate configuration
1119
+ config = load_config_from_string(yaml_string)
1120
+ errors = validate_config(config)
1121
+ if errors:
1122
+ raise ValueError(f"Configuration errors: {'; '.join(errors)}")
1123
+
1124
+ return cls._from_graph_config(config, spark, backend, Path(base_path) if base_path else Path('.'),
1125
+ "from_yaml", kwargs)
1126
+
1127
+
1128
+ def _detect_language(self, query: str) -> str:
1129
+ """Detect the query language from the query string.
1130
+
1131
+ Uses heuristics to determine which language the query is written in:
1132
+ - Starts with 'g.' -> Gremlin (refused: not supported)
1133
+ - Contains GRAPH_TABLE or PROPERTY GRAPH -> SQL/PGQ (refused: not supported)
1134
+ - Starts with PREFIX, SELECT, ... -> SPARQL
1135
+ - Otherwise Cypher
1136
+
1137
+ Args:
1138
+ query: The query string to analyze
1139
+
1140
+ Returns:
1141
+ Language identifier: "cypher", "sparql", or, for the two
1142
+ languages ProGraph does not support, "gremlin" or "sqlpgq"
1143
+ (which `_get_plan_builder` refuses).
1144
+ """
1145
+ query_stripped = query.strip()
1146
+
1147
+ # Gremlin queries start with 'g.'
1148
+ if query_stripped.startswith('g.'):
1149
+ return "gremlin"
1150
+
1151
+ # SQL/PGQ: contains GRAPH_TABLE or PROPERTY GRAPH
1152
+ query_upper = query_stripped.upper()
1153
+ if 'GRAPH_TABLE' in query_upper or 'PROPERTY GRAPH' in query_upper:
1154
+ return "sqlpgq"
1155
+
1156
+ # SPARQL: starts with SELECT (without MATCH), PREFIX, BASE, CONSTRUCT, DESCRIBE, or ASK
1157
+ sparql_keywords = ('SELECT', 'PREFIX', 'BASE', 'CONSTRUCT', 'DESCRIBE', 'ASK')
1158
+ if any(query_upper.startswith(kw) for kw in sparql_keywords):
1159
+ # Distinguish from Cypher: SPARQL SELECT doesn't have MATCH
1160
+ if query_upper.startswith('SELECT') and 'MATCH' in query_upper:
1161
+ pass # This is likely Cypher, fall through
1162
+ else:
1163
+ return "sparql"
1164
+
1165
+ # Default to Cypher
1166
+ return "cypher"
1167
+
1168
+
1169
+ def _convert_results(self, table: Any, return_type: ReturnType) -> Any:
1170
+ """The answer, a pyarrow.Table, in the asked return type."""
1171
+ import pyarrow as pa
1172
+ if isinstance(table, pa.RecordBatchReader):
1173
+ if return_type == "batches":
1174
+ return table
1175
+ table = table.read_all()
1176
+ if return_type == "batches":
1177
+ size = getattr(self, "_batch_size", None) or 100_000
1178
+ return pa.RecordBatchReader.from_batches(table.schema, table.to_batches(max_chunksize=size))
1179
+ if return_type == "arrow":
1180
+ return table
1181
+ if return_type == "list":
1182
+ return table.to_pylist(maps_as_pydicts="strict")
1183
+ if self.backend == "spark":
1184
+ from pyspark.sql import SparkSession
1185
+ spark = SparkSession.builder.getOrCreate()
1186
+ if table.num_columns == 0:
1187
+ # Rows with no column: a Spark DataFrame keeps the count.
1188
+ return spark.range(table.num_rows).select()
1189
+ return spark.createDataFrame(table)
1190
+ from prograph.results import table_to_pandas
1191
+ return table_to_pandas(table)
1192
+
1193
+ def _language_of(self, text: str, language: Optional[LanguageType]) -> str:
1194
+ """The query's language: the call's, else the engine's default,
1195
+ detected from the text where that is 'auto'. Gremlin, SQL/PGQ and
1196
+ GQL are refused by name."""
1197
+ lang = language if language is not None else self.default_language
1198
+ if lang == "auto":
1199
+ lang = self._detect_language(text)
1200
+ if lang in _UNSUPPORTED_LANGUAGES:
1201
+ raise ValueError(f"{_UNSUPPORTED_LANGUAGES[lang]} is not supported in ProGraph 1.0. "
1202
+ f"Supported languages: sparql, cypher")
1203
+ if lang not in ("sparql", "cypher"):
1204
+ raise ValueError(f"Unsupported query language: {lang}. "
1205
+ f"Supported languages: sparql, cypher")
1206
+ return lang
1207
+
1208
+ def execute(
1209
+ self,
1210
+ query: str,
1211
+ language: Optional[LanguageType] = None,
1212
+ return_type: Optional[ReturnType] = None,
1213
+ context: Optional['QueryContext'] = None,
1214
+ batch_size: int = 100_000,
1215
+ parameters: Optional[Dict[str, Any]] = None,
1216
+ step_key: Optional[str] = None,
1217
+ dataset: Optional[Tuple[Sequence[str], Sequence[str]]] = None,
1218
+ ) -> Any:
1219
+ """Answer a query: it is translated into one relational plan over
1220
+ the mapping's source maps, lowered to SQL, and run on the backend
1221
+ (docs/architecture/one-plan.md).
1222
+
1223
+ Args:
1224
+ query: The query text, optionally after a hint block
1225
+ (``/*+ terms=all */``).
1226
+ language: "sparql", "cypher" or "auto" (detect from the text);
1227
+ default the engine's default_language.
1228
+ return_type: "arrow" (a pyarrow.Table, the default), "batches"
1229
+ (a pyarrow.RecordBatchReader of at most batch_size rows),
1230
+ "list" (one dict per row) or "dataframe" (a Spark DataFrame
1231
+ on Spark, else pandas).
1232
+ context: A QueryContext, to cancel the query from another thread:
1233
+ its cancel() stops the statement the database is running, and the
1234
+ query raises QueryCancelledException (a streamed answer is
1235
+ cancellable until its reader ends).
1236
+ parameters: Values for the query's parameters.
1237
+ dataset: A SPARQL query's dataset, (default graph IRIs, named
1238
+ graph IRIs), as the SPARQL Protocol's ``default-graph-uri``
1239
+ and ``named-graph-uri``: used in place of the query's own
1240
+ FROM and FROM NAMED clauses (SPARQL 1.1 Query 13.2).
1241
+ step_key: A query that writes (Cypher's CREATE, into the
1242
+ engine's owned store) is one write step; this is its
1243
+ idempotency key. A key the store applied already writes
1244
+ nothing, and the query answers as the first run did.
1245
+ None: a new key for each run.
1246
+
1247
+ Raises:
1248
+ NotImplementedError: For what the plan does not build yet; its
1249
+ message starts "one-plan:".
1250
+ ValueError: For a language ProGraph does not support.
1251
+ """
1252
+ from prograph.plan.ir import entry
1253
+ with self._execute_lock:
1254
+ text, hints = parse_hints(query)
1255
+ if dataset is not None:
1256
+ hints.dataset = (tuple(dataset[0]), tuple(dataset[1]))
1257
+ lang = self._language_of(text, language)
1258
+ wanted = return_type if return_type is not None else self.default_return_type
1259
+ self._batch_size = batch_size
1260
+ from prograph.execution import running, until_read
1261
+ if context:
1262
+ context.mark_running()
1263
+ try:
1264
+ # Every statement the query starts registers with `context`, whose cancel()
1265
+ # stops it (`StatementGuard`); no later one starts.
1266
+ with running(context):
1267
+ if context:
1268
+ context.check_cancelled()
1269
+ table = entry.run(self, text, lang, hints=hints, parameters=parameters,
1270
+ streaming=wanted == "batches", step_key=step_key)
1271
+ import pyarrow as pa
1272
+ if context and isinstance(table, pa.RecordBatchReader):
1273
+ # Streamed: the query runs until its reader ends, and is cancellable until then.
1274
+ return until_read(table, context)
1275
+ if context:
1276
+ context.check_cancelled()
1277
+ context.mark_completed()
1278
+ if not isinstance(table, pa.Table):
1279
+ # A CONSTRUCT's answer is an RDF graph (16.2), which no
1280
+ # return type reshapes.
1281
+ return table
1282
+ return self._convert_results(table, wanted)
1283
+ except Exception as e:
1284
+ if context:
1285
+ context.mark_failed(str(e))
1286
+ raise
1287
+
1288
+ def describe_dataset(self, format: str = "graph", *, counts: bool = False, endpoint: Optional[str] = None,
1289
+ dataset_iri: Optional[str] = None, validation: Any = None,
1290
+ include_report: bool = False) -> Any:
1291
+ """The mapped graph described as DCAT 3, VoID and a SPARQL 1.1 Service Description
1292
+ (`prograph.describe`), from the source maps alone.
1293
+
1294
+ Args:
1295
+ format: "graph" gives an rdflib Graph; an rdflib serialization name ("turtle", "xml",
1296
+ "nt", "json-ld") gives that text.
1297
+ counts: Add the VoID counts (`void:triples`, `void:entities`, per class and per property).
1298
+ Each is a SPARQL COUNT query run by this engine, so it reads the data.
1299
+ endpoint: The SPARQL endpoint's IRI; given, the description has the `sd:Service` too.
1300
+ dataset_iri: The dataset's IRI (default `<endpoint>#dataset`, else a blank node).
1301
+ validation: A `ValidationReport` (`validate_shapes`, or `shacl_last_report`): its DQV measurements
1302
+ (`dqv:hasQualityMeasurement`; conformance and violation count) are linked to the dataset.
1303
+ include_report: With `validation`, add the SHACL report graph too, each measurement's `dcterms:source`.
1304
+ """
1305
+ import warnings
1306
+ from prograph.describe import RESULT_FORMAT_IRIS, describe
1307
+ from prograph.plan.ir.entry import source_maps
1308
+ from prograph.plan.ir.cost import ProGraphCostWarning
1309
+
1310
+ def count(query: str) -> int:
1311
+ with warnings.catch_warnings():
1312
+ warnings.simplefilter("ignore", ProGraphCostWarning)
1313
+ table = self.execute(query, language="sparql", return_type="arrow")
1314
+ return table.column(0)[0].as_py() or 0
1315
+
1316
+ graph = describe(source_maps(self.schema), default_graph=self.default_graph, endpoint=endpoint,
1317
+ dataset_iri=dataset_iri, formats=RESULT_FORMAT_IRIS if endpoint else (),
1318
+ count=count if counts else None, validation=validation, include_report=include_report)
1319
+ if format == "graph":
1320
+ return graph
1321
+ return graph.serialize(format=format)
1322
+
1323
+ def provenance(self, format: str = "graph", *, endpoint: Optional[str] = None,
1324
+ dataset_iri: Optional[str] = None, base: Optional[str] = None) -> Any:
1325
+ """Where the mapped graph's triples come from, as PROV-O (`prograph.provenance`), from the source
1326
+ maps alone: each source (table or query), source map and triple pattern (a map's subject, predicate
1327
+ and object term) with the columns it reads, the dataset `prov:wasDerivedFrom` its sources, and the
1328
+ mapping's version (`dcterms:hasVersion`, a content hash of the compiled maps).
1329
+
1330
+ Args:
1331
+ format: "graph" gives an rdflib Graph; an rdflib serialization name gives that text.
1332
+ endpoint: As in `describe_dataset`: the dataset is `<endpoint>#dataset`, so the two graphs join.
1333
+ dataset_iri: The dataset's IRI; wins over `endpoint`.
1334
+ base: The prefix of the IRIs minted (default `urn:prograph:prov:`).
1335
+ """
1336
+ from prograph.plan.ir.entry import source_maps
1337
+ from prograph.provenance import DEFAULT_BASE, provenance
1338
+ graph = provenance(source_maps(self.schema), base=base or DEFAULT_BASE,
1339
+ dataset_iri=dataset_iri or (endpoint + "#dataset" if endpoint else None))
1340
+ if format == "graph":
1341
+ return graph
1342
+ return graph.serialize(format=format)
1343
+
1344
+ def to_sql(
1345
+ self, query: str, language: Optional[LanguageType] = None,
1346
+ dialect: Optional[str] = None,
1347
+ parameters: Optional[Dict[str, Any]] = None,
1348
+ dataset: Optional[Tuple[Sequence[str], Sequence[str]]] = None,
1349
+ ) -> str:
1350
+ """The SQL `execute` runs for a query, in `dialect` (default: the
1351
+ dialect this engine's backend runs), without running it.
1352
+
1353
+ Over an owned store the SQL holds for the moment it was planned:
1354
+ which stored OWL RL facts and owl:sameAs components it reads was
1355
+ decided from the store's record then, and the SQL carries no
1356
+ condition on it, so run later, after a run under another cache
1357
+ key, it may read that key's facts. Plan again to run it later.
1358
+
1359
+ Raises:
1360
+ NotImplementedError: For what the plan does not build yet.
1361
+ ValueError: For an unknown dialect or language.
1362
+ """
1363
+ from prograph.plan.ir import entry
1364
+ if dialect is not None:
1365
+ from prograph.plan.sql.dialect import get_dialect
1366
+ get_dialect(dialect) # an unknown dialect raises ValueError
1367
+ text, hints = parse_hints(query)
1368
+ if dataset is not None:
1369
+ hints.dataset = (tuple(dataset[0]), tuple(dataset[1]))
1370
+ lang = self._language_of(text, language)
1371
+ return entry.sql(self, text, lang, dialect, hints=hints, parameters=parameters)
1372
+
1373
+ def explain(
1374
+ self, query: str, language: Optional[LanguageType] = None,
1375
+ dialect: Optional[str] = None,
1376
+ parameters: Optional[Dict[str, Any]] = None,
1377
+ dataset: Optional[Tuple[Sequence[str], Sequence[str]]] = None,
1378
+ ) -> str:
1379
+ """What `execute` would do for a query, without running it: the SQL,
1380
+ each rewrite of the plan with what it relied on, and each key the
1381
+ plan may rely on with what makes it trusted (a constraint the
1382
+ database enforces, a Databricks RELY, or the mapping's own assertion,
1383
+ `keys: trusted`, which is never shown as checked).
1384
+
1385
+ Raises:
1386
+ NotImplementedError: For what the plan does not build yet.
1387
+ ValueError: For an unknown dialect or language.
1388
+ """
1389
+ from prograph.plan.ir import entry
1390
+ if dialect is not None:
1391
+ from prograph.plan.sql.dialect import get_dialect
1392
+ get_dialect(dialect)
1393
+ text, hints = parse_hints(query)
1394
+ if dataset is not None:
1395
+ hints.dataset = (tuple(dataset[0]), tuple(dataset[1]))
1396
+ lang = self._language_of(text, language)
1397
+ return entry.explain(self, text, lang, dialect, hints=hints, parameters=parameters)
1398
+
1399
+ # ========== Live Schema Update ==========
1400
+
1401
+ def _record_column_types(self) -> None:
1402
+ """Each mapped table's column value types, on the schema, for the
1403
+ SQL translator (prograph.schema.types): from the DataFrames' dtypes,
1404
+ or from the DuckDB connection's DESCRIBE. A table whose types
1405
+ cannot be read is left out, and its columns are UNKNOWN."""
1406
+ from prograph.schema import collations as C
1407
+ from prograph.schema import types as T
1408
+ schema = self._schema
1409
+ if schema is None:
1410
+ return
1411
+ # The graph's frames are the ones writes and entailment change.
1412
+ graph_frames = getattr(getattr(getattr(self, 'executor', None), 'graph', None), 'dataframes', None)
1413
+ frames = graph_frames if isinstance(graph_frames, dict) and graph_frames else (self._dataframes or {})
1414
+ for table, frame in frames.items():
1415
+ try:
1416
+ if hasattr(frame, 'dtypes') and hasattr(frame, 'columns') and hasattr(frame, 'iloc'):
1417
+ # A frame is read through DuckDB, whose strings compare byte for byte.
1418
+ schema.column_types[table] = C.mark_bytewise(T.of_frame(frame), 'duckdb', C.binary)
1419
+ elif hasattr(frame, 'dtypes'): # a Spark DataFrame: [(name, type)]
1420
+ kinds = dict(frame.dtypes)
1421
+ schema.column_types[table] = C.mark_bytewise(
1422
+ {c: T.of_sql_type(t, timestamp_is_an_instant=True) for c, t in frame.dtypes},
1423
+ 'spark', lambda c: C.collation_in_type(kinds.get(c, "")))
1424
+ except Exception: # noqa: BLE001 - types are an optimisation of correctness, not required
1425
+ continue
1426
+ self._record_frame_decimal_digits(frames)
1427
+ self._record_query_column_types(schema)
1428
+ conn = getattr(self, '_connection', None)
1429
+ backend = getattr(self, 'backend', None)
1430
+ if conn is None or getattr(self, '_frame_backed', False):
1431
+ return
1432
+ from prograph.plan.ir.entry import mapped_tables
1433
+ sources = {s.table: s for s in mapped_tables(schema)}
1434
+ queries = {t: s.query for t, s in sources.items()}
1435
+ # Sources the engine could not read, by table: the database's message, raised by the first query that
1436
+ # reads one (`entry.check_sources`), so the cause is named where the query fails.
1437
+ self._unreadable_sources = {}
1438
+ files = set(getattr(schema, "file_sources", ()) or ())
1439
+ if backend == 'databricks_sql':
1440
+ # Every mapped table's columns in one catalog read, and which have a primary key in another; a table they do not cover is
1441
+ # described on its own below.
1442
+ self._warehouse_catalog_read(conn, schema, sources, queries)
1443
+ for table in [t for t in queries if t not in schema.column_types]:
1444
+ # A table whose types are not read has none, and every SQL form
1445
+ # chosen by a type takes the form for an unknown one: say so.
1446
+ # A whole backend was in that state unnoticed.
1447
+ try:
1448
+ rows = self._describe_table(conn, backend, table, query=queries[table],
1449
+ written=sources[table].qualified)
1450
+ except Exception as exc: # noqa: BLE001
1451
+ from prograph.logging import get_plan_logger
1452
+ get_plan_logger().warning("column_types_skip", reason="describe_failed",
1453
+ table=table, backend=backend, error=str(exc))
1454
+ if table in files:
1455
+ # A YAML file source the connection has no table or view for: refused by name now.
1456
+ from prograph.utils.exceptions import SourceNotReadableError
1457
+ raise SourceNotReadableError(
1458
+ f"the source {table!r} is a file (sources.{table}.file) and the {backend} connection has "
1459
+ f"no table or view {table!r}: load the file into a table and name it in the source "
1460
+ f"(sources.{table}.table), or build the engine with from_config/from_yaml, which read "
1461
+ f"files into frames") from exc
1462
+ if not isinstance(exc, NotImplementedError):
1463
+ # A source the database would not describe (missing, not readable); a source this backend
1464
+ # cannot describe (a query on MySQL, SQL Server, Oracle) keeps its columns untyped instead.
1465
+ self._unreadable_sources[table] = (sources[table].qualified, " ".join(str(exc).split())[:400])
1466
+ continue
1467
+ if rows:
1468
+ # Databricks' TIMESTAMP is an instant, as Spark's.
1469
+ instant = backend == 'databricks_sql'
1470
+ collations = C.string_collations(conn, backend, table, query=queries[table],
1471
+ written=sources[table].qualified, described=rows)
1472
+ schema.column_types[table] = C.mark_bytewise(
1473
+ {name: T.of_sql_type(kind, timestamp_is_an_instant=instant) for name, kind in rows},
1474
+ backend, collations)
1475
+ else:
1476
+ from prograph.logging import get_plan_logger
1477
+ get_plan_logger().warning("column_types_skip", reason="no_columns_described",
1478
+ table=table, backend=backend)
1479
+ self._unreadable_sources[table] = (sources[table].qualified, "the database described no columns")
1480
+
1481
+ def _warehouse_catalog_read(self, conn, schema, sources, queries) -> None:
1482
+ """A Databricks SQL warehouse's column types for every mapped table (not a query source) from one read
1483
+ of `system.information_schema.columns` (`full_data_type`: DESCRIBE's vocabulary), and the tables that
1484
+ declare a primary key from one read of `table_constraints` (whether it is RELY is in DESCRIBE TABLE
1485
+ EXTENDED only: read per such table when a plan asks, `keys._constraint_keys`). A read that fails
1486
+ leaves the tables to be described one by one."""
1487
+ from prograph.graph.drivers import fetch_rows
1488
+ from prograph.schema import collations as C
1489
+ from prograph.schema import types as T
1490
+
1491
+ def parts(written):
1492
+ text = str(written)
1493
+ if text.count('`') and any('.' in seg for seg in text.split('`')[1::2]):
1494
+ return None # a delimited name holding a dot: described on its own
1495
+ names = [p.strip('`') for p in text.split('.')]
1496
+ if len(names) > 3:
1497
+ return None
1498
+ names = [None] * (3 - len(names)) + names
1499
+ return [n.lower() if n else None for n in names]
1500
+ wanted = {t: parts(sources[t].qualified) for t in queries if queries[t] is None}
1501
+ wanted = {t: spec for t, spec in wanted.items() if spec is not None}
1502
+ if not wanted:
1503
+ return
1504
+
1505
+ def where():
1506
+ ors = []
1507
+ for cat, sch, name in wanted.values():
1508
+ ors.append("(table_catalog = " + (f"'{cat}'" if cat else "current_catalog()") +
1509
+ " AND table_schema = " + (f"'{sch}'" if sch else "current_schema()") +
1510
+ f" AND table_name = '{name}')")
1511
+ return " OR ".join(ors)
1512
+ if any("'" in (n or "") for spec in wanted.values() for n in spec):
1513
+ return # a name with a quote: described one by one
1514
+ try:
1515
+ rows = fetch_rows(conn, "SELECT table_catalog, table_schema, table_name, column_name, full_data_type, "
1516
+ "current_catalog(), current_schema() "
1517
+ f"FROM system.information_schema.columns WHERE {where()} "
1518
+ "ORDER BY table_catalog, table_schema, table_name, ordinal_position")[1]
1519
+ keyed = fetch_rows(conn, "SELECT DISTINCT table_catalog, table_schema, table_name "
1520
+ "FROM system.information_schema.table_constraints "
1521
+ f"WHERE constraint_type = 'PRIMARY KEY' AND ({where()})")[1]
1522
+ except Exception as exc: # noqa: BLE001 - described one by one instead
1523
+ from prograph.logging import get_plan_logger
1524
+ get_plan_logger().info("warehouse_catalog_read_skipped", error=str(exc)[:200])
1525
+ return
1526
+ if not rows:
1527
+ return
1528
+ cur_cat, cur_sch = str(rows[0][5]).lower(), str(rows[0][6]).lower()
1529
+
1530
+ def full(spec):
1531
+ cat, sch, name = spec
1532
+ return (cat or cur_cat, sch or cur_sch, name)
1533
+ by_table: Dict[tuple, list] = {}
1534
+ for cat, sch, name, column, kind, _, _ in rows:
1535
+ by_table.setdefault((str(cat).lower(), str(sch).lower(), str(name).lower()), []).append(
1536
+ (str(column), str(kind)))
1537
+ for table, spec in wanted.items():
1538
+ columns = by_table.get(full(spec))
1539
+ if columns:
1540
+ kinds = dict(columns)
1541
+ schema.column_types[table] = C.mark_bytewise(
1542
+ {c: T.of_sql_type(k, timestamp_is_an_instant=True) for c, k in columns},
1543
+ 'databricks_sql', lambda c, kinds=kinds: C.collation_in_type(kinds.get(c, "")))
1544
+ graph = getattr(getattr(self, 'executor', None), 'graph', None)
1545
+ if graph is not None:
1546
+ # Only a table the columns read found is known to have no primary key; one it missed (a view, a
1547
+ # name it did not match, no privilege) is asked on its own (DESCRIBE TABLE EXTENDED) as before.
1548
+ with_key = {tuple(str(v).lower() for v in r) for r in keyed}
1549
+ graph.__dict__["_warehouse_keyed"] = {sources[t].qualified: full(spec) in with_key
1550
+ for t, spec in wanted.items() if full(spec) in by_table}
1551
+
1552
+ def _record_query_column_types(self, schema) -> None:
1553
+ """The column types of each rr:sqlQuery source over frames (`query_sources.describe`)."""
1554
+ from prograph.schema.query_sources import describe
1555
+ describe(self, schema)
1556
+
1557
+ def _record_frame_decimal_digits(self, frames) -> None:
1558
+ """A pandas frame's DECIMAL columns' (precision, scale) as the
1559
+ graph's embedded DuckDB reads them, the type its SQL sees: a column
1560
+ of Decimals has no size of its own in pandas, and is read at the
1561
+ size all its values fit (`pandas_graph.frames_connection`). A decimal quotient's
1562
+ SQL form is chosen by it (ExprTranslator.decimal_digits). A column DuckDB does not read as a DECIMAL keeps none."""
1563
+ from dataclasses import replace
1564
+ from prograph.schema import types as T
1565
+ local = getattr(getattr(getattr(self, 'executor', None), 'graph', None), '_local_sql', None)
1566
+ schema = self._schema
1567
+ if local is None or schema is None:
1568
+ return
1569
+ for table, frame in frames.items():
1570
+ types = schema.column_types.get(table) or {}
1571
+ if not any(t.exact and t.digits is None for t in types.values()):
1572
+ continue
1573
+ cursor = None
1574
+ try:
1575
+ cursor = local.cursor()
1576
+ relation = cursor.from_df(frame)
1577
+ described = list(zip(relation.columns, (str(t) for t in relation.types)))
1578
+ except Exception: # noqa: BLE001 - a size is an optimisation, not required
1579
+ continue
1580
+ finally:
1581
+ if cursor is not None:
1582
+ cursor.close()
1583
+ for name, kind in described:
1584
+ held = types.get(name)
1585
+ if held is not None and held.exact and held.digits is None:
1586
+ read = T.of_sql_type(str(kind))
1587
+ if read.exact and read.digits is not None:
1588
+ types[name] = replace(held, digits=read.digits)
1589
+
1590
+ @staticmethod
1591
+ def _describe_table(conn, backend, table, query=None, written=None):
1592
+ """(column, type name) for a table on a SQL connection; `query`:
1593
+ the SQL query that is the source instead (an R2RML view,
1594
+ `TableMapping.source_query`), described by the database where it
1595
+ describes a query (DuckDB's DESCRIBE, Databricks' DESCRIBE QUERY).
1596
+ `written`: the name as the statement reads it (`Source.qualified`); a
1597
+ part not delimited there is looked up as the database folds it
1598
+ (`Dialect.fold_identifier`: PostgreSQL's catalog holds `Person` as
1599
+ person)."""
1600
+ def fetch(sql, params=()):
1601
+ from prograph.graph.drivers import fetch_rows
1602
+ return [(str(r[0]), str(r[1])) for r in fetch_rows(conn, sql, params)[1]]
1603
+
1604
+ schema_name, _, name = table.rpartition('.')
1605
+ if query is not None and backend == 'postgresql':
1606
+ return _describe_postgresql_query(conn, query)
1607
+ if query is not None and backend not in ('duckdb', 'databricks_sql'):
1608
+ raise NotImplementedError(f"no description of a query on {backend}")
1609
+ if backend == 'duckdb':
1610
+ from prograph.plan.sql.dialect import get_dialect_instance
1611
+ # The name as the statement reads it where it differs (a qualification: `written` is SQL text
1612
+ # already), else the source's own name, quoted.
1613
+ quoted = query or (written if written and written != table
1614
+ else get_dialect_instance("duckdb").quote_qualified(table))
1615
+ return [(r[0], r[1]) for r in conn.execute(f"DESCRIBE {quoted}").fetchall()]
1616
+ if backend == 'oracle':
1617
+ # Which NUMBER holds integers is oracle_graph.number_is_integer's
1618
+ # rule, the one the result types follow; the table's name is kept
1619
+ # in upper case unless it was created quoted.
1620
+ from prograph.graph.drivers import fetch_rows
1621
+ from prograph.graph.oracle_graph import number_digits, number_is_integer
1622
+ rows = fetch_rows(conn, "SELECT column_name, data_type, data_scale, data_precision FROM user_tab_columns "
1623
+ "WHERE UPPER(table_name) = :1", (name.upper(),))[1]
1624
+ described = []
1625
+ for column, kind, scale, precision in rows:
1626
+ digits = number_digits(precision, scale) if kind == 'NUMBER' else None
1627
+ if kind == 'NUMBER' and number_is_integer(scale):
1628
+ kind = 'BIGINT'
1629
+ elif digits:
1630
+ # A NUMBER(p, s) is the exact decimal of its digits, as its answers are read
1631
+ # (`oracle_graph.number_digits`), so the plan knows them.
1632
+ kind = f'DECIMAL({digits[0]},{digits[1]})'
1633
+ described.append((str(column), str(kind)))
1634
+ return described
1635
+ if backend in ('postgresql', 'mysql', 'sqlserver'):
1636
+ from prograph.plan.sql.dialect import get_dialect_instance
1637
+ marker = get_dialect_instance(backend).param_placeholder
1638
+ kind = "data_type"
1639
+ if backend == 'postgresql':
1640
+ # An array column's data_type is the one word ARRAY, whatever
1641
+ # it holds, which names no type: every array column was
1642
+ # untyped, and two of them compared with `=` were compared as
1643
+ # the database compares arrays. The element
1644
+ # type is the array type's own: `integer[]`, `text[]`. A
1645
+ # domain over an array reports its base type here too.
1646
+ # PostgreSQL keeps no dimensions: an `integer[][]` column is
1647
+ # `integer[]` to its catalog and to this.
1648
+ kind = ("CASE WHEN data_type = 'ARRAY' THEN (SELECT pg_catalog.format_type("
1649
+ "t.typelem, NULL) FROM pg_catalog.pg_type t JOIN pg_catalog.pg_namespace n "
1650
+ "ON n.oid = t.typnamespace WHERE t.typname = udt_name "
1651
+ "AND n.nspname = udt_schema) || '[]' "
1652
+ # A NUMERIC(p, s) column's digits, which its answers are read back at
1653
+ # (postgresql_graph.numeric_digits); an unconstrained NUMERIC has none.
1654
+ "WHEN data_type = 'numeric' AND numeric_precision IS NOT NULL "
1655
+ "AND numeric_scale BETWEEN 0 AND numeric_precision THEN "
1656
+ "'numeric(' || numeric_precision || ',' || numeric_scale || ')' "
1657
+ # A bit string (bit(n), even bit(1): PostgreSQL has no boolean in it) is its text.
1658
+ "WHEN data_type = 'bit' THEN "
1659
+ "'bit(' || character_maximum_length || ')' "
1660
+ # The citext extension's type, which the catalog calls USER-DEFINED.
1661
+ "WHEN data_type = 'USER-DEFINED' AND udt_name = 'citext' THEN 'citext' "
1662
+ "ELSE data_type END")
1663
+ sql = (f"SELECT column_name, {kind} FROM information_schema.columns "
1664
+ f"WHERE table_name = {marker(1)}")
1665
+ if backend == 'postgresql':
1666
+ # The table the statement reads, resolved as PostgreSQL resolves the written
1667
+ # name (to_regclass: its search path for a bare name, its folding), so a
1668
+ # same-named table in another schema lends no column or type.
1669
+ sql = (f"SELECT column_name, {kind} FROM information_schema.columns "
1670
+ "WHERE (table_schema, table_name) = (SELECT n.nspname, c.relname "
1671
+ "FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n "
1672
+ f"ON n.oid = c.relnamespace WHERE c.oid = to_regclass({marker(1)})) "
1673
+ "ORDER BY ordinal_position")
1674
+ rows = fetch(sql, (str(written or table),))
1675
+ unconstrained = [c for c, k in rows if k == "numeric"]
1676
+ if unconstrained:
1677
+ warn_unknown_digits(f"The table {str(written or table)!r}", unconstrained,
1678
+ "that declare no precision and scale")
1679
+ return rows
1680
+ fold = get_dialect_instance(backend).fold_identifier
1681
+ delimited = [p[:1] == '"' for p in str(written or table).split('.')]
1682
+ params = [name if delimited[-1] else fold(name)]
1683
+ if schema_name:
1684
+ sql += f" AND table_schema = {marker(2)}"
1685
+ part = schema_name.split('.')[-1]
1686
+ params.append(part if len(delimited) > 1 and delimited[-2] else fold(part))
1687
+ return fetch(sql, tuple(params))
1688
+ if backend == 'databricks_sql':
1689
+ # DESCRIBE TABLE answers for a table and for a view, by a two-
1690
+ # or a three-part name, with the whole type (`array<bigint>`,
1691
+ # `decimal(10,2)`); information_schema needs the catalog and
1692
+ # gives another vocabulary (LONG, SHORT). After the columns it
1693
+ # lists a partitioned table's partition columns again, under
1694
+ # lines that start with `#`. Without this branch a warehouse
1695
+ # engine knew no column's type, and an integer column divided
1696
+ # by an integer was not an integer division.
1697
+ from prograph.graph.drivers import fetch_rows
1698
+ from prograph.plan.sql.dialect import get_dialect_instance
1699
+ described = (f"DESCRIBE QUERY {query}" if query is not None else
1700
+ f"DESCRIBE TABLE {get_dialect_instance('databricks_sql').quote_qualified(table)}")
1701
+ columns = []
1702
+ for row in fetch_rows(conn, described)[1]:
1703
+ column, kind = str(row[0] or ''), str(row[1] or '')
1704
+ if not column or column.startswith('#'):
1705
+ break
1706
+ columns.append((column, kind))
1707
+ return columns
1708
+ return []
1709
+
1710
+ @_one_at_a_time
1711
+ def update_schema(
1712
+ self,
1713
+ add_nodes: Optional[List[NodeMapping]] = None,
1714
+ remove_node_labels: Optional[List[str]] = None,
1715
+ replace_nodes: Optional[List[NodeMapping]] = None,
1716
+ add_relationships: Optional[List[RelationshipMapping]] = None,
1717
+ remove_rel_types: Optional[List[Union[str, Tuple[str, str, str]]]] = None,
1718
+ add_dataframes: Optional[Dict[str, Any]] = None,
1719
+ add_table_stats: Optional[Dict[str, int]] = None,
1720
+ ) -> SchemaChangeSet:
1721
+ """Update the schema on a live engine without recreating it.
1722
+
1723
+ Mutates the shared GraphSchema in place and selectively invalidates
1724
+ caches. Only rebuilds VL indexes if relationship mappings changed.
1725
+
1726
+ Ordering: removes relationships first, then nodes, then adds
1727
+ replacements, then adds new mappings (prevents FK constraint errors).
1728
+
1729
+ Args:
1730
+ add_nodes: Node mappings to add
1731
+ remove_node_labels: Node labels to remove
1732
+ replace_nodes: Node mappings to replace (matched by label)
1733
+ add_relationships: Relationship mappings to add
1734
+ remove_rel_types: Relationship types to remove. Each entry is
1735
+ either a string (removes all mappings for that type) or a
1736
+ tuple of (rel_type, start_label, end_label) for targeted
1737
+ removal.
1738
+ add_dataframes: New DataFrames to inject into the engine
1739
+ add_table_stats: Table name -> row count for new tables
1740
+
1741
+ Returns:
1742
+ SchemaChangeSet describing what changed
1743
+
1744
+ Raises:
1745
+ ValueError: If required DataFrames are missing or a node label
1746
+ has dependent relationships
1747
+ """
1748
+ if self._schema is None:
1749
+ raise ValueError("Cannot update schema: engine has no schema")
1750
+
1751
+ changes = SchemaChangeSet()
1752
+ schema = self._schema
1753
+ add_dataframes = add_dataframes or {}
1754
+
1755
+ # Phase 1: Validate new mappings have DataFrames available
1756
+ graph = getattr(self.executor, 'graph', None)
1757
+ existing_dfs = getattr(graph, 'dataframes', {}) if graph else {}
1758
+
1759
+ for mapping in (add_nodes or []) + (replace_nodes or []) + (add_relationships or []):
1760
+ table = mapping.source_table
1761
+ if (
1762
+ table not in existing_dfs
1763
+ and table not in add_dataframes
1764
+ ):
1765
+ # For Spark backend, try loading from catalog
1766
+ if self.backend == 'spark' and self._spark is not None:
1767
+ try:
1768
+ add_dataframes[table] = self._spark.table(
1769
+ mapping.qualified_source_table
1770
+ )
1771
+ except Exception as e:
1772
+ raise ValueError(
1773
+ f"Table '{table}' not available in existing "
1774
+ f"dataframes, add_dataframes, or catalog: {e}"
1775
+ )
1776
+ else:
1777
+ raise ValueError(
1778
+ f"Table '{table}' not in existing dataframes "
1779
+ f"and not provided in add_dataframes"
1780
+ )
1781
+
1782
+ # The node mappings as they will be: refused before anything
1783
+ # changes if their IRIs could not name one node each.
1784
+ from types import SimpleNamespace
1785
+ from prograph.schema.iri import check_identities
1786
+ going = set(remove_node_labels or []) | {m.label for m in (replace_nodes or [])}
1787
+ check_identities(SimpleNamespace(node_mappings=(
1788
+ [m for m in schema.node_mappings if m.label not in going]
1789
+ + list(replace_nodes or []) + list(add_nodes or []))))
1790
+
1791
+ # Phase 2: Mutate schema (order matters)
1792
+
1793
+ # 2a. Remove relationships first
1794
+ for entry in (remove_rel_types or []):
1795
+ if isinstance(entry, tuple):
1796
+ rel_type, start_label, end_label = entry
1797
+ removed = schema.remove_relationship_mapping(
1798
+ rel_type, start_label=start_label, end_label=end_label
1799
+ )
1800
+ else:
1801
+ removed = schema.remove_relationship_mapping(entry)
1802
+ if removed:
1803
+ changes.removed_rel_types.add(
1804
+ entry if isinstance(entry, str) else entry[0]
1805
+ )
1806
+ for m in removed:
1807
+ changes.removed_tables.add(m.source_table)
1808
+
1809
+ # 2b. Remove nodes
1810
+ for label in (remove_node_labels or []):
1811
+ removed = schema.remove_node_mapping(label)
1812
+ if removed is not None:
1813
+ changes.removed_node_labels.add(label)
1814
+ changes.removed_tables.add(removed.source_table)
1815
+
1816
+ # 2c. Replace nodes
1817
+ for mapping in (replace_nodes or []):
1818
+ old = schema.replace_node_mapping(mapping)
1819
+ if old is not None:
1820
+ changes.replaced_node_labels.add(mapping.label)
1821
+ if old.source_table != mapping.source_table:
1822
+ changes.removed_tables.add(old.source_table)
1823
+ changes.added_tables.add(mapping.source_table)
1824
+ else:
1825
+ changes.added_node_labels.add(mapping.label)
1826
+ changes.added_tables.add(mapping.source_table)
1827
+
1828
+ # 2d. Add new nodes
1829
+ for mapping in (add_nodes or []):
1830
+ schema.add_node_mapping(mapping)
1831
+ changes.added_node_labels.add(mapping.label)
1832
+ changes.added_tables.add(mapping.source_table)
1833
+
1834
+ # 2e. Add new relationships
1835
+ for mapping in (add_relationships or []):
1836
+ schema.add_relationship_mapping(mapping)
1837
+ changes.added_rel_types.add(mapping.rel_type)
1838
+ changes.added_tables.add(mapping.source_table)
1839
+
1840
+ # Phase 3: Inject new DataFrames
1841
+ if add_dataframes and graph is not None:
1842
+ graph.dataframes.update(add_dataframes)
1843
+ # Also update executor-level dataframes if present
1844
+ if add_dataframes and hasattr(self.executor, 'dataframes'):
1845
+ self.executor.dataframes.update(add_dataframes)
1846
+ if add_dataframes:
1847
+ if self._dataframes is None:
1848
+ self._dataframes = {}
1849
+ self._dataframes.update(add_dataframes)
1850
+
1851
+ # Phase 4: Add table stats
1852
+ if add_table_stats:
1853
+ for table_name, row_count in add_table_stats.items():
1854
+ schema.set_table_stats(table_name, row_count=row_count)
1855
+
1856
+ # Phase 5: Invalidate caches
1857
+ if graph is not None and hasattr(graph, '_invalidate_caches'):
1858
+ graph._invalidate_caches(changes)
1859
+
1860
+ # Rebind ontology if present and any mapping changed
1861
+ if changes.has_changes and self._ontology is not None and schema is not None:
1862
+ self._ontology.bind_to_schema(schema, strict=False)
1863
+
1864
+ self._record_column_types()
1865
+ return changes
1866
+
1867
+ def add_mapping(
1868
+ self,
1869
+ mapping: Union[NodeMapping, RelationshipMapping],
1870
+ dataframe: Optional[Any] = None,
1871
+ row_count: Optional[int] = None,
1872
+ ) -> None:
1873
+ """Add a single node or relationship mapping.
1874
+
1875
+ Convenience wrapper around update_schema().
1876
+
1877
+ Args:
1878
+ mapping: A NodeMapping or RelationshipMapping to add
1879
+ dataframe: Optional DataFrame for the mapping's source table
1880
+ row_count: Optional row count for broadcast join decisions
1881
+ """
1882
+ add_dfs = {mapping.source_table: dataframe} if dataframe is not None else None
1883
+ add_stats = {mapping.source_table: row_count} if row_count is not None else None
1884
+
1885
+ if isinstance(mapping, NodeMapping):
1886
+ self.update_schema(
1887
+ add_nodes=[mapping],
1888
+ add_dataframes=add_dfs,
1889
+ add_table_stats=add_stats,
1890
+ )
1891
+ elif isinstance(mapping, RelationshipMapping):
1892
+ self.update_schema(
1893
+ add_relationships=[mapping],
1894
+ add_dataframes=add_dfs,
1895
+ add_table_stats=add_stats,
1896
+ )
1897
+ else:
1898
+ raise TypeError(
1899
+ f"Expected NodeMapping or RelationshipMapping, got {type(mapping).__name__}"
1900
+ )
1901
+
1902
+ def remove_mapping(
1903
+ self,
1904
+ label_or_type: str,
1905
+ kind: str = 'node',
1906
+ ) -> None:
1907
+ """Remove a single mapping by label (node) or type (relationship).
1908
+
1909
+ Convenience wrapper around update_schema().
1910
+
1911
+ Args:
1912
+ label_or_type: The node label or relationship type to remove
1913
+ kind: 'node' or 'relationship'
1914
+
1915
+ Raises:
1916
+ ValueError: If kind is not 'node' or 'relationship', or if
1917
+ the node has dependent relationships
1918
+ """
1919
+ if kind == 'node':
1920
+ self.update_schema(remove_node_labels=[label_or_type])
1921
+ elif kind == 'relationship':
1922
+ self.update_schema(remove_rel_types=[label_or_type])
1923
+ else:
1924
+ raise ValueError(f"kind must be 'node' or 'relationship', got '{kind}'")
1925
+
1926
+
1927
+ @property
1928
+ def ontology(self) -> Optional['OntologyLayer']:
1929
+ """Get the ontology layer, if configured."""
1930
+ return self._ontology
1931
+
1932
+ @property
1933
+ def profile(self) -> Optional[str]:
1934
+ """The OWL profile the ontology is reasoned under (``'ql'`` or ``'rl'``); None with no
1935
+ ontology, where a query matches simply."""
1936
+ return self._profile
1937
+
1938
+ def ontology_report(self):
1939
+ """Which W3C OWL 2 profiles the engine's ontology fits and which of its axioms the engine answers
1940
+ (``prograph.plan.ir.profiles.OntologyReport``): every axiom with the profiles it belongs to
1941
+ (``profiles``: of ``'ql'``, ``'rl'``; ``why`` it is outside the others) and those ProGraph answers it
1942
+ under today (``answered``); ``fits`` ('both', 'ql', 'rl' or 'neither'); ``refused``, the axioms the
1943
+ engine's profile does not answer, and ``dropped``, those ``on_unsupported="ignore"`` dropped.
1944
+ None with no ontology."""
1945
+ if self._ontology is None:
1946
+ return None
1947
+ from prograph.plan.ir.entry import profile_report
1948
+ return profile_report(self)[1]
1949
+
1950
+ @property
1951
+ def default_shapes(self) -> Optional[str]:
1952
+ """Operator-configured SHACL shapes for the runtime guardrail.
1953
+
1954
+ A Turtle string or path to a ``.ttl`` file, or ``None``. When set,
1955
+ the MCP server exposes a zero-argument ``check_data_quality`` tool
1956
+ that validates the engine's current data against these shapes, the
1957
+ deterministic honesty boundary, surfaced to agents automatically.
1958
+ """
1959
+ return self._default_shapes
1960
+
1961
+ def iri_of(self, label: str, key: Any) -> str:
1962
+ """The IRI of the node of `label` with this key: a value, or a
1963
+ tuple in ``id_columns`` order for a composite key. The one way a
1964
+ caller holding a key gets the identity the semantic layer uses
1965
+ (SHACL focus nodes, SPARQL results, provenance); the inverse of
1966
+ :meth:`resolve_iri`. The node is not looked up.
1967
+
1968
+ Raises ``ValueError`` for an unknown label, a key of the wrong
1969
+ arity or a missing key part, and ``UnstableIdentityError`` where
1970
+ the label's ``iri_template`` reads columns other than its key.
1971
+ """
1972
+ return self.iris_of(label, [key])[0]
1973
+
1974
+ def iris_of(self, label: str, keys: Sequence[Any]) -> List[str]:
1975
+ """:meth:`iri_of` for many keys of one label: one IRI per key, in
1976
+ order."""
1977
+ from prograph.schema.iri import node_iri_from_id, require_key_template
1978
+ from prograph.schema.types import is_missing
1979
+
1980
+ if self._schema is None:
1981
+ raise ValueError("iri_of needs a schema")
1982
+ mapping = self._schema.get_node_mapping(self._schema.require_name('label', label))
1983
+ require_key_template(mapping)
1984
+ arity = len(mapping.id_columns)
1985
+ iris = []
1986
+ for key in keys:
1987
+ parts = tuple(key) if isinstance(key, (tuple, list)) else (key,)
1988
+ if len(parts) != arity:
1989
+ raise ValueError(
1990
+ f"{label} is keyed by {', '.join(mapping.id_columns)}: "
1991
+ f"a key has {arity} part(s), got {key!r}")
1992
+ if any(is_missing(part) for part in parts):
1993
+ raise ValueError(f"{key!r} is no key of {label}: a part of it is missing")
1994
+ iris.append(node_iri_from_id(mapping, parts if arity > 1 else parts[0]))
1995
+ return iris
1996
+
1997
+ def set_properties(
1998
+ self,
1999
+ label: str,
2000
+ identities: Sequence[Any],
2001
+ values: Dict[str, Any],
2002
+ *,
2003
+ by: str = "key",
2004
+ ) -> int:
2005
+ """Set properties on the nodes of `label` named by `identities`,
2006
+ and return how many rows were written.
2007
+
2008
+ Args:
2009
+ label: The nodes' label. Its mapping must be ``mutable``.
2010
+ identities: Keys (a value, or a tuple in ``id_columns`` order
2011
+ for a composite key), or with ``by="iri"`` the nodes' IRIs
2012
+ (:meth:`iri_of`). An IRI that names no node of the label
2013
+ writes nothing.
2014
+ values: Property name to literal value (None, bool, int, float
2015
+ or str). Names are checked against the schema.
2016
+ by: ``"key"`` or ``"iri"``: what `identities` holds. It is
2017
+ stated, not guessed, because a key may be text that looks
2018
+ like an IRI.
2019
+
2020
+ Only rows the label's mapping owns are written (its
2021
+ ``filter_predicate``, on a table labels share), and never a row a
2022
+ reasoner materialized (``_inferred``): the rule of every write by
2023
+ identity (:mod:`prograph.writes`). The key and the columns of
2024
+ the ``filter_predicate`` cannot be set. On a SQL database it is one
2025
+ ``UPDATE``; on Spark, a Delta ``UPDATE``; on DataFrames, the frame
2026
+ is changed in place.
2027
+
2028
+ It is the one write ProGraph makes to a table it maps on request,
2029
+ and only to a mapping marked ``mutable`` (decided 2026-11-04); a
2030
+ query's CREATE, SET or DELETE and a SPARQL Update stay refused
2031
+ (:mod:`prograph.read_only`).
2032
+
2033
+ Raises ``ReadOnlyGraphError`` for a mapping that is not mutable and
2034
+ on the memory backend (which maps no table), ``ValueError`` for an
2035
+ unknown label or property, a key of the wrong arity, or a value
2036
+ that is no literal.
2037
+ """
2038
+ from prograph.r2rml.schema import require_property_graph
2039
+ require_property_graph(self._schema, "set_properties")
2040
+ from prograph.writes import set_properties
2041
+ return set_properties(self, label, identities, values, by=by)
2042
+
2043
+ def own_rows_sql(self, label: str, columns: Optional[Sequence[str]] = None) -> str:
2044
+ """SQL text, in the backend's dialect, selecting the rows that are
2045
+ this label's nodes: its table, the rows its mapping owns (its
2046
+ ``filter_predicate``, on a table labels share), asserted (no row a
2047
+ reasoner materialized). ``columns`` are column names of the table
2048
+ and default to the label's key columns.
2049
+
2050
+ For a caller that writes its own SQL for :meth:`execute_sql` and
2051
+ has to stay on one label's nodes: join against it as a subquery,
2052
+ ``... JOIN (<own_rows_sql>) o ON o.id = q.key``. It is the read
2053
+ side of the rule every write by identity follows (``prograph/writes.py``), so what such a query scores is what
2054
+ :meth:`set_properties` can write.
2055
+
2056
+ Raises ``ValueError`` for an unknown label, for a column the
2057
+ label's table does not have.
2058
+ """
2059
+ from prograph.plan.sql.dialect import get_dialect_instance
2060
+ from prograph.writes import INFERRED_COLUMN, own_rows_sql
2061
+
2062
+ if self._schema is None:
2063
+ raise ValueError("own_rows_sql needs a schema")
2064
+ mapping = self._schema.get_node_mapping(self._schema.require_name('label', label))
2065
+ graph = getattr(self.executor, 'graph', None)
2066
+ dialect_name = graph.sql_dialect() if hasattr(graph, 'sql_dialect') else None
2067
+ if dialect_name is None:
2068
+ raise ValueError(
2069
+ f"own_rows_sql needs a SQL connection, and the {self.backend} backend has none")
2070
+ frames = getattr(graph, 'dataframes', None) if self._frame_backed else None
2071
+ if frames is not None and mapping.source_table in frames:
2072
+ present = list(frames[mapping.source_table].columns)
2073
+ else:
2074
+ table = get_dialect_instance(dialect_name).quote_qualified(
2075
+ mapping.qualified_source_table)
2076
+ present = graph._execute_sql_arrow(f"SELECT * FROM {table} WHERE 1 = 0").column_names
2077
+ unknown = [c for c in (columns or ()) if c not in present]
2078
+ if unknown:
2079
+ raise ValueError(
2080
+ f"{', '.join(unknown)} is not a column of {mapping.source_table}, "
2081
+ f"the table of {label}")
2082
+ return own_rows_sql(mapping, get_dialect_instance(dialect_name),
2083
+ INFERRED_COLUMN in present, columns)
2084
+
2085
+ def resolve_iri(self, iri: str) -> Optional[Dict[str, Any]]:
2086
+ """The node an instance IRI names, or None if there is none.
2087
+
2088
+ The inverse of the IRIs the semantic layer mints (SHACL focus nodes,
2089
+ SPARQL results, provenance keys; :mod:`prograph.schema.iri`): an
2090
+ action layer holding a violation's focus node gets back what a Cypher
2091
+ template needs, ``{"label": str, "labels": [str, ...], "id": value}``
2092
+ (``id`` a tuple for a composite key). The node is looked up, so the id
2093
+ comes back with its column's type and a well-formed IRI of a missing
2094
+ node resolves to None. ``labels`` lists every label the row carries,
2095
+ for mappings sharing a table. Synthesized IRIs are read under the
2096
+ schema's ``base_iri``.
2097
+ """
2098
+ from prograph.schema.iri import parse_node_iri
2099
+
2100
+ from prograph.plan.ir.entry import node_keys
2101
+ found: Dict[str, Any] = {}
2102
+ labels: List[str] = []
2103
+ # Over the source maps: each node map's rows whose subject is the IRI, R2RML's
2104
+ # (labelled by the map's id) and YAML's alike, the key as its columns hold it.
2105
+ for m, answer, names in node_keys(self, iri):
2106
+ rows = self._convert_results(answer, "list") # the answer's one conversion
2107
+ if not rows:
2108
+ continue
2109
+ values = tuple(rows[0][n] for n in names)
2110
+ label = m.label or m.id
2111
+ labels.append(label)
2112
+ found.setdefault("label", label)
2113
+ found.setdefault("id", values[0] if len(values) == 1 else values)
2114
+ if not labels:
2115
+ return None
2116
+ found["labels"] = labels
2117
+ return found
2118
+
2119
+ # =========================================================================
2120
+ # OWL entailment
2121
+ # =========================================================================
2122
+
2123
+ @_one_at_a_time
2124
+ def materialize_entailments(self, max_rounds: int = 64):
2125
+ """Materialize the ontology's OWL 2 RL entailments into the owned store's RL graph
2126
+ (``urn:prograph:store:rl``: in the default graph of an engine of this cache key, never a named
2127
+ graph), as one write step: the rules' closure over the
2128
+ graph SPARQL answers now, written as its difference from what the RL graph holds (asserts for
2129
+ new facts, retracts for facts no longer derived), with each mapped table's version as the run
2130
+ read it (its declared version column, its Delta version, or a small table's checksum). A query
2131
+ refreshes the facts itself where a table proves a change (``freshness``); under
2132
+ ``freshness='last_run'`` call this after the tables change. Rules applied: prp-inv1/2, prp-symp, prp-trp, prp-spo2,
2133
+ cls-hv1/2, cls-svf1/2, cls-avf, cls-int1/2, cls-uni, cls-oo (subClassOf, subPropertyOf,
2134
+ equivalence, domain and range stay query rewrites). owl:sameAs: the graph's links and those
2135
+ prp-fp, prp-ifp, prp-key, cls-maxc2 and cls-maxqc3/4 derive are closed into components, written
2136
+ as a new generation of the store's alias table, through which queries match (``sameas``).
2137
+ ``max_rounds``: the rounds of the fixpoint at most; more raise ``MaterializationBudgetExceeded``
2138
+ with nothing written.
2139
+
2140
+ Returns a ``MaterializationReport``: statements asserted and retracted per rule, the rounds,
2141
+ the consistency rules (cax-dw, prp-irp, prp-asyp, cls-nothing2, cls-maxc1 ...) that
2142
+ matched, each with a sample (``report.consistent``), and the owl:sameAs components merged and
2143
+ left unmerged (``report.unmerged``).
2144
+
2145
+ Where the store already holds a run under this engine's key (its rules, ontology and mapping) and every
2146
+ mapped table is proven unchanged since it (a declared version column, a Delta version, or a small
2147
+ table's checksum), nothing is recomputed: the call returns that run's record (``seq`` None, ``rounds``
2148
+ 0, nothing asserted or retracted, its ``digest``, ``ran_at``, ``sources`` and consistency verdict) and
2149
+ settles the stored verdict as the engine's, raising under ``inconsistency='error'`` if it is
2150
+ inconsistent. A table no version proves counts as changed, so the closure is computed. ``max_rounds`` is
2151
+ checked against the recorded run's rounds: a budget below them (or a run that recorded none) computes the
2152
+ closure, so the budget error is raised as before."""
2153
+ from prograph.plan.ir.dataset import provably_current
2154
+ if self._store is not None and self._ontology is not None and self._profile == "rl" and max_rounds >= 1 \
2155
+ and provably_current(self):
2156
+ from prograph.plan.ir.consistency import recorded, settle
2157
+ from prograph.plan.ir.owl_rl import reused_report
2158
+ held = recorded(self, "by this call to materialize_entailments()")
2159
+ report = reused_report(self, self._rl_rules(), held, max_rounds)
2160
+ if report is not None:
2161
+ settle(self, held)
2162
+ return report
2163
+ return self._materialize_run(max_rounds)
2164
+
2165
+ @_one_at_a_time
2166
+ def _materialize_run(self, max_rounds: int = 64):
2167
+ """`materialize_entailments`' run: the closure computed and its difference written, whether or not the
2168
+ store's facts are current."""
2169
+ if self._store is None or self._ontology is None:
2170
+ raise ValueError("materialize_entailments() writes the ontology's OWL 2 RL entailments into the "
2171
+ "owned store: give the engine an ontology= and a store=")
2172
+ if self._profile != "rl":
2173
+ raise ValueError("materialize_entailments() stores OWL 2 RL's facts for an engine that reads them: "
2174
+ "build the engine with profile='rl' (it materializes when it is built)")
2175
+ from prograph.plan.ir.consistency import rl_report, settle
2176
+ from prograph.plan.ir.entry import ontology_of
2177
+ from prograph.plan.ir.owl_rl import materialize
2178
+ ontology_of(self) # the ontology read (its profile checked at construction) before anything runs
2179
+ report = materialize(self, self._rl_rules(), max_rounds=max_rounds)
2180
+ # Written with the facts; raised here under inconsistency='error'.
2181
+ settle(self, rl_report(report.inconsistencies, report.ran_at, report.not_run))
2182
+ return report
2183
+
2184
+ def consistency_report(self):
2185
+ """Whether the ontology and the data are consistent under the engine's profile
2186
+ (``prograph.plan.ir.consistency.ConsistencyReport``): ``consistent``, and each rule or check that
2187
+ matched (``inconsistencies``: the OWL 2 RL rule id or the QL check, the axiom in short Turtle, how many
2188
+ facts and a few of them), with ``text()`` the whole in words. Under profile='rl' the verdict of the
2189
+ last materialization (stored with its facts), under profile='ql' the check made when the engine was
2190
+ built. Under ``inconsistency='error'`` (the default) an engine with an inconsistent one is never built,
2191
+ so it is read under ``inconsistency='report'``. None with no ontology."""
2192
+ if self._ontology is None:
2193
+ return None
2194
+ held = self.__dict__.get("_consistency")
2195
+ if held is None and self._profile == "rl":
2196
+ from prograph.plan.ir.consistency import recorded
2197
+ held = recorded(self)
2198
+ return held
2199
+
2200
+ def explain_identity(self, a: str, b: str):
2201
+ """Why two IRIs are one entity under owl:sameAs, or not (``profile='rl'``): an
2202
+ ``IdentityExplanation`` with a shortest chain of links from ``a`` to ``b`` (each a sameAs triple of
2203
+ the graph, by the maps that may give it, or a link an OWL 2 RL rule derived: prp-fp, prp-ifp,
2204
+ prp-key, cls-maxc2, cls-maxqc3/4, with its axiom), each IRI's representative and component size,
2205
+ and, where a component holding either was left unmerged (inconsistent, or past the cap), why.
2206
+ Read from the last materialization's components, in the default graph."""
2207
+ from prograph.plan.ir.identity import explain
2208
+ return explain(self, a, b)
2209
+
2210
+ def _rl_rules(self):
2211
+ """The ontology's OWL 2 RL rules (`owl_rl_rules.compile_rules`), compiled once per ontology content."""
2212
+ from prograph.plan.ir.owl_rl_rules import compile_rules
2213
+ from prograph.plan.ir.entry import profile_report
2214
+ # The graph the profile check kept: the axioms on_unsupported="ignore" dropped are not compiled.
2215
+ key, _, graph = profile_report(self)
2216
+ held = self.__dict__.get("_rl_compiled")
2217
+ if held is None or held[0] != key:
2218
+ held = (key, compile_rules(graph))
2219
+ self.__dict__["_rl_compiled"] = held
2220
+ return held[1]
2221
+
2222
+ def _materialize_on_open(self):
2223
+ """The stored facts made current when the engine is built: a run where the store holds none under the
2224
+ engine's key, or a source or the store changed since that run (`dataset.stale`).
2225
+ Under freshness='last_run' a store that cannot be written keeps its last run."""
2226
+ from prograph.plan.ir.dataset import engine_key, recorded_key, stale_sources, stored_sources
2227
+ from prograph.plan.ir.entry import ontology_of
2228
+ from prograph.plan.ir.freshness import refresh
2229
+ from prograph.utils.exceptions import StoredFactsNotCurrentError
2230
+ from prograph.plan.ir.consistency import recorded, settle
2231
+ ontology_of(self)
2232
+ stored_sources(self) # a version column on a table no map reads is refused first
2233
+ changed = stale_sources(self)
2234
+ if changed:
2235
+ if recorded_key(self._store) != engine_key(self):
2236
+ self._materialize_run()
2237
+ return
2238
+ try:
2239
+ refresh(self, changed)
2240
+ return
2241
+ except StoredFactsNotCurrentError:
2242
+ if self._freshness != "last_run":
2243
+ raise
2244
+ # The facts are current (or read as the last run's): that run's verdict, not checked again.
2245
+ settle(self, recorded(self))
2246
+
2247
+ def shapes(self, format: str = "graph", *, shape_base: Optional[str] = None) -> Any:
2248
+ """SHACL shapes generated from the mapping and the database's catalog (`prograph.shacl.generate`):
2249
+ one `sh:NodeShape` per class, a property shape per predicate with its datatype, node kind or class,
2250
+ and `sh:minCount 1` / `sh:maxCount 1` only where the mapping or the catalog proves them. They hold
2251
+ on the graph SPARQL answers, so `validate_shapes(engine.shapes())` conforms; add constraints to them to
2252
+ check more. Over frames or Spark (no catalog) no NOT NULL is known and no `sh:minCount` is written.
2253
+
2254
+ Args:
2255
+ format: "graph" gives an rdflib Graph; an rdflib serialization name ("turtle", ...) the text.
2256
+ shape_base: The prefix of the node shapes' IRIs (default `urn:prograph:shape:`).
2257
+ """
2258
+ from prograph.plan.ir.entry import source_maps
2259
+ from prograph.plan.ir.keys import trusted_keys
2260
+ from prograph.shacl.generate import DEFAULT_SHAPE_BASE, generate_shapes
2261
+ from prograph.r2rml import catalog
2262
+ from prograph.schema import xsd
2263
+
2264
+ schema = self.schema
2265
+ maps = source_maps(schema)
2266
+ graph = getattr(getattr(self, "executor", None), "graph", None)
2267
+ read: Dict[Any, Any] = {}
2268
+
2269
+ def not_null(source: Any) -> Any:
2270
+ if self._connection is None or self.backend not in catalog.DIALECTS or source.query is not None:
2271
+ return None
2272
+ qualifier, _, table = source.table.rpartition(".")
2273
+ if qualifier not in read:
2274
+ try:
2275
+ read[qualifier] = catalog.read_not_null(self._connection, self.backend, qualifier or None)
2276
+ except Exception: # noqa: BLE001 - a catalog that cannot be read proves nothing
2277
+ read[qualifier] = None
2278
+ found = read[qualifier]
2279
+ if found is None:
2280
+ return None
2281
+ if table in found:
2282
+ return found[table]
2283
+ folded = [v for k, v in found.items() if k.lower() == table.lower()]
2284
+ return folded[0] if len(folded) == 1 else frozenset()
2285
+
2286
+ try:
2287
+ from prograph.plan.sql.dialect import get_dialect_instance
2288
+ fold = get_dialect_instance(self.backend).fold_identifier
2289
+ except Exception: # noqa: BLE001 - a backend with no dialect keeps names as written
2290
+ fold = lambda name: name # noqa: E731
2291
+
2292
+ graph_out = generate_shapes(
2293
+ maps, shape_base=shape_base or DEFAULT_SHAPE_BASE, not_null=not_null,
2294
+ column_datatype=lambda t, c: _shape_datatype(schema.table_column_type(t, c)),
2295
+ keys=lambda m: trusted_keys(graph, m, maps), fold=fold)
2296
+ return graph_out if format == "graph" else graph_out.serialize(format=format)
2297
+
2298
+ def validate_shapes(
2299
+ self,
2300
+ shapes: Union[str, "Path", "rdflib.Graph"],
2301
+ *,
2302
+ inference: str = "none",
2303
+ ) -> "ValidationReport":
2304
+ """Validate the RDF graph SPARQL answers against SHACL shapes.
2305
+
2306
+ The data graph is ``CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }`` over the default graph, the
2307
+ graph :meth:`shapes` holds on, for any mapping (an R2RML one too). It is what a SPARQL query
2308
+ sees: the mapping's joins, its quoted-case columns and its typed values are all in it.
2309
+
2310
+ Args:
2311
+ shapes: Path to a ``.ttl`` file, an ``rdflib.Graph``, or a Turtle string of SHACL shapes.
2312
+ inference: ``pyshacl``'s pre-validation entailment profile, ``'none'`` | ``'rdfs'`` |
2313
+ ``'owlrl'`` | ``'both'``, applied to that graph.
2314
+
2315
+ Returns:
2316
+ A :class:`prograph.shacl.ValidationReport` with structured violations plus the canonical
2317
+ SHACL report graph as Turtle.
2318
+ """
2319
+ import warnings
2320
+ from prograph.plan.ir.cost import ProGraphCostWarning
2321
+ from prograph.shacl import validate
2322
+ from prograph.shacl.validator import _coerce_shapes_graph
2323
+ with warnings.catch_warnings():
2324
+ warnings.simplefilter("ignore", ProGraphCostWarning)
2325
+ try:
2326
+ data_graph = self.execute("CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }", language="sparql")
2327
+ except Exception as error:
2328
+ error.add_note("validate_shapes reads the mapped graph with CONSTRUCT { ?s ?p ?o } WHERE "
2329
+ "{ ?s ?p ?o }; the error above is that query's on this engine.")
2330
+ raise
2331
+ report = validate(data_graph, shapes=_coerce_shapes_graph(shapes), inference=inference)
2332
+ self._shacl_last_report = report
2333
+ return report
2334
+
2335
+ @property
2336
+ def shacl_last_report(self) -> Optional["ValidationReport"]:
2337
+ """The :class:`ValidationReport` from the most recent ``validate_shapes`` call.
2338
+
2339
+ ``None`` if validation has never been run on this engine.
2340
+ """
2341
+ return getattr(self, "_shacl_last_report", None)
2342
+
2343
+ def generate_views(
2344
+ self,
2345
+ target_catalog: Optional[str] = None,
2346
+ target_schema: str = "prograph_views",
2347
+ include_labels: Optional[List[str]] = None,
2348
+ exclude_labels: Optional[List[str]] = None,
2349
+ dialect: Optional[Any] = None,
2350
+ ) -> List[Any]:
2351
+ """Generate Genie-consumable SQL views from this engine's schema.
2352
+
2353
+ Wraps :func:`prograph.genie.generate_views`. Returns a list of
2354
+ :class:`prograph.genie.ViewDefinition` instances ready to be
2355
+ rendered as ``CREATE OR REPLACE VIEW`` statements. The engine's
2356
+ ontology (if any) drives subclass-closure and OWL-defined-class
2357
+ view emission.
2358
+
2359
+ This method only *builds* view definitions; it does **not**
2360
+ execute any DDL against the backend. To create the views on the
2361
+ warehouse, use :meth:`write_views` (which calls this method
2362
+ internally) or hand the result to
2363
+ :func:`prograph.genie.view_writer.write_views`.
2364
+
2365
+ Args:
2366
+ target_catalog / target_schema: Naming convention metadata
2367
+ passed through to the writer if you later call
2368
+ :meth:`write_views`. Ignored at definition-build time;
2369
+ provided here so callers can construct the writer
2370
+ arguments without re-deriving them.
2371
+ include_labels / exclude_labels: Filter node mappings by
2372
+ label before generating views.
2373
+ dialect: Optional :class:`DialectConfig` override.
2374
+
2375
+ Returns:
2376
+ List of :class:`ViewDefinition`.
2377
+ """
2378
+ from prograph.genie import generate_views as _gen
2379
+ return _gen(
2380
+ self._schema,
2381
+ ontology=self._ontology,
2382
+ dialect=self._view_dialect(dialect),
2383
+ include_labels=include_labels,
2384
+ exclude_labels=exclude_labels,
2385
+ )
2386
+
2387
+ def write_views(
2388
+ self,
2389
+ target_catalog: Optional[str] = None,
2390
+ target_schema: str = "prograph_views",
2391
+ include_labels: Optional[List[str]] = None,
2392
+ exclude_labels: Optional[List[str]] = None,
2393
+ dialect: Optional[Any] = None,
2394
+ ensure_schema: bool = True,
2395
+ ) -> List[Tuple[str, str]]:
2396
+ """Generate Genie views and execute ``CREATE OR REPLACE VIEW`` against this engine.
2397
+
2398
+ Only available on SQL backends that own a connection
2399
+ (``databricks_sql``, ``duckdb``, ``postgresql``, ``sqlserver``,
2400
+ ``mysql``, ``oracle``). Other backends raise :class:`ValueError`
2401
+ because there's no SQL connection to write views against.
2402
+
2403
+ When ``dialect`` is left ``None`` it defaults to the dialect of
2404
+ this engine's backend, so the emitted DDL uses the backend's own
2405
+ identifier quoting (e.g. double quotes on DuckDB rather than
2406
+ Spark backticks).
2407
+
2408
+ Args:
2409
+ target_catalog / target_schema: Where the views live.
2410
+ Convention is ``<catalog>.prograph_views.*`` on
2411
+ three-part backends; ``prograph_views.*`` on two-part
2412
+ backends.
2413
+ ensure_schema: Issue ``CREATE SCHEMA IF NOT EXISTS`` before
2414
+ creating views. Set ``False`` if the schema is managed
2415
+ externally.
2416
+ (others): See :meth:`generate_views`.
2417
+
2418
+ Returns:
2419
+ List of ``(view_name, fully_qualified_name)`` tuples for
2420
+ the views actually created. Empty list if the schema has
2421
+ no mappings; raises if the backend doesn't support DDL.
2422
+ """
2423
+ from prograph.genie.view_writer import write_views as _write
2424
+
2425
+ connection = self._resolve_sql_connection()
2426
+ if connection is None and self.backend == "duckdb":
2427
+ raise ValueError(
2428
+ "write_views needs a DuckDB database to write the views "
2429
+ "into: construct the engine with connection= (a DuckDB "
2430
+ "connection) instead of dataframes=. Over DataFrames, call "
2431
+ "generate_views() to get the SQL."
2432
+ )
2433
+ if connection is None:
2434
+ raise ValueError(
2435
+ f"write_views is not supported on the {self.backend!r} "
2436
+ "backend. It requires an open SQL connection; use "
2437
+ "'databricks_sql', 'duckdb', 'postgresql', 'sqlserver', "
2438
+ "'mysql', or 'oracle'. On 'spark', or 'duckdb' over DataFrames, call generate_views() to get the SQL and execute "
2439
+ "it through your own client."
2440
+ )
2441
+
2442
+ dialect = self._view_dialect(dialect)
2443
+ views = self.generate_views(
2444
+ target_catalog=target_catalog,
2445
+ target_schema=target_schema,
2446
+ include_labels=include_labels,
2447
+ exclude_labels=exclude_labels,
2448
+ dialect=dialect,
2449
+ )
2450
+ return _write(
2451
+ views,
2452
+ connection,
2453
+ target_catalog=target_catalog,
2454
+ target_schema=target_schema,
2455
+ dialect=dialect,
2456
+ ensure_schema=ensure_schema,
2457
+ )
2458
+
2459
+ def emit_genie_space(
2460
+ self,
2461
+ name: Optional[str] = None,
2462
+ description: str = "",
2463
+ target_catalog: Optional[str] = None,
2464
+ target_schema: str = "prograph_views",
2465
+ dialect: Optional[Any] = None,
2466
+ include_labels: Optional[List[str]] = None,
2467
+ exclude_labels: Optional[List[str]] = None,
2468
+ mcp_servers: Optional[List[Any]] = None,
2469
+ summary_instructions: bool = False,
2470
+ max_sample_queries: int = 25,
2471
+ ) -> Any:
2472
+ """Build a complete :class:`GenieSpace` from this engine's schema.
2473
+
2474
+ Bundles :meth:`generate_views`, instruction generation, and
2475
+ sample-query generation into one :class:`GenieSpace` ready to
2476
+ serialize as YAML. Pure function of the engine's schema +
2477
+ ontology; no DDL or network I/O.
2478
+
2479
+ Pair with :func:`prograph.genie.render_space_yaml` /
2480
+ :func:`prograph.genie.write_space_yaml` to write the result to
2481
+ disk for import into Databricks Genie.
2482
+
2483
+ Args:
2484
+ name: Display name for the Space. Defaults to the schema's
2485
+ ``name`` attribute when set.
2486
+ description: Free-form description.
2487
+ target_catalog / target_schema: Where generated views live
2488
+ in the warehouse, the Space references views under
2489
+ this prefix.
2490
+ dialect: SQL dialect for view bodies. Defaults to
2491
+ Spark/Databricks SQL backticks.
2492
+ include_labels / exclude_labels: Filter on NodeMapping
2493
+ labels.
2494
+ mcp_servers: Optional list of
2495
+ :class:`McpServerRegistration` for the Path B escape
2496
+ hatch.
2497
+ summary_instructions: When ``True``, emit just the
2498
+ overview section.
2499
+ max_sample_queries: Cap on sample queries.
2500
+
2501
+ Returns:
2502
+ A :class:`prograph.genie.GenieSpace`.
2503
+ """
2504
+ from prograph.genie import emit_genie_space as _emit
2505
+ return _emit(
2506
+ schema=self._schema,
2507
+ ontology=self._ontology,
2508
+ name=name,
2509
+ description=description,
2510
+ target_catalog=target_catalog,
2511
+ target_schema=target_schema,
2512
+ dialect=self._view_dialect(dialect),
2513
+ include_labels=include_labels,
2514
+ exclude_labels=exclude_labels,
2515
+ mcp_servers=mcp_servers,
2516
+ summary_instructions=summary_instructions,
2517
+ max_sample_queries=max_sample_queries,
2518
+ )
2519
+
2520
+ def _view_dialect(self, dialect: Optional[Any]) -> Any:
2521
+ """Resolve the SQL dialect to use for view generation/writing.
2522
+
2523
+ When the caller passes an explicit ``dialect``, use it. Otherwise
2524
+ default to the dialect that matches this engine's backend, so
2525
+ ``generate_views`` / ``write_views`` emit valid SQL for the
2526
+ backend they run against (e.g. double-quoted identifiers on
2527
+ DuckDB, not Spark backticks). A backend with no dialect of its own
2528
+ falls back to the Spark dialect.
2529
+ """
2530
+ if dialect is not None:
2531
+ return dialect
2532
+ from prograph.plan.sql.dialect import get_dialect, SPARK_DIALECT
2533
+ try:
2534
+ return get_dialect(self.backend)
2535
+ except ValueError:
2536
+ return SPARK_DIALECT
2537
+
2538
+ def _resolve_sql_connection(self) -> Optional[Any]:
2539
+ """Return the engine's SQL connection if any; ``None`` otherwise.
2540
+
2541
+ Different SQL backends store the connection in different
2542
+ places. Centralize the lookup here so :meth:`write_views` and
2543
+ any future SQL-DDL methods stay consistent.
2544
+ """
2545
+ # postgresql / sqlserver backends, connection on the executor's graph.
2546
+ graph = getattr(self.executor, "graph", None)
2547
+ for attr in ("_connection", "connection", "_conn"):
2548
+ conn = getattr(graph, attr, None)
2549
+ if conn is not None:
2550
+ return conn
2551
+ # databricks_sql backend, likely on the executor directly.
2552
+ for attr in ("_connection", "connection", "_conn"):
2553
+ conn = getattr(self.executor, attr, None)
2554
+ if conn is not None:
2555
+ return conn
2556
+ return None
2557
+
2558
+ def _collect_dataframes(self) -> Dict[str, Any]:
2559
+ """Gather the dataframes visible to schema mappings.
2560
+
2561
+ Prefers the engine's ``_dataframes`` dict (the constructor input)
2562
+ but falls back to ``executor.graph.dataframes`` if the engine
2563
+ was constructed without dataframes but later had them injected.
2564
+ """
2565
+ if self._dataframes:
2566
+ return dict(self._dataframes)
2567
+ graph = getattr(self.executor, "graph", None)
2568
+ if graph is not None and hasattr(graph, "dataframes"):
2569
+ return dict(graph.dataframes)
2570
+ return {}
2571
+
2572
+
2573
+ def __repr__(self) -> str:
2574
+ return f"ProGraph(backend='{self.backend}')"
2575
+
2576
+
2577
+ # Backward compatibility alias
2578
+ CypherEngine = ProGraph
2579
+
2580
+
2581
+ def _describe_postgresql_query(conn, query: str):
2582
+ """(column, type name) of an R2RML view (rr:sqlQuery) on PostgreSQL, from the database's
2583
+ description of the statement, run on no row: the result's names from `LIMIT 0`, its types from
2584
+ a prepared statement's `result_types` (pg_prepared_statements, PostgreSQL 14+), deallocated
2585
+ after. Nothing is written (D6): without it a query source's literals were not built."""
2586
+ import uuid
2587
+ from prograph.graph.drivers import fetch_rows
2588
+ names, _ = fetch_rows(conn, f"SELECT * FROM ({query}\n) AS q LIMIT 0")
2589
+ statement = "prograph_describe_" + uuid.uuid4().hex[:12]
2590
+ fetch_rows(conn, f"PREPARE {statement} AS SELECT * FROM ({query}\n) AS q")
2591
+ try:
2592
+ _, rows = fetch_rows(conn, "SELECT unnest(result_types)::text FROM pg_prepared_statements "
2593
+ f"WHERE name = '{statement}'")
2594
+ finally:
2595
+ fetch_rows(conn, f"DEALLOCATE {statement}")
2596
+ kinds = [str(r[0]) for r in rows]
2597
+ if len(kinds) != len(names):
2598
+ return []
2599
+ described = list(zip([str(n) for n in names], kinds))
2600
+ numerics = [name for name, kind in described if kind == "numeric"]
2601
+ if numerics:
2602
+ digits = _postgresql_query_numeric_digits(conn, query, numerics)
2603
+ described = [(name, digits.get(name, kind) if kind == "numeric" else kind) for name, kind in described]
2604
+ return described
2605
+
2606
+
2607
+ class UnknownNumericDigitsWarning(UserWarning):
2608
+ """A PostgreSQL source's NUMERIC columns of no known precision and scale (a table's unconstrained
2609
+ NUMERIC, or an R2RML view's columns where no temporary view could be made), read as an unconstrained
2610
+ NUMERIC is: once per source and engine, naming why and the way to their exact types."""
2611
+
2612
+
2613
+ # The CREATE TEMPORARY VIEW failures that mean the session may not make one, not that the query is wrong:
2614
+ # a read-only transaction (a hot standby or read-only endpoint gives the same, PostgreSQL's
2615
+ # PreventCommandIfReadOnly and PreventCommandDuringRecovery) and no TEMP privilege.
2616
+ _NO_TEMPORARY_VIEW = {"25006", "42501"}
2617
+
2618
+
2619
+ @home("postgresql-column-digits", "A PostgreSQL source's NUMERIC column digits (a table's, an R2RML view's)",
2620
+ terms=("rr:sqlQuery", "R2RML view", "temporary view", "unconstrained NUMERIC", "precision", "scale",
2621
+ "UnknownNumericDigitsWarning", "read-only", "PostgreSQL"))
2622
+ def _postgresql_query_numeric_digits(conn, query: str, numerics: Sequence[str]) -> Dict[str, str]:
2623
+ """Each `numeric` column of an R2RML view (rr:sqlQuery) named in `numerics`, as `numeric(p,s)`.
2624
+ PostgreSQL's prepared statements and the driver's schema name a result column's type but never its
2625
+ modifier, so this runs `CREATE TEMPORARY VIEW <unique name> AS <the query>`, reads that view's rows of
2626
+ `information_schema.columns` and runs `DROP VIEW`, in a `finally`. The view lives in the session's own
2627
+ temporary schema: no table and no row of the user's is written (D6). A column the catalog gives no
2628
+ precision (an unconstrained NUMERIC) keeps `numeric`.
2629
+
2630
+ Where the session may not create the view (SQLSTATE 25006, a read-only transaction, a hot standby or
2631
+ read-only endpoint; 42501, no TEMP privilege; or a passed connection whose open transaction is the
2632
+ caller's: autocommit off, where a failed statement would abort the caller's work) nothing is created,
2633
+ the columns keep plain `numeric`, read as an unconstrained NUMERIC is (`postgresql_graph.numeric_digits`:
2634
+ rounded past 18 places, a whole part past 20 digits an error naming the column; arithmetic the
2635
+ database's own, `facts.arithmetic_slot`), and `warn_unknown_digits` says so once. Any other failure
2636
+ (a query no view can hold, such as one with two output columns of one name) reads the columns the
2637
+ same way and warns too, naming the database's reason, so the query can be fixed; where the query
2638
+ itself is wrong (a column reference it cannot resolve), its own error is the database's when it runs."""
2639
+ import uuid
2640
+ from prograph.graph.drivers import fetch_rows
2641
+ view = "prograph_describe_" + uuid.uuid4().hex[:12]
2642
+ # Inside a transaction the caller holds open (a passed connection with autocommit off) a failed CREATE
2643
+ # would leave the caller's transaction aborted, and the driver then refuses even a ROLLBACK TO SAVEPOINT:
2644
+ # the view is made only in autocommit, where a failed statement aborts nothing.
2645
+ if not _autocommit(conn):
2646
+ warn_unknown_digits(_shown_query(query), numerics, "whose precision and scale ProGraph could not "
2647
+ "read: the connection passed in has a transaction of yours open (autocommit off), "
2648
+ "in which ProGraph creates no temporary view")
2649
+ return {}
2650
+ try:
2651
+ fetch_rows(conn, f"CREATE TEMPORARY VIEW {view} AS {query}")
2652
+ except Exception as error: # noqa: BLE001 - a read-only session, or names a view cannot hold: not an error
2653
+ state = getattr(error, "sqlstate", None) or getattr(error.__cause__, "sqlstate", None)
2654
+ if state in _NO_TEMPORARY_VIEW:
2655
+ warn_unknown_digits(_shown_query(query), numerics, "whose precision and scale ProGraph "
2656
+ f"could not read: the session may not create a temporary view of it "
2657
+ f"({_primary_message(error)}, SQLSTATE {state})")
2658
+ else: # the database will not hold the query as a view: its reason, so the query can be fixed
2659
+ warn_unknown_digits(_shown_query(query), numerics, "whose precision and scale ProGraph "
2660
+ f"could not read: the database would not create a view of the query "
2661
+ f"({_primary_message(error)}" + (f", SQLSTATE {state}" if state else "") + ")")
2662
+ return {}
2663
+ try:
2664
+ _, rows = fetch_rows(conn, "SELECT column_name, numeric_precision, numeric_scale "
2665
+ "FROM information_schema.columns WHERE data_type = 'numeric' "
2666
+ f"AND numeric_precision IS NOT NULL AND table_name = '{view}'")
2667
+ found = {str(c): f"numeric({int(p)},{int(sc)})" for c, p, sc in rows if 0 <= int(sc) <= int(p)}
2668
+ finally:
2669
+ try:
2670
+ fetch_rows(conn, f"DROP VIEW IF EXISTS pg_temp.{view}")
2671
+ except Exception: # noqa: BLE001 - the view dies with the session
2672
+ pass
2673
+ unconstrained = [name for name in numerics if name not in found]
2674
+ if unconstrained:
2675
+ warn_unknown_digits(_shown_query(query), unconstrained, "whose precision and scale the query does not "
2676
+ "state (an unconstrained NUMERIC)")
2677
+ return found
2678
+
2679
+
2680
+ def warn_unknown_digits(source: str, numerics: Sequence[str], why: str) -> None:
2681
+ """`postgresql-column-digits`' one warning for a source whose `numerics` have no known digits (each
2682
+ engine describes a source once). Given with a registry of its own, so Python's once-per-text filter
2683
+ does not swallow a second engine's first sight of the same source."""
2684
+ import sys
2685
+ import warnings
2686
+ from prograph.graph.postgresql_graph import numeric_digits
2687
+ (p, s), _ = numeric_digits(None)
2688
+ text = (f"{source} has NUMERIC columns ({', '.join(numerics)}) {why}. They are read as an unconstrained "
2689
+ f"NUMERIC is: as DECIMAL({p}, {s}), rounded past {s} places after the point, a value whose whole "
2690
+ f"part passes {p - s} digits an error naming the column. For their exact types, map a table, or a "
2691
+ f"view created in the database, whose columns declare NUMERIC(p, s), by rr:tableName (or a YAML "
2692
+ f"source's table).")
2693
+ caller = sys._getframe(1)
2694
+ warnings.warn_explicit(text, UnknownNumericDigitsWarning, caller.f_code.co_filename, caller.f_lineno,
2695
+ module=caller.f_globals.get("__name__"), registry={})
2696
+
2697
+
2698
+ def _shown_query(query: str) -> str:
2699
+ shown = " ".join(query.split())
2700
+ return "The R2RML view (rr:sqlQuery) " + repr(shown if len(shown) <= 120 else shown[:117] + "...")
2701
+
2702
+
2703
+ def _primary_message(error: BaseException) -> str:
2704
+ for key, value in getattr(error, "details", None) or ():
2705
+ if key == b"PG_DIAG_MESSAGE_PRIMARY":
2706
+ return value.decode("utf-8", "replace")
2707
+ said = " ".join(str(error).split())
2708
+ return said.rpartition("ERROR:")[2].split(" SQLSTATE")[0].strip(" .")[:200]
2709
+
2710
+
2711
+ def _autocommit(conn) -> bool:
2712
+ """Whether an ADBC connection runs each statement in a transaction of its own."""
2713
+ try:
2714
+ return str(conn.adbc_connection.get_option("adbc.connection.autocommit")).lower() == "true"
2715
+ except Exception: # noqa: BLE001 - a driver that cannot say: treat it as the caller's transaction
2716
+ return False
2717
+
2718
+