prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,556 @@
1
+ """SparkGraph - A virtual graph backed by Spark DataFrames.
2
+
3
+ This holds the Spark DataFrames and runs the generated SQL on them (or on
4
+ a SQL warehouse), read-only.
5
+ """
6
+
7
+ from contextlib import contextmanager
8
+ from typing import Any, Dict, List, Optional, Set, Union, TYPE_CHECKING
9
+
10
+ if TYPE_CHECKING:
11
+ from prograph.execution import QueryContext
12
+ import pyarrow as pa
13
+
14
+ try:
15
+ from pyspark.sql import DataFrame as SparkDataFrame
16
+ SPARK_AVAILABLE = True
17
+ except ImportError:
18
+ SPARK_AVAILABLE = False
19
+ SparkDataFrame = None
20
+
21
+ from prograph.schema.mapping import GraphSchema
22
+ from prograph.execution import QueryCancelledException
23
+ from prograph.logging import get_backend_logger
24
+ from prograph.read_only import GraphRefusesWrites
25
+
26
+ def _nullable_spark_type(dt):
27
+ """`dt` with every field, element and map value nullable."""
28
+ from pyspark.sql.types import ArrayType, MapType, StructField, StructType
29
+ if isinstance(dt, StructType):
30
+ return StructType([StructField(f.name, _nullable_spark_type(f.dataType), True, f.metadata)
31
+ for f in dt.fields])
32
+ if isinstance(dt, ArrayType):
33
+ return ArrayType(_nullable_spark_type(dt.elementType), True)
34
+ if isinstance(dt, MapType):
35
+ return MapType(dt.keyType, _nullable_spark_type(dt.valueType), True)
36
+ return dt
37
+
38
+
39
+ def _nullable_arrow_type(t):
40
+ """`t` with every field, element and map value nullable (a map's key
41
+ stays not null, as Arrow requires)."""
42
+ import pyarrow as pa
43
+ if pa.types.is_struct(t):
44
+ return pa.struct([t.field(i).with_type(_nullable_arrow_type(t.field(i).type))
45
+ .with_nullable(True) for i in range(t.num_fields)])
46
+ if pa.types.is_map(t):
47
+ return pa.map_(t.key_field, t.item_field.with_type(
48
+ _nullable_arrow_type(t.item_type)).with_nullable(True), t.keys_sorted)
49
+ if pa.types.is_large_list(t):
50
+ return pa.large_list(t.value_field.with_type(
51
+ _nullable_arrow_type(t.value_type)).with_nullable(True))
52
+ if pa.types.is_list(t):
53
+ return pa.list_(t.value_field.with_type(
54
+ _nullable_arrow_type(t.value_type)).with_nullable(True))
55
+ return t
56
+
57
+
58
+ def _spark_arrow(df: 'SparkDataFrame') -> 'pa.Table':
59
+ """A Spark result as Spark's own Arrow (``DataFrame.toArrow``), every
60
+ field nullable at every level and nothing else changed. Spark's not-null flags describe its plan, not the data, and
61
+ vary from query to query, so one term would not keep one Arrow type;
62
+ and toArrow refuses a null struct whose fields are not
63
+ null, or an empty list (array<void>), when the flags are kept. A
64
+ column with nested not-null flags is cast to its nullable type first
65
+ (the same type, so no value changes), and a not-null void column (`UNWIND
66
+ []`) is a null; the rest is the Arrow schema's. Columns are taken by
67
+ position, as a result may hold two of one name.
68
+ """
69
+ import pyarrow as pa
70
+ from pyspark.sql import functions as F
71
+ from pyspark.sql.types import NullType
72
+ fields = df.schema.fields
73
+ void = [isinstance(f.dataType, NullType) and not f.nullable for f in fields]
74
+ nested = [_nullable_spark_type(f.dataType) != f.dataType for f in fields]
75
+ if any(void) or any(nested):
76
+ names = df.columns
77
+ at = [f"_c{i}" for i in range(len(fields))]
78
+ df = df.toDF(*at).select([
79
+ F.lit(None).alias(a) if v
80
+ else F.col(a).cast(_nullable_spark_type(f.dataType)).alias(a) if n
81
+ else F.col(a)
82
+ for a, f, v, n in zip(at, fields, void, nested)]).toDF(*names)
83
+ table = df.toArrow()
84
+ return table.cast(pa.schema(
85
+ [pa.field(f.name, _nullable_arrow_type(f.type), True, f.metadata) for f in table.schema],
86
+ metadata=table.schema.metadata))
87
+
88
+
89
+ def register_temp_views(dataframes: Dict[str, Any], registered: Set[str]) -> None:
90
+ """Register each frame of `dataframes` not in `registered` as a temp
91
+ view under its name. A qualified name (catalog.schema.table) names a
92
+ catalog table, which needs none: temp views take single-part names."""
93
+ for table_name, df in dataframes.items():
94
+ if table_name not in registered:
95
+ if '.' not in table_name:
96
+ df.createOrReplaceTempView(table_name)
97
+ registered.add(table_name)
98
+
99
+
100
+
101
+ # The part of a relationship variable's identity column name: `r.__identity__`.
102
+ REL_IDENTITY_PART = "__identity__"
103
+
104
+ # The row number a relationship mapping's rows get while their identity is made.
105
+ _RELATIONSHIP_ROW = "__relationship_row__"
106
+
107
+ # The column that holds each relationship row's identity (its mapping's position in
108
+ # the schema, its ends and its row number among the rows of the same ends).
109
+ RELATIONSHIP_IDENTITY_COLUMN = "__relationship_identity__"
110
+
111
+
112
+ # The key under which an expression's frame travels in a variables mapping: the frame
113
+ # the expression's columns are read from, so a column's Spark type can be read where
114
+ # an expression needs it (`SparkGraph._divide_column`).
115
+ TYPING_FRAME = "_typing_frame"
116
+
117
+
118
+ @contextmanager
119
+ def _job_group(spark: Any):
120
+ """The jobs this thread starts inside belong to one new job group (a tag on Spark Connect), and
121
+ the stop that cancels them: `cancelJobGroup` (`interruptOnCancel`, so a running task's thread
122
+ is interrupted), `interruptTag` on Connect. The group's id is ProGraph's own, never the query's
123
+ id. None where the session has neither."""
124
+ import uuid
125
+ group = f"prograph-{uuid.uuid4().hex}"
126
+ try:
127
+ sc = spark.sparkContext
128
+ except Exception: # noqa: BLE001 - Spark Connect has no SparkContext
129
+ sc = None
130
+ if sc is not None:
131
+ sc.setJobGroup(group, "ProGraph query", interruptOnCancel=True)
132
+ try:
133
+ yield lambda: sc.cancelJobGroup(group)
134
+ finally:
135
+ sc.setLocalProperty("spark.jobGroup.id", None)
136
+ sc.setLocalProperty("spark.job.description", None)
137
+ sc.setLocalProperty("spark.job.interruptOnCancel", None)
138
+ return
139
+ if hasattr(spark, "addTag") and hasattr(spark, "interruptTag"):
140
+ spark.addTag(group)
141
+ try:
142
+ yield lambda: spark.interruptTag(group)
143
+ finally:
144
+ spark.removeTag(group)
145
+ return
146
+ yield None
147
+
148
+
149
+ class SparkGraph(GraphRefusesWrites):
150
+ """A virtual graph backed by Spark DataFrames.
151
+
152
+ Runs the generated SQL over the Spark DataFrames. Read-only: its mutators refuse.
153
+
154
+ Example:
155
+ >>> schema = GraphSchema()
156
+ >>> schema.add_node_mapping(NodeMapping(
157
+ ... source_table='movies',
158
+ ... label='Movie',
159
+ ... id_column='movie_id',
160
+ ... property_columns={'title': 'title'}
161
+ ... ))
162
+ >>> graph = SparkGraph(schema, {'movies': movies_df})
163
+ >>> graph._execute_sql_arrow('SELECT * FROM movies')
164
+ """
165
+
166
+ def __init__(
167
+ self,
168
+ schema: GraphSchema,
169
+ dataframes: Dict[str, 'SparkDataFrame'],
170
+ sql_warehouse_id: Optional[str] = None,
171
+ warehouse_timeout: Optional[float] = None,
172
+ ansi_session: bool = True,
173
+ ):
174
+ """Create a SparkGraph.
175
+
176
+ Args:
177
+ schema: GraphSchema defining the mapping
178
+ dataframes: Dictionary mapping table names to Spark DataFrames
179
+ sql_warehouse_id: Optional Databricks SQL Warehouse ID for optimized
180
+ SQL execution. If provided, generated SQL queries will be routed
181
+ to the SQL Warehouse instead of Spark compute. Falls back to
182
+ Spark if the warehouse is unavailable.
183
+ warehouse_timeout: Optional timeout in seconds for SQL Warehouse
184
+ queries. When set, queries that exceed this time are cancelled
185
+ on the warehouse and raise TimeoutError. None (default) means
186
+ poll indefinitely until completion.
187
+ ansi_session: Run each SQL statement with ANSI mode on, in a
188
+ session of its own setting (prograph/graph/ansi_session.py).
189
+ """
190
+ if not SPARK_AVAILABLE:
191
+ raise ImportError("pyspark is required for the Spark backend")
192
+ self._ansi_session = ansi_session
193
+
194
+ self.schema = schema
195
+ self.dataframes = dataframes
196
+ # Per written table, (the frame first given for it, the frame put
197
+ # in its place after the last write): _refresh_frame.
198
+ self._written_frames: Dict[str, tuple] = {}
199
+ # SQL Warehouse configuration
200
+ self._sql_warehouse_id = sql_warehouse_id
201
+ self._sql_warehouse_connection = None
202
+ self._warehouse_timeout = warehouse_timeout
203
+
204
+ def _execute_mutation_sql(self, sql: str, mapping) -> None:
205
+ """Execute a mutation SQL statement on `mapping`'s table, then
206
+ refresh the engine's frame of that table. The one write a
207
+ SparkGraph makes: `set_properties`' UPDATE on a mutable mapping
208
+ (prograph.writes); its mutators refuse.
209
+
210
+ Args:
211
+ sql: The SQL statement to execute
212
+ mapping: The node or relationship mapping whose table it writes
213
+
214
+ TODO: Route through _execute_sql() for SQL Warehouse support.
215
+ Currently calls spark.sql() directly, bypassing warehouse
216
+ routing. SQL Warehouse supports DML (INSERT/UPDATE/DELETE/MERGE).
217
+ """
218
+ # On the session itself, not a clone of it (ansi_session.py): the
219
+ # statement writes literals, which cannot overflow, and a write
220
+ # made in a clone leaves the session's own cached relation of a
221
+ # file table as it was.
222
+ spark = next(iter(self.dataframes.values())).sparkSession
223
+ spark.sql(sql)
224
+ self._refresh_frame(mapping.source_table)
225
+
226
+ def _refresh_frame(self, table: str) -> None:
227
+ """Make the engine's frame of `table` see the files a write added.
228
+
229
+ A frame of a file table (parquet) holds the list of the table's
230
+ files that Spark made when the frame was read. Spark's INSERT
231
+ refreshes the list of the relation it resolved, which is the
232
+ frame's own only until the catalog's cached relation is replaced:
233
+ after a table's first write the frame kept its list, and every
234
+ later read through it (the SQL path reads it as a temp view under
235
+ the table's name, which a later `spark.table` of that name returns
236
+ too) lacked the rows of the second write on. Refreshing the
237
+ frame's own plan re-lists its files in place, so the frame and the
238
+ temp views over it see the table as written. A frame with no JVM
239
+ plan (Spark Connect) reads the table anew at each action.
240
+
241
+ A frame that has run keeps the physical plan it ran with, and its
242
+ scan the files listed then: SPARQL in dataframe mode collects the
243
+ frame itself, so after its first read no write was seen. The frame's place therefore takes a new Dataset over the
244
+ refreshed plan, planned at its first action. It is made over the
245
+ frame first given, not over the last one put, so writes do not
246
+ stack a projection each; a frame put there by the caller since is
247
+ taken as the one given."""
248
+ given = self._given_frame(table)
249
+ if getattr(given, '_jdf', None) is None:
250
+ return
251
+ given._jdf.queryExecution().analyzed().refresh()
252
+ fresh = given.select("*")
253
+ self._written_frames[table] = (given, fresh)
254
+ self.dataframes[table] = fresh
255
+
256
+ def _given_frame(self, table: str) -> Optional['SparkDataFrame']:
257
+ """The frame given for `table`, under the one `_refresh_frame` put
258
+ in its place: a temp view is made over it, since a view over the
259
+ put frame's projection takes no INSERT (UNSUPPORTED_INSERT)."""
260
+ frame = self.dataframes.get(table)
261
+ given, put = self._written_frames.get(table, (None, None))
262
+ return given if put is not None and frame is put else frame
263
+
264
+ def _invalidate_caches(self, changes: 'SchemaChangeSet') -> None:
265
+ """Drop the frames of tables a schema change removed."""
266
+ if not changes.has_changes:
267
+ return
268
+ for table in changes.removed_tables:
269
+ self.dataframes.pop(table, None)
270
+
271
+ def close(self) -> None:
272
+ """Close any open connections (e.g., SQL Warehouse connection)."""
273
+ if self._sql_warehouse_connection is not None:
274
+ try:
275
+ self._sql_warehouse_connection.close()
276
+ except Exception as e:
277
+ get_backend_logger().debug("warehouse_close_failed", error=str(e))
278
+ self._sql_warehouse_connection = None
279
+
280
+ def _reset_warehouse_connection(self):
281
+ """Reset cached warehouse connection (e.g., after auth failure)."""
282
+ if self._sql_warehouse_connection is not None:
283
+ try:
284
+ self._sql_warehouse_connection.close()
285
+ except Exception as e:
286
+ get_backend_logger().debug("warehouse_close_failed", error=str(e))
287
+ self._sql_warehouse_connection = None
288
+
289
+ def __del__(self):
290
+ """Cleanup on garbage collection."""
291
+ self.close()
292
+
293
+ def _execute_sql(
294
+ self,
295
+ sql: str,
296
+ spark: 'SparkSession',
297
+ use_warehouse: Optional[bool] = None,
298
+ query_context: Optional['QueryContext'] = None,
299
+ adaptive_off: bool = False,
300
+ ) -> Union[List[Dict[str, Any]], 'pa.Table']:
301
+ """Execute SQL query, routing to SQL Warehouse if configured.
302
+
303
+ On Spark the result is Spark's own Arrow (``DataFrame.toArrow``),
304
+ every field nullable (_spark_arrow), as every other SQL backend
305
+ hands back its database's: its types are Spark's, as
306
+ Databricks gives them. Collected
307
+ as rows, a TIMESTAMP was a naive datetime in the Python process's
308
+ zone, an INT an int64 and a MAP a struct. A
309
+ statement routed to a SQL warehouse returns rows.
310
+
311
+ ``adaptive_off``: run the statement on Spark with adaptive execution
312
+ off, for a statement that holds a slice below its result (the planner reads it from the statement's tree). A
313
+ statement routed to a SQL warehouse is not affected.
314
+
315
+ If sql_warehouse_id is set, attempts to execute via SQL Warehouse
316
+ for better performance (Photon engine, no cluster contention).
317
+ Falls back to spark.sql() if warehouse is unavailable.
318
+
319
+ Args:
320
+ sql: The SQL query to execute
321
+ spark: SparkSession to use as fallback
322
+ use_warehouse: Override warehouse routing. None=auto (use if configured),
323
+ True=force warehouse, False=skip warehouse (use Spark).
324
+ query_context: Optional QueryContext for cancellation support.
325
+
326
+ Returns:
327
+ List of result dictionaries
328
+ """
329
+ from prograph.logging import get_backend_logger
330
+ import time
331
+ logger = get_backend_logger()
332
+
333
+ # The query running here (`execution.running`), if none is given.
334
+ if query_context is None:
335
+ from prograph.execution import _CURRENT
336
+ query_context = _CURRENT.get()
337
+
338
+ # Check for cancellation before starting
339
+ if query_context:
340
+ query_context.check_cancelled()
341
+
342
+ # Determine whether to use warehouse
343
+ should_use_warehouse = (
344
+ use_warehouse is True or
345
+ (use_warehouse is None and self._sql_warehouse_id is not None)
346
+ ) and use_warehouse is not False
347
+
348
+ # If SQL Warehouse should be used, try it
349
+ if should_use_warehouse and self._sql_warehouse_id:
350
+ try:
351
+ start_time = time.time()
352
+ results = self._execute_via_warehouse(sql, query_context=query_context)
353
+ elapsed = time.time() - start_time
354
+ logger.debug(
355
+ "sql_warehouse_executed",
356
+ warehouse_id=self._sql_warehouse_id,
357
+ result_count=len(results),
358
+ elapsed_ms=round(elapsed * 1000, 2)
359
+ )
360
+ return results
361
+ except TimeoutError:
362
+ # Warehouse is the fast path; if it times out, Spark will
363
+ # be slower. Propagate as a hard error, no fallback.
364
+ raise
365
+ except QueryCancelledException:
366
+ # Cancelled queries should not fall back to Spark,
367
+ # Python/Spark execution will be even slower.
368
+ raise
369
+ except Exception as e:
370
+ # Reset connection on auth errors so next attempt creates fresh client
371
+ error_str = str(e).lower()
372
+ if any(term in error_str for term in ('unauthorized', '401', 'token', 'authentication', 'forbidden', '403')):
373
+ self._reset_warehouse_connection()
374
+ # Log and fall back to Spark; use warning level so users
375
+ # can see why warehouse routing failed
376
+ logger.warning(
377
+ "sql_warehouse_fallback",
378
+ warehouse_id=self._sql_warehouse_id,
379
+ error=str(e),
380
+ error_type=type(e).__name__,
381
+ )
382
+
383
+ # Log if warehouse was skipped due to smart routing
384
+ if use_warehouse is False and self._sql_warehouse_id:
385
+ logger.debug(
386
+ "sql_warehouse_skipped",
387
+ reason="smart_routing",
388
+ warehouse_id=self._sql_warehouse_id
389
+ )
390
+
391
+ # Default: execute via Spark, with ANSI mode on, and fetched
392
+ # while it is. The statement's jobs are one job
393
+ # group, which a cancel or the deadline cancels (`_job_group`).
394
+ from prograph.execution import StatementGuard
395
+ from prograph.graph.ansi_session import statement_session
396
+ with _job_group(spark) as stop, StatementGuard(stop=stop):
397
+ with statement_session(spark, self._ansi_session, adaptive_off=adaptive_off) as session:
398
+ return _spark_arrow(session.sql(sql))
399
+
400
+ def _execute_via_warehouse(
401
+ self,
402
+ sql: str,
403
+ query_context: Optional['QueryContext'] = None
404
+ ) -> List[Dict[str, Any]]:
405
+ """Execute SQL via Databricks SQL Warehouse using SDK REST API.
406
+
407
+ Uses the Databricks SDK for optimized SQL execution with server-side
408
+ result caching. ~2-3x faster than Thrift connector on cold queries,
409
+ ~12x faster on cached queries.
410
+
411
+ Args:
412
+ sql: The SQL query to execute
413
+ query_context: Optional QueryContext for cancellation support.
414
+ If provided, the statement_id will be tracked for
415
+ cancellation via statement_execution.cancel_execution().
416
+
417
+ Returns:
418
+ List of result dictionaries
419
+ """
420
+ # Lazy import to avoid dependency issues
421
+ try:
422
+ from databricks.sdk import WorkspaceClient
423
+ except ImportError:
424
+ raise ImportError(
425
+ "databricks-sdk is required for SQL Warehouse support. "
426
+ "Install with: pip install databricks-sdk"
427
+ )
428
+
429
+ # Create SDK client (reuse if possible)
430
+ # WorkspaceClient() uses the SDK's unified credential chain:
431
+ # 1. Environment variables (DATABRICKS_HOST + DATABRICKS_TOKEN)
432
+ # 2. Databricks CLI profile (~/.databrickscfg)
433
+ # 3. Azure/GCP/AWS managed identity
434
+ # 4. Notebook context (auto-detected)
435
+ # The SDK handles token refresh automatically for OAuth/CLI-based auth.
436
+ if self._sql_warehouse_connection is None:
437
+ try:
438
+ self._sql_warehouse_connection = WorkspaceClient()
439
+ except Exception as e:
440
+ raise ValueError(
441
+ "Could not determine Databricks connection details. "
442
+ "Configure credentials via databricks-cli (`databricks auth login`), "
443
+ "set DATABRICKS_HOST and DATABRICKS_TOKEN environment variables, "
444
+ "or run in a Databricks notebook."
445
+ ) from e
446
+
447
+ # Check for cancellation before executing
448
+ if query_context:
449
+ query_context.check_cancelled()
450
+
451
+ # Execute query via REST API
452
+ import time
453
+ response = self._sql_warehouse_connection.statement_execution.execute_statement(
454
+ warehouse_id=self._sql_warehouse_id,
455
+ statement=sql,
456
+ wait_timeout="50s" # Max allowed by API
457
+ )
458
+
459
+ # Track statement_id in context for cancellation support
460
+ if query_context and response.statement_id:
461
+ query_context.set_statement_id(
462
+ response.statement_id,
463
+ self._sql_warehouse_connection
464
+ )
465
+
466
+ # Poll if statement is still running after initial wait
467
+ try:
468
+ from databricks.sdk.service.sql import StatementState
469
+ except ImportError:
470
+ StatementState = None
471
+
472
+ if StatementState is not None and response.status and response.status.state in (
473
+ StatementState.PENDING, StatementState.RUNNING
474
+ ):
475
+ from prograph.logging import get_backend_logger
476
+ logger = get_backend_logger()
477
+ logger.debug(
478
+ "warehouse_poll_start",
479
+ statement_id=response.statement_id,
480
+ warehouse_timeout=self._warehouse_timeout,
481
+ )
482
+ poll_start = time.time()
483
+ while True:
484
+ # Check for cancellation
485
+ if query_context:
486
+ query_context.check_cancelled()
487
+
488
+ # Check warehouse_timeout
489
+ if self._warehouse_timeout is not None:
490
+ elapsed = time.time() - poll_start + 50 # include initial wait
491
+ if elapsed >= self._warehouse_timeout:
492
+ # Cancel the running statement on the warehouse
493
+ try:
494
+ self._sql_warehouse_connection.statement_execution.cancel_execution(
495
+ response.statement_id
496
+ )
497
+ except Exception as e:
498
+ logger.debug("timeout_cancel_failed", statement_id=response.statement_id, error=str(e))
499
+ raise TimeoutError(
500
+ f"Warehouse query exceeded timeout of {self._warehouse_timeout}s "
501
+ f"(statement_id={response.statement_id})"
502
+ )
503
+
504
+ time.sleep(2)
505
+ response = self._sql_warehouse_connection.statement_execution.get_statement(
506
+ response.statement_id
507
+ )
508
+ if response.status and response.status.state not in (
509
+ StatementState.PENDING, StatementState.RUNNING
510
+ ):
511
+ break
512
+
513
+ logger.debug(
514
+ "warehouse_poll_complete",
515
+ statement_id=response.statement_id,
516
+ poll_elapsed_ms=round((time.time() - poll_start) * 1000, 2),
517
+ )
518
+
519
+ # Check for failure after polling
520
+ if StatementState is not None and response.status:
521
+ if response.status.state == StatementState.FAILED:
522
+ error_msg = (
523
+ response.status.error.message
524
+ if response.status.error else "Unknown error"
525
+ )
526
+ raise RuntimeError(f"Warehouse statement failed: {error_msg}")
527
+ if response.status.state == StatementState.CANCELED:
528
+ raise QueryCancelledException("Warehouse statement was cancelled")
529
+
530
+ # Extract results from response
531
+ if response.result is None or response.result.data_array is None:
532
+ return []
533
+
534
+ # Get column names from schema
535
+ columns = [col.name for col in response.manifest.schema.columns]
536
+
537
+ # Convert data_array to list of dicts
538
+ return [dict(zip(columns, row)) for row in response.result.data_array]
539
+
540
+ def sql_dialect(self) -> Optional[str]:
541
+ """The dialect this graph runs SQL in: Spark SQL, through spark.sql."""
542
+ return "spark"
543
+
544
+ def _ensure_temp_views(self) -> None:
545
+ """Ensure all DataFrames are registered as temporary views.
546
+
547
+ Only creates temp views for simple (non-qualified) table names.
548
+ Fully qualified names (catalog.schema.table) are assumed to reference
549
+ existing catalog tables that don't need temp views.
550
+ """
551
+ if not hasattr(self, '_views_registered'):
552
+ self._views_registered = set()
553
+ register_temp_views({t: self._given_frame(t) for t in self.dataframes}, self._views_registered)
554
+
555
+ def __repr__(self) -> str:
556
+ return f"SparkGraph(tables={sorted(self.dataframes)})"
@@ -0,0 +1,83 @@
1
+ """Base class for SQL-database-backed graphs.
2
+
3
+ Provides a simple interface for executing SQL queries against relational
4
+ databases (PostgreSQL, SQL Server, Oracle). Much simpler than SparkGraph:
5
+ no DataFrames, no VL paths.
6
+
7
+ Architecture:
8
+
9
+ ProGraph(backend='postgresql', connection=conn, schema=schema)
10
+ → PostgreSQLExecutor
11
+ → PostgreSQLGraph(SQLGraph)
12
+ → build_sql(dialect='postgresql') → SQL string
13
+ → _execute_sql(sql) → List[Dict]
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import logging
19
+ from typing import Any, Dict, List, Optional, Tuple
20
+ from prograph.read_only import GraphRefusesWrites
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+
25
+ class SQLGraph(GraphRefusesWrites):
26
+ """Base class for SQL-database-backed graphs.
27
+
28
+ Subclasses implement ``_execute_sql()`` for their specific database
29
+ driver.
30
+ """
31
+
32
+ def __init__(
33
+ self,
34
+ schema: Any,
35
+ connection: Any,
36
+ dialect: str,
37
+ table_stats: Optional[Dict[str, int]] = None,
38
+ ):
39
+ """Initialize the SQL graph.
40
+
41
+ Args:
42
+ schema: GraphSchema with node/relationship mappings
43
+ connection: Database connection (driver-specific)
44
+ dialect: SQL dialect name ('postgresql', etc.)
45
+ table_stats: Optional table row counts for optimization
46
+ """
47
+ self.schema = schema
48
+ self._connection = connection
49
+ self._dialect = dialect
50
+ self._table_stats = table_stats or {}
51
+
52
+ def _fetch(self, sql: str) -> Tuple[List[str], List[tuple]]:
53
+ """Execute SQL and return its column names and its rows. A
54
+ DB-API driver implements this; one returning Arrow overrides
55
+ _execute_sql and _execute_sql_arrow instead (Databricks, DuckDB)."""
56
+ raise NotImplementedError
57
+
58
+ def _execute_sql(self, sql: str) -> List[Dict[str, Any]]:
59
+ """Execute a SQL query and return its rows as dicts."""
60
+ columns, rows = self._fetch(sql)
61
+ return [dict(zip(columns, row)) for row in rows]
62
+
63
+ def _execute_sql_arrow(self, sql: str):
64
+ """Execute SQL and return a pyarrow.Table, built once from the
65
+ driver's rows. The column names come from the cursor, so a result
66
+ with no rows still has its columns."""
67
+ from prograph.results import rows_to_arrow
68
+ columns, rows = self._fetch(sql)
69
+ return rows_to_arrow([dict(zip(columns, row)) for row in rows], columns)
70
+
71
+ def sql_dialect(self) -> str:
72
+ """The dialect this graph executes SQL in."""
73
+ return self._dialect
74
+
75
+ @property
76
+ def dataframes(self) -> Dict[str, Any]:
77
+ """Compatibility shim for _try_compositional_backend.
78
+
79
+ SQL-database graphs don't use DataFrames. This property returns
80
+ a non-empty dict so the compositional backend path activates,
81
+ but none of the values are real DataFrames.
82
+ """
83
+ return {'_sql_graph': True}