prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,300 @@
1
+ """Databricks SQL Graph Backend.
2
+
3
+ A pure-SQL graph backend for Databricks SQL Warehouses (Photon).
4
+ No DataFrames and no SparkSession: all operations are SQL over a PEP 249
5
+ connection from ``databricks-sql-connector``.
6
+
7
+ Usage:
8
+
9
+ from databricks import sql as databricks_sql
10
+ conn = databricks_sql.connect(
11
+ server_hostname="...", http_path="...", access_token="..."
12
+ )
13
+ graph = DatabricksSQLGraph(schema, conn, catalog="main", schema_name="graph")
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import logging
19
+ from typing import Any, Dict, List, Optional
20
+
21
+ from prograph.rules import home
22
+
23
+ from .sql_graph import SQLGraph
24
+
25
+ logger = logging.getLogger(__name__)
26
+
27
+
28
+
29
+ # How often a submitted statement's state is asked for: from the first interval, each wait half as long
30
+ # again as the one before, to at most the last (a short statement pays a few round trips; a long one a
31
+ # poll a second).
32
+ POLL_FIRST, POLL_GROWTH, POLL_MOST = 0.05, 1.5, 1.0
33
+
34
+
35
+ # The Arrow type the Thrift protocol gives each Databricks type in a result (the connector asks for Arrow
36
+ # timestamps, decimals and complex types: `_use_arrow_native_*`; intervals arrive as text). Keys are the
37
+ # description's type codes: Thrift's TTypeId names lowered without `_TYPE`, which SEA's are normalized to
38
+ # (`byte`, `short`, `long` kept as well, should a description carry SEA's own names).
39
+ _THRIFT_ARROW_TYPES = {
40
+ "string": "string", "varchar": "string", "char": "string",
41
+ "interval_year_month": "string", "interval_day_time": "string", "interval": "string",
42
+ "tinyint": "int8", "byte": "int8", "smallint": "int16", "short": "int16",
43
+ "int": "int32", "integer": "int32", "bigint": "int64", "long": "int64",
44
+ "boolean": "bool", "float": "float32", "double": "float64",
45
+ "date": "date32", "binary": "binary",
46
+ # An untyped NULL column: SEA's description says `null` (or `void`), Thrift's says `string` and its Arrow
47
+ # column is a string, so an empty answer has one schema on either path.
48
+ "null": "string", "void": "string",
49
+ }
50
+
51
+
52
+ @home("databricks-empty-answer", "A Databricks answer of no rows, with its columns",
53
+ terms=("SEA", "Statement Execution API", "empty answer", "0 columns", "fetchall_arrow", "description"))
54
+ def empty_answer(description: Any):
55
+ """The empty Arrow table of a statement whose answer has no rows, its fields from the cursor's
56
+ `description`: over the Statement Execution API (SEA) the connector has no Arrow schema for a result
57
+ (`arrow_schema_bytes` is None), so an empty answer arrives as a table of 0 columns, where Thrift sends
58
+ the columns. Each field is the description's name, at the type Thrift gives that Databricks type
59
+ (`_THRIFT_ARROW_TYPES`; a decimal its precision and scale, a TIMESTAMP an instant in UTC, a
60
+ TIMESTAMP_NTZ a naive one). An ARRAY, STRUCT, MAP or VARIANT (or a type not listed) is `pa.null()`: the
61
+ description names the kind but not its element or field types, and a guessed type (a list of
62
+ strings, a struct of no field) would be a wrong type that a reader may act on, where a null column
63
+ holds no value and casts to any type a consumer needs (concatenated with a later batch, compared
64
+ with another engine's answer)."""
65
+ import pyarrow as pa
66
+ fields = []
67
+ for column in description:
68
+ name = column[0]
69
+ code = str(column[1] if len(column) > 1 and column[1] is not None else "").lower()
70
+ if code == "decimal" and len(column) > 5 and column[4] is not None and column[5] is not None:
71
+ kind = pa.decimal128(int(column[4]), int(column[5]))
72
+ elif code == "timestamp":
73
+ kind = pa.timestamp("us", tz="Etc/UTC")
74
+ elif code == "timestamp_ntz":
75
+ kind = pa.timestamp("us")
76
+ else:
77
+ kind = pa.type_for_alias(_THRIFT_ARROW_TYPES.get(code, "null"))
78
+ fields.append(pa.field(name, kind))
79
+ return pa.schema(fields).empty_table()
80
+
81
+
82
+ def _with_columns(table: Any, description: Any):
83
+ """`table` as the driver gave it; an empty answer of 0 columns whose statement has columns (SEA,
84
+ `empty_answer`) as the empty table of those columns. Only a 0-column table is replaced, so every
85
+ answer that has its columns is returned untouched."""
86
+ if table.num_columns == 0 and description:
87
+ return empty_answer(description)
88
+ return table
89
+
90
+
91
+ def run_statement(cursor: Any, sql: str, guard: Any) -> None:
92
+ """Run `sql` on `cursor`: blocking, its result in the first response, unless the query is expected to
93
+ run long (`execution.long_expected`: a closure or a costly shape), where it runs so that a cancel
94
+ reaches it at once: submitted without waiting
95
+ (`execute_async`), which gives the cursor its command id as soon as the warehouse has the
96
+ statement, then its state polled until it ends and its result taken. A blocking `execute` holds
97
+ the call, with no command id, for the warehouse's first wait (several seconds), and a
98
+ `cursor.cancel()` in that window has nothing to cancel (measured on a SQL warehouse: the stop
99
+ landed 6-7 s after the cancel). The stop is sent at once; a cancel or the deadline (`guard`) ends the
100
+ wait at the next poll (at most a second). A
101
+ connector without `execute_async`, or a backend that refuses it (NotSupportedError,
102
+ NotImplementedError: checked against the kernel backend of a Lakehouse RT warehouse), runs
103
+ `execute`."""
104
+ import time
105
+ from prograph.execution import long_expected
106
+ submit = getattr(cursor, "execute_async", None) if long_expected() else None
107
+ session = getattr(getattr(cursor, "connection", None), "session", None)
108
+ if getattr(session, "use_sea", False) is True:
109
+ # SEA's async result raises a bare "Operation status FAILED", without the server's text, which the
110
+ # closure ladder (a depth guard, Lakehouse RT's 0A000) and the typed errors read: run blocking there.
111
+ submit = None
112
+ if submit is None:
113
+ cursor.execute(sql)
114
+ return
115
+ try:
116
+ submit(sql)
117
+ except NotImplementedError:
118
+ cursor.execute(sql)
119
+ return
120
+ except Exception as error: # noqa: BLE001 - the connector's NotSupportedError (or a subclass), by name
121
+ if not any(k.__name__ == "NotSupportedError" for k in type(error).__mro__):
122
+ raise
123
+ cursor.execute(sql)
124
+ return
125
+ wait = POLL_FIRST
126
+ while cursor.is_query_pending():
127
+ guard.check()
128
+ time.sleep(wait)
129
+ wait = min(wait * POLL_GROWTH, POLL_MOST)
130
+ guard.check()
131
+ cursor.get_async_execution_result()
132
+
133
+ class DatabricksSQLGraph(SQLGraph):
134
+ """Databricks SQL Warehouse-backed graph.
135
+
136
+ Executes SQL via the ``databricks-sql-connector`` PEP 249 cursor.
137
+ All queries go to Photon, no Spark compute is involved.
138
+ """
139
+
140
+ def __init__(
141
+ self,
142
+ schema: Any,
143
+ connection: Any,
144
+ catalog: Optional[str] = None,
145
+ schema_name: Optional[str] = None,
146
+ table_stats: Optional[Dict[str, int]] = None,
147
+ ansi_session: bool = False,
148
+ ):
149
+ """Initialize the Databricks SQL graph.
150
+
151
+ Args:
152
+ schema: GraphSchema with node/relationship mappings.
153
+ connection: A ``databricks.sql`` connection (PEP 249).
154
+ catalog: Unity Catalog name (e.g. "main"). Sets default
155
+ catalog for table resolution.
156
+ schema_name: Unity Catalog schema (e.g. "graph"). Sets
157
+ default schema for table resolution.
158
+ table_stats: Optional table row counts for optimization.
159
+ ansi_session: Turn ANSI mode on for the connection, so that an
160
+ integer overflow is an error and not a wrapped number
161
+ (prograph/graph/ansi_session.py). ``close()`` puts back
162
+ the value it had. The engine passes its own
163
+ ``ansi_session`` (on by default); a graph built by hand
164
+ leaves the connection alone unless asked.
165
+ """
166
+ super().__init__(
167
+ schema, connection, dialect='databricks_sql', table_stats=table_stats
168
+ )
169
+ self._catalog = catalog
170
+ self._schema_name = schema_name
171
+ self._ansi_before: Optional[str] = None
172
+ if ansi_session:
173
+ from prograph.graph.ansi_session import connection_ansi_on
174
+ self._ansi_before = connection_ansi_on(connection)
175
+
176
+ def close(self) -> None:
177
+ """Put the connection's ANSI mode back as it was. The connection
178
+ is the caller's and stays open."""
179
+ from prograph.graph.ansi_session import connection_ansi_restore
180
+ before, self._ansi_before = self._ansi_before, None
181
+ connection_ansi_restore(self._connection, before)
182
+
183
+ def _execute_sql(self, sql: str) -> List[Dict[str, Any]]:
184
+ """Execute SQL against Databricks SQL Warehouse; rows as dicts."""
185
+ return self._execute_sql_arrow(sql).to_pylist(maps_as_pydicts="strict")
186
+
187
+ def _execute_sql_batches(self, sql: str, batch_size: int):
188
+ """Stream the warehouse's result as Arrow record batches
189
+ (cursor.fetchmany_arrow), holding at most batch_size rows at a time.
190
+ The cursor stays open while the reader is read and closes when it
191
+ is exhausted, closed or dropped."""
192
+ import pyarrow as pa
193
+ from prograph.execution import StatementGuard, checkpoint
194
+ checkpoint()
195
+ cursor = self._connection.cursor()
196
+ # The deadline and a cancel stop the statement: cursor.cancel(), queued or running.
197
+ guard = StatementGuard(cursor)
198
+ try:
199
+ try:
200
+ run_statement(cursor, sql, guard)
201
+ except Exception:
202
+ guard.check() # cancelled, or at the deadline: that, not the cancel's error
203
+ raise
204
+ guard.check()
205
+ if cursor.description is None:
206
+ guard.stop()
207
+ cursor.close()
208
+ return pa.RecordBatchReader.from_batches(pa.schema([]), [])
209
+ first = _with_columns(cursor.fetchmany_arrow(batch_size), cursor.description)
210
+ guard.check()
211
+ except BaseException:
212
+ guard.stop()
213
+ cursor.close()
214
+ raise
215
+
216
+ def batches(table):
217
+ try:
218
+ while table.num_rows:
219
+ # One batch per fetch: the connector returns its rows in chunks of its own
220
+ # (4,096 rows), which would otherwise each be a batch.
221
+ yield from table.combine_chunks().to_batches()
222
+ if table.num_rows < batch_size:
223
+ return
224
+ table = cursor.fetchmany_arrow(batch_size)
225
+ guard.check()
226
+ finally:
227
+ guard.stop()
228
+ cursor.close()
229
+
230
+ return pa.RecordBatchReader.from_batches(first.schema, batches(first))
231
+
232
+ def _execute_sql_arrow(self, sql: str):
233
+ """Execute SQL and return the warehouse's Arrow result as is.
234
+
235
+ The warehouse sends Arrow (inline or through CloudFetch) over Thrift
236
+ and SEA alike; cursor.fetchall() converts it to pandas and then to a
237
+ Python row per row (where SEA's numpy arrays came from), while
238
+ fetchall_arrow() hands it over untouched."""
239
+ import pyarrow as pa
240
+ from prograph.execution import StatementGuard, checkpoint
241
+ checkpoint()
242
+ cursor = self._connection.cursor()
243
+ # The deadline and a cancel stop the statement: cursor.cancel(), queued or running.
244
+ guard = StatementGuard(cursor)
245
+ try:
246
+ try:
247
+ run_statement(cursor, sql, guard)
248
+ if cursor.description is None:
249
+ return pa.table({})
250
+ table = _with_columns(cursor.fetchall_arrow(), cursor.description)
251
+ except Exception:
252
+ guard.check() # cancelled, or at the deadline: that, not the cancel's error
253
+ raise
254
+ guard.check()
255
+ return table
256
+ finally:
257
+ guard.stop()
258
+ cursor.close()
259
+
260
+ def describe_table(self, table_name: str) -> List[Dict[str, str]]:
261
+ """Retrieve column metadata for a table via DESCRIBE TABLE.
262
+
263
+ Args:
264
+ table_name: Fully-qualified or unqualified table name.
265
+
266
+ Returns:
267
+ List of dicts with 'col_name', 'data_type', 'comment' keys.
268
+ """
269
+ rows = self._execute_sql(f"DESCRIBE TABLE {table_name}")
270
+ # DESCRIBE TABLE returns rows with col_name, data_type, comment
271
+ # Filter out partition/metadata separator rows (empty col_name or '#')
272
+ return [
273
+ row for row in rows
274
+ if row.get('col_name') and not row['col_name'].startswith('#')
275
+ ]
276
+
277
+ def table_exists(self, table_name: str) -> bool:
278
+ """Check if a table exists in the catalog."""
279
+ try:
280
+ self._execute_sql(f"DESCRIBE TABLE {table_name}")
281
+ return True
282
+ except Exception:
283
+ return False
284
+
285
+ def set_catalog_context(self) -> None:
286
+ """Issue USE CATALOG / USE SCHEMA if configured.
287
+
288
+ Called once at construction to set the session's default catalog
289
+ and schema so unqualified table names resolve correctly.
290
+ """
291
+ if self._catalog:
292
+ self._execute_sql(f"USE CATALOG `{self._catalog}`")
293
+ if self._schema_name:
294
+ self._execute_sql(f"USE SCHEMA `{self._schema_name}`")
295
+
296
+ # ─── Mutations ───────────────────────────────────────────────
297
+
298
+
299
+
300
+
@@ -0,0 +1,143 @@
1
+ """The Arrow driver of a SQL backend.
2
+
3
+ A SQL backend's result is the Arrow its driver returns, so a backend whose
4
+ driver is missing refuses rather than building rows, and a driver's error
5
+ text, which can carry the URL or connection string it was given, is
6
+ scrubbed of credentials before it is raised or logged.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import importlib
12
+ import re
13
+ from typing import Any, Iterable, List, Sequence, Tuple
14
+ from urllib.parse import unquote, urlsplit
15
+
16
+ from prograph.utils.exceptions import BackendConnectionError, MissingDriverError
17
+
18
+ SCRUBBED = "***"
19
+ _URL_CREDENTIALS = re.compile(r"(?P<head>[A-Za-z][\w+.-]*://[^:/@\s]*):(?P<password>[^@\s]*)@")
20
+ _KEYWORD_CREDENTIALS = re.compile(
21
+ r"(?i)\b(?P<key>password|pwd)\s*=\s*(?P<value>\{[^}]*\}|'[^']*'|\"[^\"]*\"|[^;\s]*)")
22
+
23
+
24
+ def require_driver(module: str, package: str, backend: str) -> Any:
25
+ """The driver's module, or a refusal naming the package to install."""
26
+ try:
27
+ return importlib.import_module(module)
28
+ except ImportError:
29
+ raise MissingDriverError(
30
+ f"The {backend} backend reads through {package}, which is not installed "
31
+ f"(pip install {package}). It has no other result path.",
32
+ package=package, backend=backend) from None
33
+
34
+
35
+ def secrets_of(target: str) -> List[str]:
36
+ """The passwords a URL or connection string holds, as written and as
37
+ decoded, so they can be removed from text that repeats them in either
38
+ form."""
39
+ found: List[str] = []
40
+ try:
41
+ password = urlsplit(target).password
42
+ except ValueError:
43
+ password = None
44
+ if password:
45
+ found += [password, unquote(password)]
46
+ for match in _KEYWORD_CREDENTIALS.finditer(target):
47
+ value = match.group("value")
48
+ found += [value, value.strip("{}'\"")]
49
+ return [s for s in dict.fromkeys(found) if s]
50
+
51
+
52
+ def scrub_credentials(text: str, secrets: Iterable[str] = ()) -> str:
53
+ """``text`` without the given secrets, and without the password of any
54
+ URL (``scheme://user:password@``) or ``password=`` / ``pwd=`` pair in it."""
55
+ for secret in sorted(set(secrets), key=len, reverse=True):
56
+ text = text.replace(secret, SCRUBBED)
57
+ text = _URL_CREDENTIALS.sub(lambda m: f"{m.group('head')}:{SCRUBBED}@", text)
58
+ return _KEYWORD_CREDENTIALS.sub(lambda m: f"{m.group('key')}={SCRUBBED}", text)
59
+
60
+
61
+ def connection_refused(backend: str, error: BaseException, secrets: Iterable[str]) -> BackendConnectionError:
62
+ """The error for a connection that could not be opened, with the
63
+ driver's reason and none of the credentials."""
64
+ return BackendConnectionError(
65
+ f"Could not connect to {backend}: {scrub_credentials(str(error), secrets)}", backend=backend)
66
+
67
+
68
+ def fetch_rows(connection: Any, sql: str, parameters: Sequence[Any] = ()) -> Tuple[List[str], List[tuple]]:
69
+ """(column names, rows) of ``sql`` on a backend's connection, for
70
+ ProGraph's own readers of the database: the OWL reasoner, the SHACL
71
+ bridge, statistics and table descriptions. A statement
72
+ with no result gives ``([], [])``.
73
+
74
+ A query's result goes to the caller as the driver's Arrow and never
75
+ through here. These readers need Python values, and they need the same
76
+ types a query returns: on PostgreSQL the ADBC driver gives a ``numeric``
77
+ as a string and a ``uuid`` as bytes, so the statement is read as a
78
+ result is (``postgresql_graph.read_table``: a numeric a double, so a
79
+ NaN in a scanned table is a value; a uuid its text). On Oracle a zoned
80
+ timestamp is read as its UTC instant (``oracle_graph.utc_statement``), as
81
+ a result is; the driver would give the naive time in its own zone.
82
+ """
83
+ if hasattr(connection, "adbc_get_info"): # ADBC: PostgreSQL
84
+ from .postgresql_graph import read_table
85
+ table = read_table(connection, sql, parameters)
86
+ if table is None:
87
+ return [], []
88
+ return table.column_names, list(zip(*(c.to_pylist() for c in table.columns)))
89
+ if hasattr(connection, "fetch_df_all"): # oracledb: zoned timestamps as UTC
90
+ from .oracle_graph import utc_statement
91
+ cursor = connection.cursor()
92
+ statement = utc_statement(connection, sql)
93
+ # A LOB as its value, not the driver's LOB handle.
94
+ cursor.execute(statement, parameters or None, fetch_lobs=False)
95
+ elif hasattr(connection, "cursor"):
96
+ cursor = connection.cursor()
97
+ cursor.execute(sql, parameters) if parameters else cursor.execute(sql)
98
+ else:
99
+ cursor = connection.execute(sql, parameters) if parameters else connection.execute(sql)
100
+ try:
101
+ if not cursor.description:
102
+ return [], []
103
+ return [column[0] for column in cursor.description], cursor.fetchall()
104
+ finally:
105
+ if cursor is not connection:
106
+ cursor.close()
107
+
108
+
109
+ def duckdb_arrow_table(connection: Any, sql: str, parameters: Any = None) -> Any:
110
+ """A DuckDB statement's rows as an Arrow table, read through the relation API.
111
+
112
+ DuckDB 2.0 warns that Arrow output of a DB-API result (`execute(...)` then
113
+ `to_arrow_table()`) copies the rows; a relation (`connection.sql`) hands
114
+ them over without the copy, and answers alike on 1.5. A statement that
115
+ is no query (an INSERT, a CREATE: DuckDB 1.5 runs it and returns no
116
+ relation, 2.0 returns its count) is run as a statement and its count read
117
+ as it always was; it holds no rows worth a copy.
118
+ """
119
+ import duckdb
120
+ import pyarrow as pa
121
+ # DuckDB 2.0 keeps a statement's unfinished result (`execute(...).fetchone()` reads one row of it) open on
122
+ # the connection and refuses a relation until it is closed; `execute` closed it itself. `abort` closes it
123
+ # and leaves the transaction as it was; 1.5 has no such state.
124
+ if hasattr(connection, "abort"):
125
+ connection.abort()
126
+ statements = connection.extract_statements(sql)
127
+ if statements and statements[-1].type == duckdb.StatementType.SELECT:
128
+ relation = connection.sql(sql, params=parameters) if parameters else connection.sql(sql)
129
+ if relation is not None:
130
+ return relation.to_arrow_table()
131
+ cursor = connection.execute(sql, parameters) if parameters else connection.execute(sql)
132
+ if cursor.description is None:
133
+ return pa.table({})
134
+ import warnings
135
+ with warnings.catch_warnings():
136
+ warnings.filterwarnings("ignore", message="Arrow output on a DB-API result")
137
+ return cursor.to_arrow_table()
138
+
139
+
140
+ def duckdb_arrow_reader(connection: Any, sql: str, batch_size: int) -> Any:
141
+ """A DuckDB query's rows as an Arrow reader of at most `batch_size` rows a batch, through the
142
+ relation API (`duckdb_arrow_table` says why)."""
143
+ return connection.sql(sql).to_arrow_reader(batch_size)
@@ -0,0 +1,68 @@
1
+ """DuckDB Graph Backend.
2
+
3
+ Embedded SQL graph backend, zero infrastructure, every query in SQL.
4
+ DuckDB runs in-process; no server, no Docker, no credentials.
5
+
6
+ Usage:
7
+
8
+ import duckdb
9
+ conn = duckdb.connect(":memory:") # or "my_graph.duckdb"
10
+ # Load data from Parquet, CSV, Delta, or INSERT directly
11
+ conn.execute("CREATE TABLE persons AS SELECT * FROM 'persons.parquet'")
12
+ graph = DuckDBGraph(schema, conn)
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Any, Dict, List, Optional
18
+
19
+ from .sql_graph import SQLGraph
20
+
21
+
22
+ class DuckDBGraph(SQLGraph):
23
+ """DuckDB-backed graph.
24
+
25
+ Embedded columnar engine, no server process. Supports Parquet,
26
+ CSV, Delta Lake, and JSON as data sources via DuckDB extensions.
27
+ """
28
+
29
+ def __init__(
30
+ self,
31
+ schema: Any,
32
+ connection: Any,
33
+ table_stats: Optional[Dict[str, int]] = None,
34
+ ):
35
+ super().__init__(schema, connection, dialect='duckdb', table_stats=table_stats)
36
+
37
+ def _execute_sql(self, sql: str) -> List[Dict[str, Any]]:
38
+ """Execute SQL against DuckDB."""
39
+ from prograph.execution import StatementGuard
40
+ # A cancel interrupts the connection's statement (`StatementGuard`).
41
+ with StatementGuard(stop=self._connection.interrupt):
42
+ result = self._connection.execute(sql)
43
+ if result.description is None:
44
+ return []
45
+ columns = [desc[0] for desc in result.description]
46
+ return [dict(zip(columns, row)) for row in result.fetchall()]
47
+
48
+ def _execute_sql_batches(self, sql: str, batch_size: int):
49
+ """Stream the result as Arrow record batches, on a cursor of its own
50
+ (the reader is read after execute() returns, and another query on
51
+ the shared connection would end its result). A cancel interrupts the
52
+ cursor's statement until the reader ends."""
53
+ from prograph.execution import StatementGuard, checkpoint
54
+ from prograph.graph.drivers import duckdb_arrow_reader
55
+ checkpoint()
56
+ cursor = self._connection.cursor()
57
+ guard = StatementGuard(stop=cursor.interrupt)
58
+ with guard.opening(close=cursor.close):
59
+ reader = duckdb_arrow_reader(cursor, sql, batch_size)
60
+ return guard.reader(reader, close=cursor.close)
61
+
62
+ def _execute_sql_arrow(self, sql: str):
63
+ """Execute SQL against DuckDB and return its Arrow table as is."""
64
+ from prograph.execution import StatementGuard
65
+ from prograph.graph.drivers import duckdb_arrow_table
66
+ # A cancel interrupts the connection's statement (`StatementGuard`).
67
+ with StatementGuard(stop=self._connection.interrupt):
68
+ return duckdb_arrow_table(self._connection, sql)
@@ -0,0 +1,38 @@
1
+ """MySQL Graph Backend.
2
+
3
+ Executes SQL queries against a MySQL 8.0+ database via the
4
+ ``mysql-connector-python`` driver (PEP 249).
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Any, Dict, Optional
10
+
11
+ from .sql_graph import SQLGraph
12
+
13
+
14
+ class MySQLGraph(SQLGraph):
15
+ """MySQL-backed graph."""
16
+
17
+ def __init__(
18
+ self,
19
+ schema: Any,
20
+ connection: Any,
21
+ table_stats: Optional[Dict[str, int]] = None,
22
+ ):
23
+ super().__init__(schema, connection, dialect='mysql', table_stats=table_stats)
24
+
25
+ def _fetch(self, sql: str):
26
+ """Execute SQL against MySQL. mysql-connector has no cancel for a
27
+ running statement: a cancelled query stops before its next statement
28
+ (`StatementGuard` with no stop)."""
29
+ from prograph.execution import StatementGuard
30
+ with StatementGuard():
31
+ cursor = self._connection.cursor()
32
+ try:
33
+ cursor.execute(sql)
34
+ if cursor.description is None:
35
+ return [], []
36
+ return [desc[0] for desc in cursor.description], cursor.fetchall()
37
+ finally:
38
+ cursor.close()