prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
"""Databricks SQL Graph Backend.
|
|
2
|
+
|
|
3
|
+
A pure-SQL graph backend for Databricks SQL Warehouses (Photon).
|
|
4
|
+
No DataFrames and no SparkSession: all operations are SQL over a PEP 249
|
|
5
|
+
connection from ``databricks-sql-connector``.
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
|
|
9
|
+
from databricks import sql as databricks_sql
|
|
10
|
+
conn = databricks_sql.connect(
|
|
11
|
+
server_hostname="...", http_path="...", access_token="..."
|
|
12
|
+
)
|
|
13
|
+
graph = DatabricksSQLGraph(schema, conn, catalog="main", schema_name="graph")
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import logging
|
|
19
|
+
from typing import Any, Dict, List, Optional
|
|
20
|
+
|
|
21
|
+
from prograph.rules import home
|
|
22
|
+
|
|
23
|
+
from .sql_graph import SQLGraph
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# How often a submitted statement's state is asked for: from the first interval, each wait half as long
|
|
30
|
+
# again as the one before, to at most the last (a short statement pays a few round trips; a long one a
|
|
31
|
+
# poll a second).
|
|
32
|
+
POLL_FIRST, POLL_GROWTH, POLL_MOST = 0.05, 1.5, 1.0
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# The Arrow type the Thrift protocol gives each Databricks type in a result (the connector asks for Arrow
|
|
36
|
+
# timestamps, decimals and complex types: `_use_arrow_native_*`; intervals arrive as text). Keys are the
|
|
37
|
+
# description's type codes: Thrift's TTypeId names lowered without `_TYPE`, which SEA's are normalized to
|
|
38
|
+
# (`byte`, `short`, `long` kept as well, should a description carry SEA's own names).
|
|
39
|
+
_THRIFT_ARROW_TYPES = {
|
|
40
|
+
"string": "string", "varchar": "string", "char": "string",
|
|
41
|
+
"interval_year_month": "string", "interval_day_time": "string", "interval": "string",
|
|
42
|
+
"tinyint": "int8", "byte": "int8", "smallint": "int16", "short": "int16",
|
|
43
|
+
"int": "int32", "integer": "int32", "bigint": "int64", "long": "int64",
|
|
44
|
+
"boolean": "bool", "float": "float32", "double": "float64",
|
|
45
|
+
"date": "date32", "binary": "binary",
|
|
46
|
+
# An untyped NULL column: SEA's description says `null` (or `void`), Thrift's says `string` and its Arrow
|
|
47
|
+
# column is a string, so an empty answer has one schema on either path.
|
|
48
|
+
"null": "string", "void": "string",
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@home("databricks-empty-answer", "A Databricks answer of no rows, with its columns",
|
|
53
|
+
terms=("SEA", "Statement Execution API", "empty answer", "0 columns", "fetchall_arrow", "description"))
|
|
54
|
+
def empty_answer(description: Any):
|
|
55
|
+
"""The empty Arrow table of a statement whose answer has no rows, its fields from the cursor's
|
|
56
|
+
`description`: over the Statement Execution API (SEA) the connector has no Arrow schema for a result
|
|
57
|
+
(`arrow_schema_bytes` is None), so an empty answer arrives as a table of 0 columns, where Thrift sends
|
|
58
|
+
the columns. Each field is the description's name, at the type Thrift gives that Databricks type
|
|
59
|
+
(`_THRIFT_ARROW_TYPES`; a decimal its precision and scale, a TIMESTAMP an instant in UTC, a
|
|
60
|
+
TIMESTAMP_NTZ a naive one). An ARRAY, STRUCT, MAP or VARIANT (or a type not listed) is `pa.null()`: the
|
|
61
|
+
description names the kind but not its element or field types, and a guessed type (a list of
|
|
62
|
+
strings, a struct of no field) would be a wrong type that a reader may act on, where a null column
|
|
63
|
+
holds no value and casts to any type a consumer needs (concatenated with a later batch, compared
|
|
64
|
+
with another engine's answer)."""
|
|
65
|
+
import pyarrow as pa
|
|
66
|
+
fields = []
|
|
67
|
+
for column in description:
|
|
68
|
+
name = column[0]
|
|
69
|
+
code = str(column[1] if len(column) > 1 and column[1] is not None else "").lower()
|
|
70
|
+
if code == "decimal" and len(column) > 5 and column[4] is not None and column[5] is not None:
|
|
71
|
+
kind = pa.decimal128(int(column[4]), int(column[5]))
|
|
72
|
+
elif code == "timestamp":
|
|
73
|
+
kind = pa.timestamp("us", tz="Etc/UTC")
|
|
74
|
+
elif code == "timestamp_ntz":
|
|
75
|
+
kind = pa.timestamp("us")
|
|
76
|
+
else:
|
|
77
|
+
kind = pa.type_for_alias(_THRIFT_ARROW_TYPES.get(code, "null"))
|
|
78
|
+
fields.append(pa.field(name, kind))
|
|
79
|
+
return pa.schema(fields).empty_table()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _with_columns(table: Any, description: Any):
|
|
83
|
+
"""`table` as the driver gave it; an empty answer of 0 columns whose statement has columns (SEA,
|
|
84
|
+
`empty_answer`) as the empty table of those columns. Only a 0-column table is replaced, so every
|
|
85
|
+
answer that has its columns is returned untouched."""
|
|
86
|
+
if table.num_columns == 0 and description:
|
|
87
|
+
return empty_answer(description)
|
|
88
|
+
return table
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def run_statement(cursor: Any, sql: str, guard: Any) -> None:
|
|
92
|
+
"""Run `sql` on `cursor`: blocking, its result in the first response, unless the query is expected to
|
|
93
|
+
run long (`execution.long_expected`: a closure or a costly shape), where it runs so that a cancel
|
|
94
|
+
reaches it at once: submitted without waiting
|
|
95
|
+
(`execute_async`), which gives the cursor its command id as soon as the warehouse has the
|
|
96
|
+
statement, then its state polled until it ends and its result taken. A blocking `execute` holds
|
|
97
|
+
the call, with no command id, for the warehouse's first wait (several seconds), and a
|
|
98
|
+
`cursor.cancel()` in that window has nothing to cancel (measured on a SQL warehouse: the stop
|
|
99
|
+
landed 6-7 s after the cancel). The stop is sent at once; a cancel or the deadline (`guard`) ends the
|
|
100
|
+
wait at the next poll (at most a second). A
|
|
101
|
+
connector without `execute_async`, or a backend that refuses it (NotSupportedError,
|
|
102
|
+
NotImplementedError: checked against the kernel backend of a Lakehouse RT warehouse), runs
|
|
103
|
+
`execute`."""
|
|
104
|
+
import time
|
|
105
|
+
from prograph.execution import long_expected
|
|
106
|
+
submit = getattr(cursor, "execute_async", None) if long_expected() else None
|
|
107
|
+
session = getattr(getattr(cursor, "connection", None), "session", None)
|
|
108
|
+
if getattr(session, "use_sea", False) is True:
|
|
109
|
+
# SEA's async result raises a bare "Operation status FAILED", without the server's text, which the
|
|
110
|
+
# closure ladder (a depth guard, Lakehouse RT's 0A000) and the typed errors read: run blocking there.
|
|
111
|
+
submit = None
|
|
112
|
+
if submit is None:
|
|
113
|
+
cursor.execute(sql)
|
|
114
|
+
return
|
|
115
|
+
try:
|
|
116
|
+
submit(sql)
|
|
117
|
+
except NotImplementedError:
|
|
118
|
+
cursor.execute(sql)
|
|
119
|
+
return
|
|
120
|
+
except Exception as error: # noqa: BLE001 - the connector's NotSupportedError (or a subclass), by name
|
|
121
|
+
if not any(k.__name__ == "NotSupportedError" for k in type(error).__mro__):
|
|
122
|
+
raise
|
|
123
|
+
cursor.execute(sql)
|
|
124
|
+
return
|
|
125
|
+
wait = POLL_FIRST
|
|
126
|
+
while cursor.is_query_pending():
|
|
127
|
+
guard.check()
|
|
128
|
+
time.sleep(wait)
|
|
129
|
+
wait = min(wait * POLL_GROWTH, POLL_MOST)
|
|
130
|
+
guard.check()
|
|
131
|
+
cursor.get_async_execution_result()
|
|
132
|
+
|
|
133
|
+
class DatabricksSQLGraph(SQLGraph):
|
|
134
|
+
"""Databricks SQL Warehouse-backed graph.
|
|
135
|
+
|
|
136
|
+
Executes SQL via the ``databricks-sql-connector`` PEP 249 cursor.
|
|
137
|
+
All queries go to Photon, no Spark compute is involved.
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
def __init__(
|
|
141
|
+
self,
|
|
142
|
+
schema: Any,
|
|
143
|
+
connection: Any,
|
|
144
|
+
catalog: Optional[str] = None,
|
|
145
|
+
schema_name: Optional[str] = None,
|
|
146
|
+
table_stats: Optional[Dict[str, int]] = None,
|
|
147
|
+
ansi_session: bool = False,
|
|
148
|
+
):
|
|
149
|
+
"""Initialize the Databricks SQL graph.
|
|
150
|
+
|
|
151
|
+
Args:
|
|
152
|
+
schema: GraphSchema with node/relationship mappings.
|
|
153
|
+
connection: A ``databricks.sql`` connection (PEP 249).
|
|
154
|
+
catalog: Unity Catalog name (e.g. "main"). Sets default
|
|
155
|
+
catalog for table resolution.
|
|
156
|
+
schema_name: Unity Catalog schema (e.g. "graph"). Sets
|
|
157
|
+
default schema for table resolution.
|
|
158
|
+
table_stats: Optional table row counts for optimization.
|
|
159
|
+
ansi_session: Turn ANSI mode on for the connection, so that an
|
|
160
|
+
integer overflow is an error and not a wrapped number
|
|
161
|
+
(prograph/graph/ansi_session.py). ``close()`` puts back
|
|
162
|
+
the value it had. The engine passes its own
|
|
163
|
+
``ansi_session`` (on by default); a graph built by hand
|
|
164
|
+
leaves the connection alone unless asked.
|
|
165
|
+
"""
|
|
166
|
+
super().__init__(
|
|
167
|
+
schema, connection, dialect='databricks_sql', table_stats=table_stats
|
|
168
|
+
)
|
|
169
|
+
self._catalog = catalog
|
|
170
|
+
self._schema_name = schema_name
|
|
171
|
+
self._ansi_before: Optional[str] = None
|
|
172
|
+
if ansi_session:
|
|
173
|
+
from prograph.graph.ansi_session import connection_ansi_on
|
|
174
|
+
self._ansi_before = connection_ansi_on(connection)
|
|
175
|
+
|
|
176
|
+
def close(self) -> None:
|
|
177
|
+
"""Put the connection's ANSI mode back as it was. The connection
|
|
178
|
+
is the caller's and stays open."""
|
|
179
|
+
from prograph.graph.ansi_session import connection_ansi_restore
|
|
180
|
+
before, self._ansi_before = self._ansi_before, None
|
|
181
|
+
connection_ansi_restore(self._connection, before)
|
|
182
|
+
|
|
183
|
+
def _execute_sql(self, sql: str) -> List[Dict[str, Any]]:
|
|
184
|
+
"""Execute SQL against Databricks SQL Warehouse; rows as dicts."""
|
|
185
|
+
return self._execute_sql_arrow(sql).to_pylist(maps_as_pydicts="strict")
|
|
186
|
+
|
|
187
|
+
def _execute_sql_batches(self, sql: str, batch_size: int):
|
|
188
|
+
"""Stream the warehouse's result as Arrow record batches
|
|
189
|
+
(cursor.fetchmany_arrow), holding at most batch_size rows at a time.
|
|
190
|
+
The cursor stays open while the reader is read and closes when it
|
|
191
|
+
is exhausted, closed or dropped."""
|
|
192
|
+
import pyarrow as pa
|
|
193
|
+
from prograph.execution import StatementGuard, checkpoint
|
|
194
|
+
checkpoint()
|
|
195
|
+
cursor = self._connection.cursor()
|
|
196
|
+
# The deadline and a cancel stop the statement: cursor.cancel(), queued or running.
|
|
197
|
+
guard = StatementGuard(cursor)
|
|
198
|
+
try:
|
|
199
|
+
try:
|
|
200
|
+
run_statement(cursor, sql, guard)
|
|
201
|
+
except Exception:
|
|
202
|
+
guard.check() # cancelled, or at the deadline: that, not the cancel's error
|
|
203
|
+
raise
|
|
204
|
+
guard.check()
|
|
205
|
+
if cursor.description is None:
|
|
206
|
+
guard.stop()
|
|
207
|
+
cursor.close()
|
|
208
|
+
return pa.RecordBatchReader.from_batches(pa.schema([]), [])
|
|
209
|
+
first = _with_columns(cursor.fetchmany_arrow(batch_size), cursor.description)
|
|
210
|
+
guard.check()
|
|
211
|
+
except BaseException:
|
|
212
|
+
guard.stop()
|
|
213
|
+
cursor.close()
|
|
214
|
+
raise
|
|
215
|
+
|
|
216
|
+
def batches(table):
|
|
217
|
+
try:
|
|
218
|
+
while table.num_rows:
|
|
219
|
+
# One batch per fetch: the connector returns its rows in chunks of its own
|
|
220
|
+
# (4,096 rows), which would otherwise each be a batch.
|
|
221
|
+
yield from table.combine_chunks().to_batches()
|
|
222
|
+
if table.num_rows < batch_size:
|
|
223
|
+
return
|
|
224
|
+
table = cursor.fetchmany_arrow(batch_size)
|
|
225
|
+
guard.check()
|
|
226
|
+
finally:
|
|
227
|
+
guard.stop()
|
|
228
|
+
cursor.close()
|
|
229
|
+
|
|
230
|
+
return pa.RecordBatchReader.from_batches(first.schema, batches(first))
|
|
231
|
+
|
|
232
|
+
def _execute_sql_arrow(self, sql: str):
|
|
233
|
+
"""Execute SQL and return the warehouse's Arrow result as is.
|
|
234
|
+
|
|
235
|
+
The warehouse sends Arrow (inline or through CloudFetch) over Thrift
|
|
236
|
+
and SEA alike; cursor.fetchall() converts it to pandas and then to a
|
|
237
|
+
Python row per row (where SEA's numpy arrays came from), while
|
|
238
|
+
fetchall_arrow() hands it over untouched."""
|
|
239
|
+
import pyarrow as pa
|
|
240
|
+
from prograph.execution import StatementGuard, checkpoint
|
|
241
|
+
checkpoint()
|
|
242
|
+
cursor = self._connection.cursor()
|
|
243
|
+
# The deadline and a cancel stop the statement: cursor.cancel(), queued or running.
|
|
244
|
+
guard = StatementGuard(cursor)
|
|
245
|
+
try:
|
|
246
|
+
try:
|
|
247
|
+
run_statement(cursor, sql, guard)
|
|
248
|
+
if cursor.description is None:
|
|
249
|
+
return pa.table({})
|
|
250
|
+
table = _with_columns(cursor.fetchall_arrow(), cursor.description)
|
|
251
|
+
except Exception:
|
|
252
|
+
guard.check() # cancelled, or at the deadline: that, not the cancel's error
|
|
253
|
+
raise
|
|
254
|
+
guard.check()
|
|
255
|
+
return table
|
|
256
|
+
finally:
|
|
257
|
+
guard.stop()
|
|
258
|
+
cursor.close()
|
|
259
|
+
|
|
260
|
+
def describe_table(self, table_name: str) -> List[Dict[str, str]]:
|
|
261
|
+
"""Retrieve column metadata for a table via DESCRIBE TABLE.
|
|
262
|
+
|
|
263
|
+
Args:
|
|
264
|
+
table_name: Fully-qualified or unqualified table name.
|
|
265
|
+
|
|
266
|
+
Returns:
|
|
267
|
+
List of dicts with 'col_name', 'data_type', 'comment' keys.
|
|
268
|
+
"""
|
|
269
|
+
rows = self._execute_sql(f"DESCRIBE TABLE {table_name}")
|
|
270
|
+
# DESCRIBE TABLE returns rows with col_name, data_type, comment
|
|
271
|
+
# Filter out partition/metadata separator rows (empty col_name or '#')
|
|
272
|
+
return [
|
|
273
|
+
row for row in rows
|
|
274
|
+
if row.get('col_name') and not row['col_name'].startswith('#')
|
|
275
|
+
]
|
|
276
|
+
|
|
277
|
+
def table_exists(self, table_name: str) -> bool:
|
|
278
|
+
"""Check if a table exists in the catalog."""
|
|
279
|
+
try:
|
|
280
|
+
self._execute_sql(f"DESCRIBE TABLE {table_name}")
|
|
281
|
+
return True
|
|
282
|
+
except Exception:
|
|
283
|
+
return False
|
|
284
|
+
|
|
285
|
+
def set_catalog_context(self) -> None:
|
|
286
|
+
"""Issue USE CATALOG / USE SCHEMA if configured.
|
|
287
|
+
|
|
288
|
+
Called once at construction to set the session's default catalog
|
|
289
|
+
and schema so unqualified table names resolve correctly.
|
|
290
|
+
"""
|
|
291
|
+
if self._catalog:
|
|
292
|
+
self._execute_sql(f"USE CATALOG `{self._catalog}`")
|
|
293
|
+
if self._schema_name:
|
|
294
|
+
self._execute_sql(f"USE SCHEMA `{self._schema_name}`")
|
|
295
|
+
|
|
296
|
+
# ─── Mutations ───────────────────────────────────────────────
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""The Arrow driver of a SQL backend.
|
|
2
|
+
|
|
3
|
+
A SQL backend's result is the Arrow its driver returns, so a backend whose
|
|
4
|
+
driver is missing refuses rather than building rows, and a driver's error
|
|
5
|
+
text, which can carry the URL or connection string it was given, is
|
|
6
|
+
scrubbed of credentials before it is raised or logged.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import importlib
|
|
12
|
+
import re
|
|
13
|
+
from typing import Any, Iterable, List, Sequence, Tuple
|
|
14
|
+
from urllib.parse import unquote, urlsplit
|
|
15
|
+
|
|
16
|
+
from prograph.utils.exceptions import BackendConnectionError, MissingDriverError
|
|
17
|
+
|
|
18
|
+
SCRUBBED = "***"
|
|
19
|
+
_URL_CREDENTIALS = re.compile(r"(?P<head>[A-Za-z][\w+.-]*://[^:/@\s]*):(?P<password>[^@\s]*)@")
|
|
20
|
+
_KEYWORD_CREDENTIALS = re.compile(
|
|
21
|
+
r"(?i)\b(?P<key>password|pwd)\s*=\s*(?P<value>\{[^}]*\}|'[^']*'|\"[^\"]*\"|[^;\s]*)")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def require_driver(module: str, package: str, backend: str) -> Any:
|
|
25
|
+
"""The driver's module, or a refusal naming the package to install."""
|
|
26
|
+
try:
|
|
27
|
+
return importlib.import_module(module)
|
|
28
|
+
except ImportError:
|
|
29
|
+
raise MissingDriverError(
|
|
30
|
+
f"The {backend} backend reads through {package}, which is not installed "
|
|
31
|
+
f"(pip install {package}). It has no other result path.",
|
|
32
|
+
package=package, backend=backend) from None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def secrets_of(target: str) -> List[str]:
|
|
36
|
+
"""The passwords a URL or connection string holds, as written and as
|
|
37
|
+
decoded, so they can be removed from text that repeats them in either
|
|
38
|
+
form."""
|
|
39
|
+
found: List[str] = []
|
|
40
|
+
try:
|
|
41
|
+
password = urlsplit(target).password
|
|
42
|
+
except ValueError:
|
|
43
|
+
password = None
|
|
44
|
+
if password:
|
|
45
|
+
found += [password, unquote(password)]
|
|
46
|
+
for match in _KEYWORD_CREDENTIALS.finditer(target):
|
|
47
|
+
value = match.group("value")
|
|
48
|
+
found += [value, value.strip("{}'\"")]
|
|
49
|
+
return [s for s in dict.fromkeys(found) if s]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def scrub_credentials(text: str, secrets: Iterable[str] = ()) -> str:
|
|
53
|
+
"""``text`` without the given secrets, and without the password of any
|
|
54
|
+
URL (``scheme://user:password@``) or ``password=`` / ``pwd=`` pair in it."""
|
|
55
|
+
for secret in sorted(set(secrets), key=len, reverse=True):
|
|
56
|
+
text = text.replace(secret, SCRUBBED)
|
|
57
|
+
text = _URL_CREDENTIALS.sub(lambda m: f"{m.group('head')}:{SCRUBBED}@", text)
|
|
58
|
+
return _KEYWORD_CREDENTIALS.sub(lambda m: f"{m.group('key')}={SCRUBBED}", text)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def connection_refused(backend: str, error: BaseException, secrets: Iterable[str]) -> BackendConnectionError:
|
|
62
|
+
"""The error for a connection that could not be opened, with the
|
|
63
|
+
driver's reason and none of the credentials."""
|
|
64
|
+
return BackendConnectionError(
|
|
65
|
+
f"Could not connect to {backend}: {scrub_credentials(str(error), secrets)}", backend=backend)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def fetch_rows(connection: Any, sql: str, parameters: Sequence[Any] = ()) -> Tuple[List[str], List[tuple]]:
|
|
69
|
+
"""(column names, rows) of ``sql`` on a backend's connection, for
|
|
70
|
+
ProGraph's own readers of the database: the OWL reasoner, the SHACL
|
|
71
|
+
bridge, statistics and table descriptions. A statement
|
|
72
|
+
with no result gives ``([], [])``.
|
|
73
|
+
|
|
74
|
+
A query's result goes to the caller as the driver's Arrow and never
|
|
75
|
+
through here. These readers need Python values, and they need the same
|
|
76
|
+
types a query returns: on PostgreSQL the ADBC driver gives a ``numeric``
|
|
77
|
+
as a string and a ``uuid`` as bytes, so the statement is read as a
|
|
78
|
+
result is (``postgresql_graph.read_table``: a numeric a double, so a
|
|
79
|
+
NaN in a scanned table is a value; a uuid its text). On Oracle a zoned
|
|
80
|
+
timestamp is read as its UTC instant (``oracle_graph.utc_statement``), as
|
|
81
|
+
a result is; the driver would give the naive time in its own zone.
|
|
82
|
+
"""
|
|
83
|
+
if hasattr(connection, "adbc_get_info"): # ADBC: PostgreSQL
|
|
84
|
+
from .postgresql_graph import read_table
|
|
85
|
+
table = read_table(connection, sql, parameters)
|
|
86
|
+
if table is None:
|
|
87
|
+
return [], []
|
|
88
|
+
return table.column_names, list(zip(*(c.to_pylist() for c in table.columns)))
|
|
89
|
+
if hasattr(connection, "fetch_df_all"): # oracledb: zoned timestamps as UTC
|
|
90
|
+
from .oracle_graph import utc_statement
|
|
91
|
+
cursor = connection.cursor()
|
|
92
|
+
statement = utc_statement(connection, sql)
|
|
93
|
+
# A LOB as its value, not the driver's LOB handle.
|
|
94
|
+
cursor.execute(statement, parameters or None, fetch_lobs=False)
|
|
95
|
+
elif hasattr(connection, "cursor"):
|
|
96
|
+
cursor = connection.cursor()
|
|
97
|
+
cursor.execute(sql, parameters) if parameters else cursor.execute(sql)
|
|
98
|
+
else:
|
|
99
|
+
cursor = connection.execute(sql, parameters) if parameters else connection.execute(sql)
|
|
100
|
+
try:
|
|
101
|
+
if not cursor.description:
|
|
102
|
+
return [], []
|
|
103
|
+
return [column[0] for column in cursor.description], cursor.fetchall()
|
|
104
|
+
finally:
|
|
105
|
+
if cursor is not connection:
|
|
106
|
+
cursor.close()
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def duckdb_arrow_table(connection: Any, sql: str, parameters: Any = None) -> Any:
|
|
110
|
+
"""A DuckDB statement's rows as an Arrow table, read through the relation API.
|
|
111
|
+
|
|
112
|
+
DuckDB 2.0 warns that Arrow output of a DB-API result (`execute(...)` then
|
|
113
|
+
`to_arrow_table()`) copies the rows; a relation (`connection.sql`) hands
|
|
114
|
+
them over without the copy, and answers alike on 1.5. A statement that
|
|
115
|
+
is no query (an INSERT, a CREATE: DuckDB 1.5 runs it and returns no
|
|
116
|
+
relation, 2.0 returns its count) is run as a statement and its count read
|
|
117
|
+
as it always was; it holds no rows worth a copy.
|
|
118
|
+
"""
|
|
119
|
+
import duckdb
|
|
120
|
+
import pyarrow as pa
|
|
121
|
+
# DuckDB 2.0 keeps a statement's unfinished result (`execute(...).fetchone()` reads one row of it) open on
|
|
122
|
+
# the connection and refuses a relation until it is closed; `execute` closed it itself. `abort` closes it
|
|
123
|
+
# and leaves the transaction as it was; 1.5 has no such state.
|
|
124
|
+
if hasattr(connection, "abort"):
|
|
125
|
+
connection.abort()
|
|
126
|
+
statements = connection.extract_statements(sql)
|
|
127
|
+
if statements and statements[-1].type == duckdb.StatementType.SELECT:
|
|
128
|
+
relation = connection.sql(sql, params=parameters) if parameters else connection.sql(sql)
|
|
129
|
+
if relation is not None:
|
|
130
|
+
return relation.to_arrow_table()
|
|
131
|
+
cursor = connection.execute(sql, parameters) if parameters else connection.execute(sql)
|
|
132
|
+
if cursor.description is None:
|
|
133
|
+
return pa.table({})
|
|
134
|
+
import warnings
|
|
135
|
+
with warnings.catch_warnings():
|
|
136
|
+
warnings.filterwarnings("ignore", message="Arrow output on a DB-API result")
|
|
137
|
+
return cursor.to_arrow_table()
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def duckdb_arrow_reader(connection: Any, sql: str, batch_size: int) -> Any:
|
|
141
|
+
"""A DuckDB query's rows as an Arrow reader of at most `batch_size` rows a batch, through the
|
|
142
|
+
relation API (`duckdb_arrow_table` says why)."""
|
|
143
|
+
return connection.sql(sql).to_arrow_reader(batch_size)
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""DuckDB Graph Backend.
|
|
2
|
+
|
|
3
|
+
Embedded SQL graph backend, zero infrastructure, every query in SQL.
|
|
4
|
+
DuckDB runs in-process; no server, no Docker, no credentials.
|
|
5
|
+
|
|
6
|
+
Usage:
|
|
7
|
+
|
|
8
|
+
import duckdb
|
|
9
|
+
conn = duckdb.connect(":memory:") # or "my_graph.duckdb"
|
|
10
|
+
# Load data from Parquet, CSV, Delta, or INSERT directly
|
|
11
|
+
conn.execute("CREATE TABLE persons AS SELECT * FROM 'persons.parquet'")
|
|
12
|
+
graph = DuckDBGraph(schema, conn)
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from typing import Any, Dict, List, Optional
|
|
18
|
+
|
|
19
|
+
from .sql_graph import SQLGraph
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class DuckDBGraph(SQLGraph):
|
|
23
|
+
"""DuckDB-backed graph.
|
|
24
|
+
|
|
25
|
+
Embedded columnar engine, no server process. Supports Parquet,
|
|
26
|
+
CSV, Delta Lake, and JSON as data sources via DuckDB extensions.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
schema: Any,
|
|
32
|
+
connection: Any,
|
|
33
|
+
table_stats: Optional[Dict[str, int]] = None,
|
|
34
|
+
):
|
|
35
|
+
super().__init__(schema, connection, dialect='duckdb', table_stats=table_stats)
|
|
36
|
+
|
|
37
|
+
def _execute_sql(self, sql: str) -> List[Dict[str, Any]]:
|
|
38
|
+
"""Execute SQL against DuckDB."""
|
|
39
|
+
from prograph.execution import StatementGuard
|
|
40
|
+
# A cancel interrupts the connection's statement (`StatementGuard`).
|
|
41
|
+
with StatementGuard(stop=self._connection.interrupt):
|
|
42
|
+
result = self._connection.execute(sql)
|
|
43
|
+
if result.description is None:
|
|
44
|
+
return []
|
|
45
|
+
columns = [desc[0] for desc in result.description]
|
|
46
|
+
return [dict(zip(columns, row)) for row in result.fetchall()]
|
|
47
|
+
|
|
48
|
+
def _execute_sql_batches(self, sql: str, batch_size: int):
|
|
49
|
+
"""Stream the result as Arrow record batches, on a cursor of its own
|
|
50
|
+
(the reader is read after execute() returns, and another query on
|
|
51
|
+
the shared connection would end its result). A cancel interrupts the
|
|
52
|
+
cursor's statement until the reader ends."""
|
|
53
|
+
from prograph.execution import StatementGuard, checkpoint
|
|
54
|
+
from prograph.graph.drivers import duckdb_arrow_reader
|
|
55
|
+
checkpoint()
|
|
56
|
+
cursor = self._connection.cursor()
|
|
57
|
+
guard = StatementGuard(stop=cursor.interrupt)
|
|
58
|
+
with guard.opening(close=cursor.close):
|
|
59
|
+
reader = duckdb_arrow_reader(cursor, sql, batch_size)
|
|
60
|
+
return guard.reader(reader, close=cursor.close)
|
|
61
|
+
|
|
62
|
+
def _execute_sql_arrow(self, sql: str):
|
|
63
|
+
"""Execute SQL against DuckDB and return its Arrow table as is."""
|
|
64
|
+
from prograph.execution import StatementGuard
|
|
65
|
+
from prograph.graph.drivers import duckdb_arrow_table
|
|
66
|
+
# A cancel interrupts the connection's statement (`StatementGuard`).
|
|
67
|
+
with StatementGuard(stop=self._connection.interrupt):
|
|
68
|
+
return duckdb_arrow_table(self._connection, sql)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""MySQL Graph Backend.
|
|
2
|
+
|
|
3
|
+
Executes SQL queries against a MySQL 8.0+ database via the
|
|
4
|
+
``mysql-connector-python`` driver (PEP 249).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any, Dict, Optional
|
|
10
|
+
|
|
11
|
+
from .sql_graph import SQLGraph
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MySQLGraph(SQLGraph):
|
|
15
|
+
"""MySQL-backed graph."""
|
|
16
|
+
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
schema: Any,
|
|
20
|
+
connection: Any,
|
|
21
|
+
table_stats: Optional[Dict[str, int]] = None,
|
|
22
|
+
):
|
|
23
|
+
super().__init__(schema, connection, dialect='mysql', table_stats=table_stats)
|
|
24
|
+
|
|
25
|
+
def _fetch(self, sql: str):
|
|
26
|
+
"""Execute SQL against MySQL. mysql-connector has no cancel for a
|
|
27
|
+
running statement: a cancelled query stops before its next statement
|
|
28
|
+
(`StatementGuard` with no stop)."""
|
|
29
|
+
from prograph.execution import StatementGuard
|
|
30
|
+
with StatementGuard():
|
|
31
|
+
cursor = self._connection.cursor()
|
|
32
|
+
try:
|
|
33
|
+
cursor.execute(sql)
|
|
34
|
+
if cursor.description is None:
|
|
35
|
+
return [], []
|
|
36
|
+
return [desc[0] for desc in cursor.description], cursor.fetchall()
|
|
37
|
+
finally:
|
|
38
|
+
cursor.close()
|