prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,556 @@
|
|
|
1
|
+
"""SparkGraph - A virtual graph backed by Spark DataFrames.
|
|
2
|
+
|
|
3
|
+
This holds the Spark DataFrames and runs the generated SQL on them (or on
|
|
4
|
+
a SQL warehouse), read-only.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
from typing import Any, Dict, List, Optional, Set, Union, TYPE_CHECKING
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from prograph.execution import QueryContext
|
|
12
|
+
import pyarrow as pa
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
from pyspark.sql import DataFrame as SparkDataFrame
|
|
16
|
+
SPARK_AVAILABLE = True
|
|
17
|
+
except ImportError:
|
|
18
|
+
SPARK_AVAILABLE = False
|
|
19
|
+
SparkDataFrame = None
|
|
20
|
+
|
|
21
|
+
from prograph.schema.mapping import GraphSchema
|
|
22
|
+
from prograph.execution import QueryCancelledException
|
|
23
|
+
from prograph.logging import get_backend_logger
|
|
24
|
+
from prograph.read_only import GraphRefusesWrites
|
|
25
|
+
|
|
26
|
+
def _nullable_spark_type(dt):
|
|
27
|
+
"""`dt` with every field, element and map value nullable."""
|
|
28
|
+
from pyspark.sql.types import ArrayType, MapType, StructField, StructType
|
|
29
|
+
if isinstance(dt, StructType):
|
|
30
|
+
return StructType([StructField(f.name, _nullable_spark_type(f.dataType), True, f.metadata)
|
|
31
|
+
for f in dt.fields])
|
|
32
|
+
if isinstance(dt, ArrayType):
|
|
33
|
+
return ArrayType(_nullable_spark_type(dt.elementType), True)
|
|
34
|
+
if isinstance(dt, MapType):
|
|
35
|
+
return MapType(dt.keyType, _nullable_spark_type(dt.valueType), True)
|
|
36
|
+
return dt
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _nullable_arrow_type(t):
|
|
40
|
+
"""`t` with every field, element and map value nullable (a map's key
|
|
41
|
+
stays not null, as Arrow requires)."""
|
|
42
|
+
import pyarrow as pa
|
|
43
|
+
if pa.types.is_struct(t):
|
|
44
|
+
return pa.struct([t.field(i).with_type(_nullable_arrow_type(t.field(i).type))
|
|
45
|
+
.with_nullable(True) for i in range(t.num_fields)])
|
|
46
|
+
if pa.types.is_map(t):
|
|
47
|
+
return pa.map_(t.key_field, t.item_field.with_type(
|
|
48
|
+
_nullable_arrow_type(t.item_type)).with_nullable(True), t.keys_sorted)
|
|
49
|
+
if pa.types.is_large_list(t):
|
|
50
|
+
return pa.large_list(t.value_field.with_type(
|
|
51
|
+
_nullable_arrow_type(t.value_type)).with_nullable(True))
|
|
52
|
+
if pa.types.is_list(t):
|
|
53
|
+
return pa.list_(t.value_field.with_type(
|
|
54
|
+
_nullable_arrow_type(t.value_type)).with_nullable(True))
|
|
55
|
+
return t
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _spark_arrow(df: 'SparkDataFrame') -> 'pa.Table':
|
|
59
|
+
"""A Spark result as Spark's own Arrow (``DataFrame.toArrow``), every
|
|
60
|
+
field nullable at every level and nothing else changed. Spark's not-null flags describe its plan, not the data, and
|
|
61
|
+
vary from query to query, so one term would not keep one Arrow type;
|
|
62
|
+
and toArrow refuses a null struct whose fields are not
|
|
63
|
+
null, or an empty list (array<void>), when the flags are kept. A
|
|
64
|
+
column with nested not-null flags is cast to its nullable type first
|
|
65
|
+
(the same type, so no value changes), and a not-null void column (`UNWIND
|
|
66
|
+
[]`) is a null; the rest is the Arrow schema's. Columns are taken by
|
|
67
|
+
position, as a result may hold two of one name.
|
|
68
|
+
"""
|
|
69
|
+
import pyarrow as pa
|
|
70
|
+
from pyspark.sql import functions as F
|
|
71
|
+
from pyspark.sql.types import NullType
|
|
72
|
+
fields = df.schema.fields
|
|
73
|
+
void = [isinstance(f.dataType, NullType) and not f.nullable for f in fields]
|
|
74
|
+
nested = [_nullable_spark_type(f.dataType) != f.dataType for f in fields]
|
|
75
|
+
if any(void) or any(nested):
|
|
76
|
+
names = df.columns
|
|
77
|
+
at = [f"_c{i}" for i in range(len(fields))]
|
|
78
|
+
df = df.toDF(*at).select([
|
|
79
|
+
F.lit(None).alias(a) if v
|
|
80
|
+
else F.col(a).cast(_nullable_spark_type(f.dataType)).alias(a) if n
|
|
81
|
+
else F.col(a)
|
|
82
|
+
for a, f, v, n in zip(at, fields, void, nested)]).toDF(*names)
|
|
83
|
+
table = df.toArrow()
|
|
84
|
+
return table.cast(pa.schema(
|
|
85
|
+
[pa.field(f.name, _nullable_arrow_type(f.type), True, f.metadata) for f in table.schema],
|
|
86
|
+
metadata=table.schema.metadata))
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def register_temp_views(dataframes: Dict[str, Any], registered: Set[str]) -> None:
|
|
90
|
+
"""Register each frame of `dataframes` not in `registered` as a temp
|
|
91
|
+
view under its name. A qualified name (catalog.schema.table) names a
|
|
92
|
+
catalog table, which needs none: temp views take single-part names."""
|
|
93
|
+
for table_name, df in dataframes.items():
|
|
94
|
+
if table_name not in registered:
|
|
95
|
+
if '.' not in table_name:
|
|
96
|
+
df.createOrReplaceTempView(table_name)
|
|
97
|
+
registered.add(table_name)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# The part of a relationship variable's identity column name: `r.__identity__`.
|
|
102
|
+
REL_IDENTITY_PART = "__identity__"
|
|
103
|
+
|
|
104
|
+
# The row number a relationship mapping's rows get while their identity is made.
|
|
105
|
+
_RELATIONSHIP_ROW = "__relationship_row__"
|
|
106
|
+
|
|
107
|
+
# The column that holds each relationship row's identity (its mapping's position in
|
|
108
|
+
# the schema, its ends and its row number among the rows of the same ends).
|
|
109
|
+
RELATIONSHIP_IDENTITY_COLUMN = "__relationship_identity__"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# The key under which an expression's frame travels in a variables mapping: the frame
|
|
113
|
+
# the expression's columns are read from, so a column's Spark type can be read where
|
|
114
|
+
# an expression needs it (`SparkGraph._divide_column`).
|
|
115
|
+
TYPING_FRAME = "_typing_frame"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@contextmanager
|
|
119
|
+
def _job_group(spark: Any):
|
|
120
|
+
"""The jobs this thread starts inside belong to one new job group (a tag on Spark Connect), and
|
|
121
|
+
the stop that cancels them: `cancelJobGroup` (`interruptOnCancel`, so a running task's thread
|
|
122
|
+
is interrupted), `interruptTag` on Connect. The group's id is ProGraph's own, never the query's
|
|
123
|
+
id. None where the session has neither."""
|
|
124
|
+
import uuid
|
|
125
|
+
group = f"prograph-{uuid.uuid4().hex}"
|
|
126
|
+
try:
|
|
127
|
+
sc = spark.sparkContext
|
|
128
|
+
except Exception: # noqa: BLE001 - Spark Connect has no SparkContext
|
|
129
|
+
sc = None
|
|
130
|
+
if sc is not None:
|
|
131
|
+
sc.setJobGroup(group, "ProGraph query", interruptOnCancel=True)
|
|
132
|
+
try:
|
|
133
|
+
yield lambda: sc.cancelJobGroup(group)
|
|
134
|
+
finally:
|
|
135
|
+
sc.setLocalProperty("spark.jobGroup.id", None)
|
|
136
|
+
sc.setLocalProperty("spark.job.description", None)
|
|
137
|
+
sc.setLocalProperty("spark.job.interruptOnCancel", None)
|
|
138
|
+
return
|
|
139
|
+
if hasattr(spark, "addTag") and hasattr(spark, "interruptTag"):
|
|
140
|
+
spark.addTag(group)
|
|
141
|
+
try:
|
|
142
|
+
yield lambda: spark.interruptTag(group)
|
|
143
|
+
finally:
|
|
144
|
+
spark.removeTag(group)
|
|
145
|
+
return
|
|
146
|
+
yield None
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class SparkGraph(GraphRefusesWrites):
|
|
150
|
+
"""A virtual graph backed by Spark DataFrames.
|
|
151
|
+
|
|
152
|
+
Runs the generated SQL over the Spark DataFrames. Read-only: its mutators refuse.
|
|
153
|
+
|
|
154
|
+
Example:
|
|
155
|
+
>>> schema = GraphSchema()
|
|
156
|
+
>>> schema.add_node_mapping(NodeMapping(
|
|
157
|
+
... source_table='movies',
|
|
158
|
+
... label='Movie',
|
|
159
|
+
... id_column='movie_id',
|
|
160
|
+
... property_columns={'title': 'title'}
|
|
161
|
+
... ))
|
|
162
|
+
>>> graph = SparkGraph(schema, {'movies': movies_df})
|
|
163
|
+
>>> graph._execute_sql_arrow('SELECT * FROM movies')
|
|
164
|
+
"""
|
|
165
|
+
|
|
166
|
+
def __init__(
|
|
167
|
+
self,
|
|
168
|
+
schema: GraphSchema,
|
|
169
|
+
dataframes: Dict[str, 'SparkDataFrame'],
|
|
170
|
+
sql_warehouse_id: Optional[str] = None,
|
|
171
|
+
warehouse_timeout: Optional[float] = None,
|
|
172
|
+
ansi_session: bool = True,
|
|
173
|
+
):
|
|
174
|
+
"""Create a SparkGraph.
|
|
175
|
+
|
|
176
|
+
Args:
|
|
177
|
+
schema: GraphSchema defining the mapping
|
|
178
|
+
dataframes: Dictionary mapping table names to Spark DataFrames
|
|
179
|
+
sql_warehouse_id: Optional Databricks SQL Warehouse ID for optimized
|
|
180
|
+
SQL execution. If provided, generated SQL queries will be routed
|
|
181
|
+
to the SQL Warehouse instead of Spark compute. Falls back to
|
|
182
|
+
Spark if the warehouse is unavailable.
|
|
183
|
+
warehouse_timeout: Optional timeout in seconds for SQL Warehouse
|
|
184
|
+
queries. When set, queries that exceed this time are cancelled
|
|
185
|
+
on the warehouse and raise TimeoutError. None (default) means
|
|
186
|
+
poll indefinitely until completion.
|
|
187
|
+
ansi_session: Run each SQL statement with ANSI mode on, in a
|
|
188
|
+
session of its own setting (prograph/graph/ansi_session.py).
|
|
189
|
+
"""
|
|
190
|
+
if not SPARK_AVAILABLE:
|
|
191
|
+
raise ImportError("pyspark is required for the Spark backend")
|
|
192
|
+
self._ansi_session = ansi_session
|
|
193
|
+
|
|
194
|
+
self.schema = schema
|
|
195
|
+
self.dataframes = dataframes
|
|
196
|
+
# Per written table, (the frame first given for it, the frame put
|
|
197
|
+
# in its place after the last write): _refresh_frame.
|
|
198
|
+
self._written_frames: Dict[str, tuple] = {}
|
|
199
|
+
# SQL Warehouse configuration
|
|
200
|
+
self._sql_warehouse_id = sql_warehouse_id
|
|
201
|
+
self._sql_warehouse_connection = None
|
|
202
|
+
self._warehouse_timeout = warehouse_timeout
|
|
203
|
+
|
|
204
|
+
def _execute_mutation_sql(self, sql: str, mapping) -> None:
|
|
205
|
+
"""Execute a mutation SQL statement on `mapping`'s table, then
|
|
206
|
+
refresh the engine's frame of that table. The one write a
|
|
207
|
+
SparkGraph makes: `set_properties`' UPDATE on a mutable mapping
|
|
208
|
+
(prograph.writes); its mutators refuse.
|
|
209
|
+
|
|
210
|
+
Args:
|
|
211
|
+
sql: The SQL statement to execute
|
|
212
|
+
mapping: The node or relationship mapping whose table it writes
|
|
213
|
+
|
|
214
|
+
TODO: Route through _execute_sql() for SQL Warehouse support.
|
|
215
|
+
Currently calls spark.sql() directly, bypassing warehouse
|
|
216
|
+
routing. SQL Warehouse supports DML (INSERT/UPDATE/DELETE/MERGE).
|
|
217
|
+
"""
|
|
218
|
+
# On the session itself, not a clone of it (ansi_session.py): the
|
|
219
|
+
# statement writes literals, which cannot overflow, and a write
|
|
220
|
+
# made in a clone leaves the session's own cached relation of a
|
|
221
|
+
# file table as it was.
|
|
222
|
+
spark = next(iter(self.dataframes.values())).sparkSession
|
|
223
|
+
spark.sql(sql)
|
|
224
|
+
self._refresh_frame(mapping.source_table)
|
|
225
|
+
|
|
226
|
+
def _refresh_frame(self, table: str) -> None:
|
|
227
|
+
"""Make the engine's frame of `table` see the files a write added.
|
|
228
|
+
|
|
229
|
+
A frame of a file table (parquet) holds the list of the table's
|
|
230
|
+
files that Spark made when the frame was read. Spark's INSERT
|
|
231
|
+
refreshes the list of the relation it resolved, which is the
|
|
232
|
+
frame's own only until the catalog's cached relation is replaced:
|
|
233
|
+
after a table's first write the frame kept its list, and every
|
|
234
|
+
later read through it (the SQL path reads it as a temp view under
|
|
235
|
+
the table's name, which a later `spark.table` of that name returns
|
|
236
|
+
too) lacked the rows of the second write on. Refreshing the
|
|
237
|
+
frame's own plan re-lists its files in place, so the frame and the
|
|
238
|
+
temp views over it see the table as written. A frame with no JVM
|
|
239
|
+
plan (Spark Connect) reads the table anew at each action.
|
|
240
|
+
|
|
241
|
+
A frame that has run keeps the physical plan it ran with, and its
|
|
242
|
+
scan the files listed then: SPARQL in dataframe mode collects the
|
|
243
|
+
frame itself, so after its first read no write was seen. The frame's place therefore takes a new Dataset over the
|
|
244
|
+
refreshed plan, planned at its first action. It is made over the
|
|
245
|
+
frame first given, not over the last one put, so writes do not
|
|
246
|
+
stack a projection each; a frame put there by the caller since is
|
|
247
|
+
taken as the one given."""
|
|
248
|
+
given = self._given_frame(table)
|
|
249
|
+
if getattr(given, '_jdf', None) is None:
|
|
250
|
+
return
|
|
251
|
+
given._jdf.queryExecution().analyzed().refresh()
|
|
252
|
+
fresh = given.select("*")
|
|
253
|
+
self._written_frames[table] = (given, fresh)
|
|
254
|
+
self.dataframes[table] = fresh
|
|
255
|
+
|
|
256
|
+
def _given_frame(self, table: str) -> Optional['SparkDataFrame']:
|
|
257
|
+
"""The frame given for `table`, under the one `_refresh_frame` put
|
|
258
|
+
in its place: a temp view is made over it, since a view over the
|
|
259
|
+
put frame's projection takes no INSERT (UNSUPPORTED_INSERT)."""
|
|
260
|
+
frame = self.dataframes.get(table)
|
|
261
|
+
given, put = self._written_frames.get(table, (None, None))
|
|
262
|
+
return given if put is not None and frame is put else frame
|
|
263
|
+
|
|
264
|
+
def _invalidate_caches(self, changes: 'SchemaChangeSet') -> None:
|
|
265
|
+
"""Drop the frames of tables a schema change removed."""
|
|
266
|
+
if not changes.has_changes:
|
|
267
|
+
return
|
|
268
|
+
for table in changes.removed_tables:
|
|
269
|
+
self.dataframes.pop(table, None)
|
|
270
|
+
|
|
271
|
+
def close(self) -> None:
|
|
272
|
+
"""Close any open connections (e.g., SQL Warehouse connection)."""
|
|
273
|
+
if self._sql_warehouse_connection is not None:
|
|
274
|
+
try:
|
|
275
|
+
self._sql_warehouse_connection.close()
|
|
276
|
+
except Exception as e:
|
|
277
|
+
get_backend_logger().debug("warehouse_close_failed", error=str(e))
|
|
278
|
+
self._sql_warehouse_connection = None
|
|
279
|
+
|
|
280
|
+
def _reset_warehouse_connection(self):
|
|
281
|
+
"""Reset cached warehouse connection (e.g., after auth failure)."""
|
|
282
|
+
if self._sql_warehouse_connection is not None:
|
|
283
|
+
try:
|
|
284
|
+
self._sql_warehouse_connection.close()
|
|
285
|
+
except Exception as e:
|
|
286
|
+
get_backend_logger().debug("warehouse_close_failed", error=str(e))
|
|
287
|
+
self._sql_warehouse_connection = None
|
|
288
|
+
|
|
289
|
+
def __del__(self):
|
|
290
|
+
"""Cleanup on garbage collection."""
|
|
291
|
+
self.close()
|
|
292
|
+
|
|
293
|
+
def _execute_sql(
|
|
294
|
+
self,
|
|
295
|
+
sql: str,
|
|
296
|
+
spark: 'SparkSession',
|
|
297
|
+
use_warehouse: Optional[bool] = None,
|
|
298
|
+
query_context: Optional['QueryContext'] = None,
|
|
299
|
+
adaptive_off: bool = False,
|
|
300
|
+
) -> Union[List[Dict[str, Any]], 'pa.Table']:
|
|
301
|
+
"""Execute SQL query, routing to SQL Warehouse if configured.
|
|
302
|
+
|
|
303
|
+
On Spark the result is Spark's own Arrow (``DataFrame.toArrow``),
|
|
304
|
+
every field nullable (_spark_arrow), as every other SQL backend
|
|
305
|
+
hands back its database's: its types are Spark's, as
|
|
306
|
+
Databricks gives them. Collected
|
|
307
|
+
as rows, a TIMESTAMP was a naive datetime in the Python process's
|
|
308
|
+
zone, an INT an int64 and a MAP a struct. A
|
|
309
|
+
statement routed to a SQL warehouse returns rows.
|
|
310
|
+
|
|
311
|
+
``adaptive_off``: run the statement on Spark with adaptive execution
|
|
312
|
+
off, for a statement that holds a slice below its result (the planner reads it from the statement's tree). A
|
|
313
|
+
statement routed to a SQL warehouse is not affected.
|
|
314
|
+
|
|
315
|
+
If sql_warehouse_id is set, attempts to execute via SQL Warehouse
|
|
316
|
+
for better performance (Photon engine, no cluster contention).
|
|
317
|
+
Falls back to spark.sql() if warehouse is unavailable.
|
|
318
|
+
|
|
319
|
+
Args:
|
|
320
|
+
sql: The SQL query to execute
|
|
321
|
+
spark: SparkSession to use as fallback
|
|
322
|
+
use_warehouse: Override warehouse routing. None=auto (use if configured),
|
|
323
|
+
True=force warehouse, False=skip warehouse (use Spark).
|
|
324
|
+
query_context: Optional QueryContext for cancellation support.
|
|
325
|
+
|
|
326
|
+
Returns:
|
|
327
|
+
List of result dictionaries
|
|
328
|
+
"""
|
|
329
|
+
from prograph.logging import get_backend_logger
|
|
330
|
+
import time
|
|
331
|
+
logger = get_backend_logger()
|
|
332
|
+
|
|
333
|
+
# The query running here (`execution.running`), if none is given.
|
|
334
|
+
if query_context is None:
|
|
335
|
+
from prograph.execution import _CURRENT
|
|
336
|
+
query_context = _CURRENT.get()
|
|
337
|
+
|
|
338
|
+
# Check for cancellation before starting
|
|
339
|
+
if query_context:
|
|
340
|
+
query_context.check_cancelled()
|
|
341
|
+
|
|
342
|
+
# Determine whether to use warehouse
|
|
343
|
+
should_use_warehouse = (
|
|
344
|
+
use_warehouse is True or
|
|
345
|
+
(use_warehouse is None and self._sql_warehouse_id is not None)
|
|
346
|
+
) and use_warehouse is not False
|
|
347
|
+
|
|
348
|
+
# If SQL Warehouse should be used, try it
|
|
349
|
+
if should_use_warehouse and self._sql_warehouse_id:
|
|
350
|
+
try:
|
|
351
|
+
start_time = time.time()
|
|
352
|
+
results = self._execute_via_warehouse(sql, query_context=query_context)
|
|
353
|
+
elapsed = time.time() - start_time
|
|
354
|
+
logger.debug(
|
|
355
|
+
"sql_warehouse_executed",
|
|
356
|
+
warehouse_id=self._sql_warehouse_id,
|
|
357
|
+
result_count=len(results),
|
|
358
|
+
elapsed_ms=round(elapsed * 1000, 2)
|
|
359
|
+
)
|
|
360
|
+
return results
|
|
361
|
+
except TimeoutError:
|
|
362
|
+
# Warehouse is the fast path; if it times out, Spark will
|
|
363
|
+
# be slower. Propagate as a hard error, no fallback.
|
|
364
|
+
raise
|
|
365
|
+
except QueryCancelledException:
|
|
366
|
+
# Cancelled queries should not fall back to Spark,
|
|
367
|
+
# Python/Spark execution will be even slower.
|
|
368
|
+
raise
|
|
369
|
+
except Exception as e:
|
|
370
|
+
# Reset connection on auth errors so next attempt creates fresh client
|
|
371
|
+
error_str = str(e).lower()
|
|
372
|
+
if any(term in error_str for term in ('unauthorized', '401', 'token', 'authentication', 'forbidden', '403')):
|
|
373
|
+
self._reset_warehouse_connection()
|
|
374
|
+
# Log and fall back to Spark; use warning level so users
|
|
375
|
+
# can see why warehouse routing failed
|
|
376
|
+
logger.warning(
|
|
377
|
+
"sql_warehouse_fallback",
|
|
378
|
+
warehouse_id=self._sql_warehouse_id,
|
|
379
|
+
error=str(e),
|
|
380
|
+
error_type=type(e).__name__,
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
# Log if warehouse was skipped due to smart routing
|
|
384
|
+
if use_warehouse is False and self._sql_warehouse_id:
|
|
385
|
+
logger.debug(
|
|
386
|
+
"sql_warehouse_skipped",
|
|
387
|
+
reason="smart_routing",
|
|
388
|
+
warehouse_id=self._sql_warehouse_id
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
# Default: execute via Spark, with ANSI mode on, and fetched
|
|
392
|
+
# while it is. The statement's jobs are one job
|
|
393
|
+
# group, which a cancel or the deadline cancels (`_job_group`).
|
|
394
|
+
from prograph.execution import StatementGuard
|
|
395
|
+
from prograph.graph.ansi_session import statement_session
|
|
396
|
+
with _job_group(spark) as stop, StatementGuard(stop=stop):
|
|
397
|
+
with statement_session(spark, self._ansi_session, adaptive_off=adaptive_off) as session:
|
|
398
|
+
return _spark_arrow(session.sql(sql))
|
|
399
|
+
|
|
400
|
+
def _execute_via_warehouse(
|
|
401
|
+
self,
|
|
402
|
+
sql: str,
|
|
403
|
+
query_context: Optional['QueryContext'] = None
|
|
404
|
+
) -> List[Dict[str, Any]]:
|
|
405
|
+
"""Execute SQL via Databricks SQL Warehouse using SDK REST API.
|
|
406
|
+
|
|
407
|
+
Uses the Databricks SDK for optimized SQL execution with server-side
|
|
408
|
+
result caching. ~2-3x faster than Thrift connector on cold queries,
|
|
409
|
+
~12x faster on cached queries.
|
|
410
|
+
|
|
411
|
+
Args:
|
|
412
|
+
sql: The SQL query to execute
|
|
413
|
+
query_context: Optional QueryContext for cancellation support.
|
|
414
|
+
If provided, the statement_id will be tracked for
|
|
415
|
+
cancellation via statement_execution.cancel_execution().
|
|
416
|
+
|
|
417
|
+
Returns:
|
|
418
|
+
List of result dictionaries
|
|
419
|
+
"""
|
|
420
|
+
# Lazy import to avoid dependency issues
|
|
421
|
+
try:
|
|
422
|
+
from databricks.sdk import WorkspaceClient
|
|
423
|
+
except ImportError:
|
|
424
|
+
raise ImportError(
|
|
425
|
+
"databricks-sdk is required for SQL Warehouse support. "
|
|
426
|
+
"Install with: pip install databricks-sdk"
|
|
427
|
+
)
|
|
428
|
+
|
|
429
|
+
# Create SDK client (reuse if possible)
|
|
430
|
+
# WorkspaceClient() uses the SDK's unified credential chain:
|
|
431
|
+
# 1. Environment variables (DATABRICKS_HOST + DATABRICKS_TOKEN)
|
|
432
|
+
# 2. Databricks CLI profile (~/.databrickscfg)
|
|
433
|
+
# 3. Azure/GCP/AWS managed identity
|
|
434
|
+
# 4. Notebook context (auto-detected)
|
|
435
|
+
# The SDK handles token refresh automatically for OAuth/CLI-based auth.
|
|
436
|
+
if self._sql_warehouse_connection is None:
|
|
437
|
+
try:
|
|
438
|
+
self._sql_warehouse_connection = WorkspaceClient()
|
|
439
|
+
except Exception as e:
|
|
440
|
+
raise ValueError(
|
|
441
|
+
"Could not determine Databricks connection details. "
|
|
442
|
+
"Configure credentials via databricks-cli (`databricks auth login`), "
|
|
443
|
+
"set DATABRICKS_HOST and DATABRICKS_TOKEN environment variables, "
|
|
444
|
+
"or run in a Databricks notebook."
|
|
445
|
+
) from e
|
|
446
|
+
|
|
447
|
+
# Check for cancellation before executing
|
|
448
|
+
if query_context:
|
|
449
|
+
query_context.check_cancelled()
|
|
450
|
+
|
|
451
|
+
# Execute query via REST API
|
|
452
|
+
import time
|
|
453
|
+
response = self._sql_warehouse_connection.statement_execution.execute_statement(
|
|
454
|
+
warehouse_id=self._sql_warehouse_id,
|
|
455
|
+
statement=sql,
|
|
456
|
+
wait_timeout="50s" # Max allowed by API
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
# Track statement_id in context for cancellation support
|
|
460
|
+
if query_context and response.statement_id:
|
|
461
|
+
query_context.set_statement_id(
|
|
462
|
+
response.statement_id,
|
|
463
|
+
self._sql_warehouse_connection
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
# Poll if statement is still running after initial wait
|
|
467
|
+
try:
|
|
468
|
+
from databricks.sdk.service.sql import StatementState
|
|
469
|
+
except ImportError:
|
|
470
|
+
StatementState = None
|
|
471
|
+
|
|
472
|
+
if StatementState is not None and response.status and response.status.state in (
|
|
473
|
+
StatementState.PENDING, StatementState.RUNNING
|
|
474
|
+
):
|
|
475
|
+
from prograph.logging import get_backend_logger
|
|
476
|
+
logger = get_backend_logger()
|
|
477
|
+
logger.debug(
|
|
478
|
+
"warehouse_poll_start",
|
|
479
|
+
statement_id=response.statement_id,
|
|
480
|
+
warehouse_timeout=self._warehouse_timeout,
|
|
481
|
+
)
|
|
482
|
+
poll_start = time.time()
|
|
483
|
+
while True:
|
|
484
|
+
# Check for cancellation
|
|
485
|
+
if query_context:
|
|
486
|
+
query_context.check_cancelled()
|
|
487
|
+
|
|
488
|
+
# Check warehouse_timeout
|
|
489
|
+
if self._warehouse_timeout is not None:
|
|
490
|
+
elapsed = time.time() - poll_start + 50 # include initial wait
|
|
491
|
+
if elapsed >= self._warehouse_timeout:
|
|
492
|
+
# Cancel the running statement on the warehouse
|
|
493
|
+
try:
|
|
494
|
+
self._sql_warehouse_connection.statement_execution.cancel_execution(
|
|
495
|
+
response.statement_id
|
|
496
|
+
)
|
|
497
|
+
except Exception as e:
|
|
498
|
+
logger.debug("timeout_cancel_failed", statement_id=response.statement_id, error=str(e))
|
|
499
|
+
raise TimeoutError(
|
|
500
|
+
f"Warehouse query exceeded timeout of {self._warehouse_timeout}s "
|
|
501
|
+
f"(statement_id={response.statement_id})"
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
time.sleep(2)
|
|
505
|
+
response = self._sql_warehouse_connection.statement_execution.get_statement(
|
|
506
|
+
response.statement_id
|
|
507
|
+
)
|
|
508
|
+
if response.status and response.status.state not in (
|
|
509
|
+
StatementState.PENDING, StatementState.RUNNING
|
|
510
|
+
):
|
|
511
|
+
break
|
|
512
|
+
|
|
513
|
+
logger.debug(
|
|
514
|
+
"warehouse_poll_complete",
|
|
515
|
+
statement_id=response.statement_id,
|
|
516
|
+
poll_elapsed_ms=round((time.time() - poll_start) * 1000, 2),
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
# Check for failure after polling
|
|
520
|
+
if StatementState is not None and response.status:
|
|
521
|
+
if response.status.state == StatementState.FAILED:
|
|
522
|
+
error_msg = (
|
|
523
|
+
response.status.error.message
|
|
524
|
+
if response.status.error else "Unknown error"
|
|
525
|
+
)
|
|
526
|
+
raise RuntimeError(f"Warehouse statement failed: {error_msg}")
|
|
527
|
+
if response.status.state == StatementState.CANCELED:
|
|
528
|
+
raise QueryCancelledException("Warehouse statement was cancelled")
|
|
529
|
+
|
|
530
|
+
# Extract results from response
|
|
531
|
+
if response.result is None or response.result.data_array is None:
|
|
532
|
+
return []
|
|
533
|
+
|
|
534
|
+
# Get column names from schema
|
|
535
|
+
columns = [col.name for col in response.manifest.schema.columns]
|
|
536
|
+
|
|
537
|
+
# Convert data_array to list of dicts
|
|
538
|
+
return [dict(zip(columns, row)) for row in response.result.data_array]
|
|
539
|
+
|
|
540
|
+
def sql_dialect(self) -> Optional[str]:
|
|
541
|
+
"""The dialect this graph runs SQL in: Spark SQL, through spark.sql."""
|
|
542
|
+
return "spark"
|
|
543
|
+
|
|
544
|
+
def _ensure_temp_views(self) -> None:
|
|
545
|
+
"""Ensure all DataFrames are registered as temporary views.
|
|
546
|
+
|
|
547
|
+
Only creates temp views for simple (non-qualified) table names.
|
|
548
|
+
Fully qualified names (catalog.schema.table) are assumed to reference
|
|
549
|
+
existing catalog tables that don't need temp views.
|
|
550
|
+
"""
|
|
551
|
+
if not hasattr(self, '_views_registered'):
|
|
552
|
+
self._views_registered = set()
|
|
553
|
+
register_temp_views({t: self._given_frame(t) for t in self.dataframes}, self._views_registered)
|
|
554
|
+
|
|
555
|
+
def __repr__(self) -> str:
|
|
556
|
+
return f"SparkGraph(tables={sorted(self.dataframes)})"
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Base class for SQL-database-backed graphs.
|
|
2
|
+
|
|
3
|
+
Provides a simple interface for executing SQL queries against relational
|
|
4
|
+
databases (PostgreSQL, SQL Server, Oracle). Much simpler than SparkGraph:
|
|
5
|
+
no DataFrames, no VL paths.
|
|
6
|
+
|
|
7
|
+
Architecture:
|
|
8
|
+
|
|
9
|
+
ProGraph(backend='postgresql', connection=conn, schema=schema)
|
|
10
|
+
→ PostgreSQLExecutor
|
|
11
|
+
→ PostgreSQLGraph(SQLGraph)
|
|
12
|
+
→ build_sql(dialect='postgresql') → SQL string
|
|
13
|
+
→ _execute_sql(sql) → List[Dict]
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import logging
|
|
19
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
20
|
+
from prograph.read_only import GraphRefusesWrites
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class SQLGraph(GraphRefusesWrites):
|
|
26
|
+
"""Base class for SQL-database-backed graphs.
|
|
27
|
+
|
|
28
|
+
Subclasses implement ``_execute_sql()`` for their specific database
|
|
29
|
+
driver.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
schema: Any,
|
|
35
|
+
connection: Any,
|
|
36
|
+
dialect: str,
|
|
37
|
+
table_stats: Optional[Dict[str, int]] = None,
|
|
38
|
+
):
|
|
39
|
+
"""Initialize the SQL graph.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
schema: GraphSchema with node/relationship mappings
|
|
43
|
+
connection: Database connection (driver-specific)
|
|
44
|
+
dialect: SQL dialect name ('postgresql', etc.)
|
|
45
|
+
table_stats: Optional table row counts for optimization
|
|
46
|
+
"""
|
|
47
|
+
self.schema = schema
|
|
48
|
+
self._connection = connection
|
|
49
|
+
self._dialect = dialect
|
|
50
|
+
self._table_stats = table_stats or {}
|
|
51
|
+
|
|
52
|
+
def _fetch(self, sql: str) -> Tuple[List[str], List[tuple]]:
|
|
53
|
+
"""Execute SQL and return its column names and its rows. A
|
|
54
|
+
DB-API driver implements this; one returning Arrow overrides
|
|
55
|
+
_execute_sql and _execute_sql_arrow instead (Databricks, DuckDB)."""
|
|
56
|
+
raise NotImplementedError
|
|
57
|
+
|
|
58
|
+
def _execute_sql(self, sql: str) -> List[Dict[str, Any]]:
|
|
59
|
+
"""Execute a SQL query and return its rows as dicts."""
|
|
60
|
+
columns, rows = self._fetch(sql)
|
|
61
|
+
return [dict(zip(columns, row)) for row in rows]
|
|
62
|
+
|
|
63
|
+
def _execute_sql_arrow(self, sql: str):
|
|
64
|
+
"""Execute SQL and return a pyarrow.Table, built once from the
|
|
65
|
+
driver's rows. The column names come from the cursor, so a result
|
|
66
|
+
with no rows still has its columns."""
|
|
67
|
+
from prograph.results import rows_to_arrow
|
|
68
|
+
columns, rows = self._fetch(sql)
|
|
69
|
+
return rows_to_arrow([dict(zip(columns, row)) for row in rows], columns)
|
|
70
|
+
|
|
71
|
+
def sql_dialect(self) -> str:
|
|
72
|
+
"""The dialect this graph executes SQL in."""
|
|
73
|
+
return self._dialect
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def dataframes(self) -> Dict[str, Any]:
|
|
77
|
+
"""Compatibility shim for _try_compositional_backend.
|
|
78
|
+
|
|
79
|
+
SQL-database graphs don't use DataFrames. This property returns
|
|
80
|
+
a non-empty dict so the compositional backend path activates,
|
|
81
|
+
but none of the values are real DataFrames.
|
|
82
|
+
"""
|
|
83
|
+
return {'_sql_graph': True}
|