prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""Render and execute the CREATE VIEW statements behind generated views.
|
|
2
|
+
|
|
3
|
+
The :mod:`prograph.genie.view_generator` module produces
|
|
4
|
+
:class:`ViewDefinition` instances whose ``sql`` field is the SELECT body
|
|
5
|
+
of a view. This module wraps each body in a dialect-correct
|
|
6
|
+
``CREATE OR REPLACE VIEW <fully_qualified_name> AS <body>`` statement
|
|
7
|
+
and (optionally) executes it against a connection.
|
|
8
|
+
|
|
9
|
+
Two surfaces:
|
|
10
|
+
|
|
11
|
+
- :func:`render_view_sql`, pure-function, returns the executable SQL
|
|
12
|
+
string for a view. Used by the CLI's ``generate-views`` subcommand
|
|
13
|
+
which writes the SQL to a ``.sql`` file for separate execution.
|
|
14
|
+
|
|
15
|
+
- :func:`write_views`, takes an open PEP 249 connection and runs the
|
|
16
|
+
CREATE OR REPLACE statements directly. Used by
|
|
17
|
+
:meth:`ProGraph.write_views` on SQL backends.
|
|
18
|
+
|
|
19
|
+
The split lets the same generator output serve "emit SQL for someone
|
|
20
|
+
else to run" and "create views on the engine's own connection" without
|
|
21
|
+
duplicating the dialect-aware naming and quoting logic.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple
|
|
27
|
+
|
|
28
|
+
if TYPE_CHECKING:
|
|
29
|
+
from prograph.genie.formats import ViewDefinition
|
|
30
|
+
from prograph.plan.sql.dialect import DialectConfig
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def render_view_sql(
|
|
34
|
+
view: "ViewDefinition",
|
|
35
|
+
target_catalog: Optional[str] = None,
|
|
36
|
+
target_schema: str = "prograph_views",
|
|
37
|
+
dialect: Optional["DialectConfig"] = None,
|
|
38
|
+
) -> str:
|
|
39
|
+
"""Render a ``CREATE OR REPLACE VIEW`` statement for one ViewDefinition.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
view: The :class:`ViewDefinition` to render.
|
|
43
|
+
target_catalog: Optional catalog prefix. Required for backends
|
|
44
|
+
that use three-part names (Databricks SQL); omitted for
|
|
45
|
+
two-part backends (Postgres, SQL Server).
|
|
46
|
+
target_schema: The schema/namespace for the view. Conventionally
|
|
47
|
+
``"prograph_views"``. The user can configure another via
|
|
48
|
+
the engine constructor or CLI argument.
|
|
49
|
+
dialect: SQL dialect for identifier quoting. Defaults to the
|
|
50
|
+
Spark / Databricks SQL backtick form.
|
|
51
|
+
|
|
52
|
+
Returns:
|
|
53
|
+
A complete ``CREATE OR REPLACE VIEW <fq> AS\\n<body>`` string,
|
|
54
|
+
no trailing semicolon (callers that need a semicolon add it).
|
|
55
|
+
"""
|
|
56
|
+
if dialect is None:
|
|
57
|
+
from prograph.plan.sql.dialect import SPARK_DIALECT
|
|
58
|
+
dialect = SPARK_DIALECT
|
|
59
|
+
|
|
60
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
61
|
+
q = get_dialect_instance(dialect.name).quote
|
|
62
|
+
parts: List[str] = []
|
|
63
|
+
if target_catalog:
|
|
64
|
+
parts.append(q(target_catalog))
|
|
65
|
+
parts.append(q(target_schema))
|
|
66
|
+
parts.append(q(view.name))
|
|
67
|
+
fq = ".".join(parts)
|
|
68
|
+
|
|
69
|
+
return f"{get_dialect_instance(dialect.name).create_or_replace_view()} {fq} AS\n{view.sql}"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def render_views_script(
|
|
73
|
+
views: Iterable["ViewDefinition"],
|
|
74
|
+
target_catalog: Optional[str] = None,
|
|
75
|
+
target_schema: str = "prograph_views",
|
|
76
|
+
dialect: Optional["DialectConfig"] = None,
|
|
77
|
+
include_header: bool = True,
|
|
78
|
+
) -> str:
|
|
79
|
+
"""Render a complete ``.sql`` script for a batch of views.
|
|
80
|
+
|
|
81
|
+
Each view becomes a ``CREATE OR REPLACE VIEW … ;`` statement
|
|
82
|
+
separated by a blank line. An optional header comment describes
|
|
83
|
+
the script's provenance.
|
|
84
|
+
|
|
85
|
+
Suitable for ``prograph generate-views --output views.sql``, the
|
|
86
|
+
resulting file runs unchanged against the target warehouse via
|
|
87
|
+
``databricks-sql-cli``, ``psql -f``, etc.
|
|
88
|
+
"""
|
|
89
|
+
statements: List[str] = []
|
|
90
|
+
if include_header:
|
|
91
|
+
statements.append(
|
|
92
|
+
"-- Generated by ProGraph (prograph.genie.view_generator)\n"
|
|
93
|
+
"-- See docs/guides/genie.md for how they are used.\n"
|
|
94
|
+
"-- Each view is a query-time projection, no storage cost."
|
|
95
|
+
)
|
|
96
|
+
for view in views:
|
|
97
|
+
sql = render_view_sql(
|
|
98
|
+
view,
|
|
99
|
+
target_catalog=target_catalog,
|
|
100
|
+
target_schema=target_schema,
|
|
101
|
+
dialect=dialect,
|
|
102
|
+
)
|
|
103
|
+
comment = f"-- {view.source.value}: {view.description}"
|
|
104
|
+
statements.append(f"{comment}\n{sql};")
|
|
105
|
+
return "\n\n".join(statements) + "\n"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def write_views(
|
|
109
|
+
views: Iterable["ViewDefinition"],
|
|
110
|
+
connection: Any,
|
|
111
|
+
target_catalog: Optional[str] = None,
|
|
112
|
+
target_schema: str = "prograph_views",
|
|
113
|
+
dialect: Optional["DialectConfig"] = None,
|
|
114
|
+
ensure_schema: bool = True,
|
|
115
|
+
) -> List[Tuple[str, str]]:
|
|
116
|
+
"""Execute ``CREATE OR REPLACE VIEW`` for each view against ``connection``.
|
|
117
|
+
|
|
118
|
+
Args:
|
|
119
|
+
views: Iterable of :class:`ViewDefinition` to write.
|
|
120
|
+
connection: PEP 249 connection (databricks-sql-connector, psycopg,
|
|
121
|
+
pyodbc, …). The function does not own the connection's
|
|
122
|
+
lifecycle.
|
|
123
|
+
target_catalog: Optional catalog prefix.
|
|
124
|
+
target_schema: Target schema/namespace for the views.
|
|
125
|
+
dialect: SQL dialect for identifier quoting.
|
|
126
|
+
ensure_schema: When ``True`` (default), issues
|
|
127
|
+
``CREATE SCHEMA IF NOT EXISTS`` before the views. Set
|
|
128
|
+
``False`` if the schema is managed externally.
|
|
129
|
+
|
|
130
|
+
Returns:
|
|
131
|
+
List of ``(view_name, fully_qualified_name)`` tuples for the
|
|
132
|
+
views actually created. Callers can use this list to confirm
|
|
133
|
+
what landed, build observability events, etc.
|
|
134
|
+
"""
|
|
135
|
+
if dialect is None:
|
|
136
|
+
from prograph.plan.sql.dialect import SPARK_DIALECT
|
|
137
|
+
dialect = SPARK_DIALECT
|
|
138
|
+
|
|
139
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
140
|
+
q = get_dialect_instance(dialect.name).quote
|
|
141
|
+
created: List[Tuple[str, str]] = []
|
|
142
|
+
|
|
143
|
+
with connection.cursor() as cur:
|
|
144
|
+
if ensure_schema:
|
|
145
|
+
schema_parts: List[str] = []
|
|
146
|
+
if target_catalog:
|
|
147
|
+
schema_parts.append(q(target_catalog))
|
|
148
|
+
schema_parts.append(q(target_schema))
|
|
149
|
+
cur.execute(
|
|
150
|
+
f"CREATE SCHEMA IF NOT EXISTS {'.'.join(schema_parts)}"
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
for view in views:
|
|
154
|
+
sql = render_view_sql(
|
|
155
|
+
view,
|
|
156
|
+
target_catalog=target_catalog,
|
|
157
|
+
target_schema=target_schema,
|
|
158
|
+
dialect=dialect,
|
|
159
|
+
)
|
|
160
|
+
cur.execute(sql)
|
|
161
|
+
# Reconstruct the fq name for the result tuple.
|
|
162
|
+
parts: List[str] = []
|
|
163
|
+
if target_catalog:
|
|
164
|
+
parts.append(target_catalog)
|
|
165
|
+
parts.append(target_schema)
|
|
166
|
+
parts.append(view.name)
|
|
167
|
+
created.append((view.name, ".".join(parts)))
|
|
168
|
+
|
|
169
|
+
return created
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
__all__ = ["render_view_sql", "render_views_script", "write_views"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The backends' graph objects: each runs the generated SQL on its database."""
|
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
"""ANSI mode on for the statements the engine runs.
|
|
2
|
+
|
|
3
|
+
With ANSI mode off, Spark and Databricks wrap an integer that overflows
|
|
4
|
+
(``9223372036854775807 + 1`` is ``-9223372036854775808``) and answer NULL
|
|
5
|
+
for what has no value. Cypher has no wrapped integer: these are errors,
|
|
6
|
+
and with ANSI mode on the database raises them. The setting belongs to the
|
|
7
|
+
session, and the session is the user's, so the engine turns it on only for
|
|
8
|
+
its own statements and leaves the user's setting as it found it:
|
|
9
|
+
|
|
10
|
+
- Spark, a classic session: each statement runs in a clone of the session
|
|
11
|
+
(``cloneSession``), which has its own configuration, sees the parent's
|
|
12
|
+
temporary views and cached frames as of the clone, and costs about a
|
|
13
|
+
millisecond. Setting and restoring the shared session instead leaks: a
|
|
14
|
+
second thread on it saw ANSI mode on in 50 of 50 queries.
|
|
15
|
+
- Spark Connect: a client session cannot be cloned, so the setting is
|
|
16
|
+
changed on the session for the length of the statement and put back in a
|
|
17
|
+
``finally``. Another thread on the same session sees ANSI mode on while
|
|
18
|
+
the statement runs.
|
|
19
|
+
- A Databricks SQL connection: ``SET ANSI_MODE = true`` once, when the
|
|
20
|
+
engine takes the connection, and the value it had is put back by
|
|
21
|
+
``close()``. A statement the user runs on that connection in between
|
|
22
|
+
sees ANSI mode on.
|
|
23
|
+
|
|
24
|
+
Adaptive execution off, on Spark, for a statement that holds a slice (a
|
|
25
|
+
LIMIT or an OFFSET) anywhere but at its result. Spark
|
|
26
|
+
4.1.3's adaptive execution sometimes loses such a slice: a LIMIT 3 OFFSET 1
|
|
27
|
+
over a LIMIT 6 OFFSET 2 over a LEFT JOIN answered 5 rows in about one run of
|
|
28
|
+
ten, with adaptive execution on only (apache/spark#59434). The engine reads
|
|
29
|
+
from the statement's tree whether it holds one
|
|
30
|
+
(``tree_utils.holds_a_slice_below_the_result``), and the statement then runs
|
|
31
|
+
with ``spark.sql.adaptive.enabled`` false in the same session as its ANSI
|
|
32
|
+
mode: a clone on a classic session, set and put back on Spark Connect, where
|
|
33
|
+
another statement on that session in between sees it off. Every other
|
|
34
|
+
statement runs with the user's setting. A Databricks SQL connection is not
|
|
35
|
+
touched: a warehouse answers these statements.
|
|
36
|
+
|
|
37
|
+
Parameters of a caller's own statement (``engine.execute_sql``). Spark 4.1 binds ``:name`` and ``?`` in its parser: the session's
|
|
38
|
+
parser is handed the values (``parsePlanWithParameters``) and puts them in
|
|
39
|
+
as literals. A session extension that wraps the parser and does not pass
|
|
40
|
+
the values on leaves every marker unbound, and the statement fails with
|
|
41
|
+
UNBOUND_SQL_PARAMETER: Delta's ``DeltaSqlParser`` in every release for
|
|
42
|
+
Spark 4.1 (4.1.0, 4.2.0, 4.3.1 and 4.4.1 tried; each inherits the default
|
|
43
|
+
of ``ParserInterface``, which drops them), and Iceberg's before
|
|
44
|
+
apache/iceberg#16626. On such a session
|
|
45
|
+
the statement is parsed with ``spark.sql.legacy.parameterSubstitution.
|
|
46
|
+
constantsOnly`` true, Spark's own setting for binding the values in the
|
|
47
|
+
analyzer as Spark 4.0 did, which no parser can drop: a value goes where a
|
|
48
|
+
constant goes (a comparison, LIMIT, IDENTIFIER), and a marker in a view's
|
|
49
|
+
body is refused by Spark. Whether a session's parser binds is asked of the
|
|
50
|
+
session once, with ``SELECT :p AS p``. On a classic session the analysis is
|
|
51
|
+
done when ``sql()`` returns, so the setting is put back then, and a
|
|
52
|
+
DataFrame returned keeps its values. The setting is read, set, used by
|
|
53
|
+
``sql()`` and put back under one lock of this module, and the session's
|
|
54
|
+
parser is asked under it too, so two threads' statements through the
|
|
55
|
+
engine never see each other's setting: without it one thread put back the
|
|
56
|
+
setting while another's statement was being parsed, and that statement was
|
|
57
|
+
unbound (8 of 120 in the review), or the setting was left on.
|
|
58
|
+
A statement the user runs on the session outside the engine while the
|
|
59
|
+
setting is on is bound in the analyzer too. On Spark Connect the server's parser cannot be asked and
|
|
60
|
+
its statement is analyzed later, so its session is left as it is.
|
|
61
|
+
|
|
62
|
+
``ansi_session=False`` on the engine leaves every session as it is, adaptive
|
|
63
|
+
execution and parameter binding included: the one switch says the engine
|
|
64
|
+
may not change the session the user gave it.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
from __future__ import annotations
|
|
68
|
+
|
|
69
|
+
import threading
|
|
70
|
+
import weakref
|
|
71
|
+
from contextlib import contextmanager
|
|
72
|
+
from typing import Any, Iterator, Optional
|
|
73
|
+
|
|
74
|
+
from prograph.logging import get_backend_logger
|
|
75
|
+
from prograph.rules import home
|
|
76
|
+
|
|
77
|
+
SPARK_ANSI_KEY = "spark.sql.ansi.enabled"
|
|
78
|
+
SPARK_ADAPTIVE_KEY = "spark.sql.adaptive.enabled"
|
|
79
|
+
# Off, a decimal result past 38 digits keeps its exact scale and an overflow
|
|
80
|
+
# raises (D7, m2-functions.md 9); on (Spark's default) the scale is cut and
|
|
81
|
+
# the value rounded.
|
|
82
|
+
SPARK_PRECISION_LOSS_KEY = "spark.sql.decimalOperations.allowPrecisionLoss"
|
|
83
|
+
# How Spark reads a backslash in a string literal: ProGraph writes its literals for `false`, the default
|
|
84
|
+
# (`Dialect.render_string`: 'it\'s', 'a\\b'); with `true` every quote and backslash in them is read as
|
|
85
|
+
# written, and a constant such as "O'Neil" matches nothing. Pinned per statement, as ANSI mode is.
|
|
86
|
+
SPARK_ESCAPED_LITERALS_KEY = "spark.sql.parser.escapedStringLiterals"
|
|
87
|
+
DATABRICKS_ANSI_KEY = "ANSI_MODE"
|
|
88
|
+
SPARK_PARAMETERS_IN_ANALYZER_KEY = "spark.sql.legacy.parameterSubstitution.constantsOnly"
|
|
89
|
+
|
|
90
|
+
_logged: set = set()
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _log_once(how: str, was: Any) -> None:
|
|
94
|
+
"""One line per way of turning it on, not one per statement."""
|
|
95
|
+
if how not in _logged:
|
|
96
|
+
_logged.add(how)
|
|
97
|
+
get_backend_logger().info("ansi_session_on", how=how, was=was, now="true")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _is_on(value: Any) -> bool:
|
|
101
|
+
return str(value).strip().lower() == "true"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# A setting the session will not let be read (Databricks serverless and shared Spark Connect sessions refuse a
|
|
105
|
+
# get of a key outside their allow-list, with or without a default).
|
|
106
|
+
UNAVAILABLE = object()
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _refused(exc: BaseException) -> bool:
|
|
110
|
+
"""Whether `exc` is the session refusing a setting: error class CONFIG_NOT_AVAILABLE or SQLSTATE 42K01
|
|
111
|
+
(pyspark's and Spark Connect's errors carry both), the message only as the last resort."""
|
|
112
|
+
for name in ("getCondition", "getErrorClass"): # getErrorClass is deprecated in pyspark 4
|
|
113
|
+
get = getattr(exc, name, None)
|
|
114
|
+
try:
|
|
115
|
+
if callable(get) and str(get() or "").startswith("CONFIG_NOT_AVAILABLE"):
|
|
116
|
+
return True
|
|
117
|
+
except Exception: # noqa: BLE001 - a getter that fails says nothing
|
|
118
|
+
pass
|
|
119
|
+
get = getattr(exc, "getSqlState", None)
|
|
120
|
+
try:
|
|
121
|
+
if callable(get) and get() == "42K01":
|
|
122
|
+
return True
|
|
123
|
+
except Exception: # noqa: BLE001
|
|
124
|
+
pass
|
|
125
|
+
return "CONFIG_NOT_AVAILABLE" in str(exc)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _get(spark: Any, key: str, default: Any = None) -> Any:
|
|
129
|
+
"""The session's value of `key` (`default` where unset), or `UNAVAILABLE` where the session refuses to
|
|
130
|
+
read it. Any other error raises."""
|
|
131
|
+
try:
|
|
132
|
+
return spark.conf.get(key, default) if default is not None or key != SPARK_ANSI_KEY \
|
|
133
|
+
else spark.conf.get(key)
|
|
134
|
+
except Exception as exc: # noqa: BLE001 - pyspark's and Spark Connect's errors differ
|
|
135
|
+
if not _refused(exc):
|
|
136
|
+
raise
|
|
137
|
+
if f"unreadable:{key}" not in _logged:
|
|
138
|
+
_logged.add(f"unreadable:{key}")
|
|
139
|
+
get_backend_logger().info("session_setting_unreadable", key=key)
|
|
140
|
+
return UNAVAILABLE
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _set(spark: Any, key: str, value: str) -> bool:
|
|
144
|
+
"""Set `key` on the session; the precision-loss setting is best-effort:
|
|
145
|
+
a session that refuses it (Databricks serverless and Spark Connect allow a short list of
|
|
146
|
+
settings) runs without it, and a `DecimalPrecisionLossWarning` says once that decimal
|
|
147
|
+
arithmetic past 38 digits rounds there (M2 review B1). Any other refusal raises."""
|
|
148
|
+
try:
|
|
149
|
+
spark.conf.set(key, value)
|
|
150
|
+
return True
|
|
151
|
+
except Exception as exc: # noqa: BLE001 - pyspark's and Spark Connect's errors differ
|
|
152
|
+
if key != SPARK_PRECISION_LOSS_KEY:
|
|
153
|
+
raise
|
|
154
|
+
if "precision_loss_refused" not in _logged:
|
|
155
|
+
_logged.add("precision_loss_refused")
|
|
156
|
+
get_backend_logger().info("precision_loss_setting_refused", error=str(exc)[:200])
|
|
157
|
+
warn_precision_loss("this session refuses the setting")
|
|
158
|
+
return False
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class DecimalPrecisionLossWarning(UserWarning):
|
|
162
|
+
"""Spark rounds a decimal product or quotient whose exact type passes 38 digits (its
|
|
163
|
+
`allowPrecisionLoss` is on), where D7 keeps the digits or makes the value unbound."""
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def warn_precision_loss(why: str) -> None:
|
|
167
|
+
"""Said once per process: decimal arithmetic is not exact on this session (M2 review B1)."""
|
|
168
|
+
if "precision_loss_warned" in _logged:
|
|
169
|
+
return
|
|
170
|
+
_logged.add("precision_loss_warned")
|
|
171
|
+
import warnings
|
|
172
|
+
warnings.warn(
|
|
173
|
+
f"Spark's {SPARK_PRECISION_LOSS_KEY} stays on ({why}): a decimal product or quotient "
|
|
174
|
+
f"whose exact type passes 38 digits is rounded to fewer places instead of exact "
|
|
175
|
+
f"(1.000000000000000001 * 1.5 gives 1.500000000000000000). See sparql-choices.md D7.",
|
|
176
|
+
DecimalPrecisionLossWarning, stacklevel=3)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@home("statement-session", "The session one statement runs on (ANSI mode, adaptive execution, p...",
|
|
180
|
+
terms=("ANSI mode", "adaptive execution", "Spark session", "parameter binding", "Databricks connection"))
|
|
181
|
+
@contextmanager
|
|
182
|
+
def statement_session(spark: Any, ansi_session: bool = True,
|
|
183
|
+
adaptive_off: bool = False) -> Iterator[Any]:
|
|
184
|
+
"""The session one statement of the engine runs on: ANSI mode on,
|
|
185
|
+
decimal precision loss off (D7: a decimal past 38 digits keeps its
|
|
186
|
+
scale or raises), backslash escapes in string literals read as escapes
|
|
187
|
+
(``spark.sql.parser.escapedStringLiterals`` false, as ProGraph writes
|
|
188
|
+
its literals), and adaptive execution off where ``adaptive_off`` (a
|
|
189
|
+
statement that holds a slice below its result).
|
|
190
|
+
|
|
191
|
+
The statement's result must be materialized inside the ``with``: on
|
|
192
|
+
Spark Connect the settings are put back when it ends, and a DataFrame
|
|
193
|
+
collected afterwards would run with the user's settings.
|
|
194
|
+
"""
|
|
195
|
+
if not ansi_session:
|
|
196
|
+
loss = _get(spark, SPARK_PRECISION_LOSS_KEY, "true")
|
|
197
|
+
if loss is UNAVAILABLE or _is_on(loss):
|
|
198
|
+
warn_precision_loss("ansi_session=False leaves the session's settings as they are")
|
|
199
|
+
yield spark
|
|
200
|
+
return
|
|
201
|
+
# Each setting read through `_get`: a session that will not let one be read (serverless, shared Spark
|
|
202
|
+
# Connect) is not refused for it. An unreadable ANSI mode is set and not put back; an unreadable
|
|
203
|
+
# precision loss is left as it is, said once; an unreadable escapedStringLiterals is off (as DBSQL forces
|
|
204
|
+
# it); an unreadable adaptive execution is left as it is.
|
|
205
|
+
wanted = {}
|
|
206
|
+
ansi_before = _get(spark, SPARK_ANSI_KEY)
|
|
207
|
+
if ansi_before is UNAVAILABLE or not _is_on(ansi_before):
|
|
208
|
+
wanted[SPARK_ANSI_KEY] = ("true", ansi_before)
|
|
209
|
+
loss_before = _get(spark, SPARK_PRECISION_LOSS_KEY, None)
|
|
210
|
+
if loss_before is UNAVAILABLE:
|
|
211
|
+
warn_precision_loss("this session does not let the setting be read")
|
|
212
|
+
elif loss_before is None or _is_on(loss_before):
|
|
213
|
+
wanted[SPARK_PRECISION_LOSS_KEY] = ("false", loss_before)
|
|
214
|
+
escaped_before = _get(spark, SPARK_ESCAPED_LITERALS_KEY, None)
|
|
215
|
+
if escaped_before is not UNAVAILABLE and _is_on(escaped_before):
|
|
216
|
+
wanted[SPARK_ESCAPED_LITERALS_KEY] = ("false", escaped_before)
|
|
217
|
+
if adaptive_off:
|
|
218
|
+
# None where the user never set it (on by default): put back unset,
|
|
219
|
+
# not set to the default.
|
|
220
|
+
adaptive_before = _get(spark, SPARK_ADAPTIVE_KEY, None)
|
|
221
|
+
if adaptive_before is not UNAVAILABLE and (adaptive_before is None or _is_on(adaptive_before)):
|
|
222
|
+
wanted[SPARK_ADAPTIVE_KEY] = ("false", adaptive_before)
|
|
223
|
+
if not wanted:
|
|
224
|
+
yield spark
|
|
225
|
+
return
|
|
226
|
+
if SPARK_ADAPTIVE_KEY in wanted:
|
|
227
|
+
get_backend_logger().debug("adaptive_execution_off", reason="slice_below_the_result")
|
|
228
|
+
jsession = getattr(spark, '_jsparkSession', None)
|
|
229
|
+
if jsession is not None:
|
|
230
|
+
clone = type(spark)(spark.sparkContext, jsession.cloneSession())
|
|
231
|
+
for key, (value, _) in wanted.items():
|
|
232
|
+
_set(clone, key, value)
|
|
233
|
+
if SPARK_ANSI_KEY in wanted:
|
|
234
|
+
_log_once("spark_cloned_session", ansi_before)
|
|
235
|
+
yield clone
|
|
236
|
+
return
|
|
237
|
+
if SPARK_ANSI_KEY in wanted:
|
|
238
|
+
_log_once("spark_connect_set_and_restore", ansi_before)
|
|
239
|
+
wanted = {k: v for k, v in wanted.items() if _set(spark, k, v[0])}
|
|
240
|
+
try:
|
|
241
|
+
yield spark
|
|
242
|
+
finally:
|
|
243
|
+
for key, (_, before) in wanted.items():
|
|
244
|
+
if before is UNAVAILABLE:
|
|
245
|
+
continue # what the session would not show is not put back
|
|
246
|
+
if before is None:
|
|
247
|
+
spark.conf.unset(key)
|
|
248
|
+
else:
|
|
249
|
+
spark.conf.set(key, before)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
# Whether a classic session's parser binds parameters, asked once per session
|
|
253
|
+
# (its extensions are fixed when it is built).
|
|
254
|
+
_parser_binds: "weakref.WeakKeyDictionary[Any, bool]" = weakref.WeakKeyDictionary()
|
|
255
|
+
# Held from reading the setting to putting it back (the module's docstring).
|
|
256
|
+
_parameters_lock = threading.Lock()
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _parser_binds_parameters(spark: Any) -> bool:
|
|
260
|
+
known = _parser_binds.get(spark)
|
|
261
|
+
if known is None:
|
|
262
|
+
try:
|
|
263
|
+
spark.sql("SELECT :p AS p", args={"p": 1})
|
|
264
|
+
known = True
|
|
265
|
+
except Exception as e: # pyspark's AnalysisException, by its condition
|
|
266
|
+
condition = getattr(e, "getCondition", None) or getattr(e, "getErrorClass", None)
|
|
267
|
+
if condition is None or condition() != "UNBOUND_SQL_PARAMETER":
|
|
268
|
+
raise
|
|
269
|
+
known = False
|
|
270
|
+
get_backend_logger().info("parameters_bound_in_analyzer",
|
|
271
|
+
reason="the session's parser drops parameters")
|
|
272
|
+
_parser_binds[spark] = known
|
|
273
|
+
return known
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
@contextmanager
|
|
277
|
+
def parameter_session(spark: Any, ansi_session: bool = True) -> Iterator[Any]:
|
|
278
|
+
"""The session a caller's statement with parameters is parsed on
|
|
279
|
+
(the module's docstring): the session itself, with
|
|
280
|
+
the values bound in the analyzer where its parser drops them. The
|
|
281
|
+
statement must be parsed (``sql()`` called) inside the ``with``."""
|
|
282
|
+
if not ansi_session or getattr(spark, '_jsparkSession', None) is None:
|
|
283
|
+
yield spark
|
|
284
|
+
return
|
|
285
|
+
with _parameters_lock:
|
|
286
|
+
before = _get(spark, SPARK_PARAMETERS_IN_ANALYZER_KEY, None)
|
|
287
|
+
if before is UNAVAILABLE: # a session that will not show it: the statement as it is
|
|
288
|
+
yield spark
|
|
289
|
+
return
|
|
290
|
+
# The user's own setting binds in the analyzer already; the parser is
|
|
291
|
+
# asked only with it off, so the answer is the parser's.
|
|
292
|
+
own_parser = not _is_on(before) and _parser_binds_parameters(spark)
|
|
293
|
+
if _is_on(before):
|
|
294
|
+
yield spark
|
|
295
|
+
return
|
|
296
|
+
if not own_parser:
|
|
297
|
+
spark.conf.set(SPARK_PARAMETERS_IN_ANALYZER_KEY, "true")
|
|
298
|
+
try:
|
|
299
|
+
yield spark
|
|
300
|
+
finally:
|
|
301
|
+
if before is None:
|
|
302
|
+
spark.conf.unset(SPARK_PARAMETERS_IN_ANALYZER_KEY)
|
|
303
|
+
else:
|
|
304
|
+
spark.conf.set(SPARK_PARAMETERS_IN_ANALYZER_KEY, before)
|
|
305
|
+
return
|
|
306
|
+
# Spark's own parser binds: the statement needs no setting and no lock.
|
|
307
|
+
yield spark
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def connection_ansi_on(connection: Any) -> Optional[str]:
|
|
311
|
+
"""Turn ANSI mode on for a Databricks SQL connection the engine was
|
|
312
|
+
handed. Returns the value to put back, or None when nothing was
|
|
313
|
+
changed: it was on already, or its value could not be read (a setting
|
|
314
|
+
that cannot be put back is not touched)."""
|
|
315
|
+
logger = get_backend_logger()
|
|
316
|
+
cursor = connection.cursor()
|
|
317
|
+
try:
|
|
318
|
+
cursor.execute(f"SET {DATABRICKS_ANSI_KEY}")
|
|
319
|
+
row = cursor.fetchone()
|
|
320
|
+
before = None if row is None else str(row[-1]).strip().lower()
|
|
321
|
+
if before not in ("true", "false"):
|
|
322
|
+
logger.warning("ansi_session_skip", reason="the connection's ANSI_MODE could not be read",
|
|
323
|
+
read=None if row is None else str(row[-1]))
|
|
324
|
+
return None
|
|
325
|
+
if before == "true":
|
|
326
|
+
return None
|
|
327
|
+
cursor.execute(f"SET {DATABRICKS_ANSI_KEY} = true")
|
|
328
|
+
logger.info("ansi_session_on", how="databricks_connection", was=before, now="true")
|
|
329
|
+
return before
|
|
330
|
+
finally:
|
|
331
|
+
cursor.close()
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def connection_ansi_restore(connection: Any, before: Optional[str]) -> None:
|
|
335
|
+
"""Put a Databricks SQL connection's ANSI mode back. A connection the
|
|
336
|
+
user has closed already has no setting left to restore."""
|
|
337
|
+
if before is None:
|
|
338
|
+
return
|
|
339
|
+
logger = get_backend_logger()
|
|
340
|
+
try:
|
|
341
|
+
cursor = connection.cursor()
|
|
342
|
+
try:
|
|
343
|
+
cursor.execute(f"SET {DATABRICKS_ANSI_KEY} = {before}")
|
|
344
|
+
finally:
|
|
345
|
+
cursor.close()
|
|
346
|
+
logger.info("ansi_session_restored", how="databricks_connection", now=before)
|
|
347
|
+
except Exception as e:
|
|
348
|
+
logger.debug("ansi_session_restore_failed", error=str(e))
|