prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,172 @@
1
+ """Render and execute the CREATE VIEW statements behind generated views.
2
+
3
+ The :mod:`prograph.genie.view_generator` module produces
4
+ :class:`ViewDefinition` instances whose ``sql`` field is the SELECT body
5
+ of a view. This module wraps each body in a dialect-correct
6
+ ``CREATE OR REPLACE VIEW <fully_qualified_name> AS <body>`` statement
7
+ and (optionally) executes it against a connection.
8
+
9
+ Two surfaces:
10
+
11
+ - :func:`render_view_sql`, pure-function, returns the executable SQL
12
+ string for a view. Used by the CLI's ``generate-views`` subcommand
13
+ which writes the SQL to a ``.sql`` file for separate execution.
14
+
15
+ - :func:`write_views`, takes an open PEP 249 connection and runs the
16
+ CREATE OR REPLACE statements directly. Used by
17
+ :meth:`ProGraph.write_views` on SQL backends.
18
+
19
+ The split lets the same generator output serve "emit SQL for someone
20
+ else to run" and "create views on the engine's own connection" without
21
+ duplicating the dialect-aware naming and quoting logic.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple
27
+
28
+ if TYPE_CHECKING:
29
+ from prograph.genie.formats import ViewDefinition
30
+ from prograph.plan.sql.dialect import DialectConfig
31
+
32
+
33
+ def render_view_sql(
34
+ view: "ViewDefinition",
35
+ target_catalog: Optional[str] = None,
36
+ target_schema: str = "prograph_views",
37
+ dialect: Optional["DialectConfig"] = None,
38
+ ) -> str:
39
+ """Render a ``CREATE OR REPLACE VIEW`` statement for one ViewDefinition.
40
+
41
+ Args:
42
+ view: The :class:`ViewDefinition` to render.
43
+ target_catalog: Optional catalog prefix. Required for backends
44
+ that use three-part names (Databricks SQL); omitted for
45
+ two-part backends (Postgres, SQL Server).
46
+ target_schema: The schema/namespace for the view. Conventionally
47
+ ``"prograph_views"``. The user can configure another via
48
+ the engine constructor or CLI argument.
49
+ dialect: SQL dialect for identifier quoting. Defaults to the
50
+ Spark / Databricks SQL backtick form.
51
+
52
+ Returns:
53
+ A complete ``CREATE OR REPLACE VIEW <fq> AS\\n<body>`` string,
54
+ no trailing semicolon (callers that need a semicolon add it).
55
+ """
56
+ if dialect is None:
57
+ from prograph.plan.sql.dialect import SPARK_DIALECT
58
+ dialect = SPARK_DIALECT
59
+
60
+ from prograph.plan.sql.dialect import get_dialect_instance
61
+ q = get_dialect_instance(dialect.name).quote
62
+ parts: List[str] = []
63
+ if target_catalog:
64
+ parts.append(q(target_catalog))
65
+ parts.append(q(target_schema))
66
+ parts.append(q(view.name))
67
+ fq = ".".join(parts)
68
+
69
+ return f"{get_dialect_instance(dialect.name).create_or_replace_view()} {fq} AS\n{view.sql}"
70
+
71
+
72
+ def render_views_script(
73
+ views: Iterable["ViewDefinition"],
74
+ target_catalog: Optional[str] = None,
75
+ target_schema: str = "prograph_views",
76
+ dialect: Optional["DialectConfig"] = None,
77
+ include_header: bool = True,
78
+ ) -> str:
79
+ """Render a complete ``.sql`` script for a batch of views.
80
+
81
+ Each view becomes a ``CREATE OR REPLACE VIEW … ;`` statement
82
+ separated by a blank line. An optional header comment describes
83
+ the script's provenance.
84
+
85
+ Suitable for ``prograph generate-views --output views.sql``, the
86
+ resulting file runs unchanged against the target warehouse via
87
+ ``databricks-sql-cli``, ``psql -f``, etc.
88
+ """
89
+ statements: List[str] = []
90
+ if include_header:
91
+ statements.append(
92
+ "-- Generated by ProGraph (prograph.genie.view_generator)\n"
93
+ "-- See docs/guides/genie.md for how they are used.\n"
94
+ "-- Each view is a query-time projection, no storage cost."
95
+ )
96
+ for view in views:
97
+ sql = render_view_sql(
98
+ view,
99
+ target_catalog=target_catalog,
100
+ target_schema=target_schema,
101
+ dialect=dialect,
102
+ )
103
+ comment = f"-- {view.source.value}: {view.description}"
104
+ statements.append(f"{comment}\n{sql};")
105
+ return "\n\n".join(statements) + "\n"
106
+
107
+
108
+ def write_views(
109
+ views: Iterable["ViewDefinition"],
110
+ connection: Any,
111
+ target_catalog: Optional[str] = None,
112
+ target_schema: str = "prograph_views",
113
+ dialect: Optional["DialectConfig"] = None,
114
+ ensure_schema: bool = True,
115
+ ) -> List[Tuple[str, str]]:
116
+ """Execute ``CREATE OR REPLACE VIEW`` for each view against ``connection``.
117
+
118
+ Args:
119
+ views: Iterable of :class:`ViewDefinition` to write.
120
+ connection: PEP 249 connection (databricks-sql-connector, psycopg,
121
+ pyodbc, …). The function does not own the connection's
122
+ lifecycle.
123
+ target_catalog: Optional catalog prefix.
124
+ target_schema: Target schema/namespace for the views.
125
+ dialect: SQL dialect for identifier quoting.
126
+ ensure_schema: When ``True`` (default), issues
127
+ ``CREATE SCHEMA IF NOT EXISTS`` before the views. Set
128
+ ``False`` if the schema is managed externally.
129
+
130
+ Returns:
131
+ List of ``(view_name, fully_qualified_name)`` tuples for the
132
+ views actually created. Callers can use this list to confirm
133
+ what landed, build observability events, etc.
134
+ """
135
+ if dialect is None:
136
+ from prograph.plan.sql.dialect import SPARK_DIALECT
137
+ dialect = SPARK_DIALECT
138
+
139
+ from prograph.plan.sql.dialect import get_dialect_instance
140
+ q = get_dialect_instance(dialect.name).quote
141
+ created: List[Tuple[str, str]] = []
142
+
143
+ with connection.cursor() as cur:
144
+ if ensure_schema:
145
+ schema_parts: List[str] = []
146
+ if target_catalog:
147
+ schema_parts.append(q(target_catalog))
148
+ schema_parts.append(q(target_schema))
149
+ cur.execute(
150
+ f"CREATE SCHEMA IF NOT EXISTS {'.'.join(schema_parts)}"
151
+ )
152
+
153
+ for view in views:
154
+ sql = render_view_sql(
155
+ view,
156
+ target_catalog=target_catalog,
157
+ target_schema=target_schema,
158
+ dialect=dialect,
159
+ )
160
+ cur.execute(sql)
161
+ # Reconstruct the fq name for the result tuple.
162
+ parts: List[str] = []
163
+ if target_catalog:
164
+ parts.append(target_catalog)
165
+ parts.append(target_schema)
166
+ parts.append(view.name)
167
+ created.append((view.name, ".".join(parts)))
168
+
169
+ return created
170
+
171
+
172
+ __all__ = ["render_view_sql", "render_views_script", "write_views"]
@@ -0,0 +1 @@
1
+ """The backends' graph objects: each runs the generated SQL on its database."""
@@ -0,0 +1,348 @@
1
+ """ANSI mode on for the statements the engine runs.
2
+
3
+ With ANSI mode off, Spark and Databricks wrap an integer that overflows
4
+ (``9223372036854775807 + 1`` is ``-9223372036854775808``) and answer NULL
5
+ for what has no value. Cypher has no wrapped integer: these are errors,
6
+ and with ANSI mode on the database raises them. The setting belongs to the
7
+ session, and the session is the user's, so the engine turns it on only for
8
+ its own statements and leaves the user's setting as it found it:
9
+
10
+ - Spark, a classic session: each statement runs in a clone of the session
11
+ (``cloneSession``), which has its own configuration, sees the parent's
12
+ temporary views and cached frames as of the clone, and costs about a
13
+ millisecond. Setting and restoring the shared session instead leaks: a
14
+ second thread on it saw ANSI mode on in 50 of 50 queries.
15
+ - Spark Connect: a client session cannot be cloned, so the setting is
16
+ changed on the session for the length of the statement and put back in a
17
+ ``finally``. Another thread on the same session sees ANSI mode on while
18
+ the statement runs.
19
+ - A Databricks SQL connection: ``SET ANSI_MODE = true`` once, when the
20
+ engine takes the connection, and the value it had is put back by
21
+ ``close()``. A statement the user runs on that connection in between
22
+ sees ANSI mode on.
23
+
24
+ Adaptive execution off, on Spark, for a statement that holds a slice (a
25
+ LIMIT or an OFFSET) anywhere but at its result. Spark
26
+ 4.1.3's adaptive execution sometimes loses such a slice: a LIMIT 3 OFFSET 1
27
+ over a LIMIT 6 OFFSET 2 over a LEFT JOIN answered 5 rows in about one run of
28
+ ten, with adaptive execution on only (apache/spark#59434). The engine reads
29
+ from the statement's tree whether it holds one
30
+ (``tree_utils.holds_a_slice_below_the_result``), and the statement then runs
31
+ with ``spark.sql.adaptive.enabled`` false in the same session as its ANSI
32
+ mode: a clone on a classic session, set and put back on Spark Connect, where
33
+ another statement on that session in between sees it off. Every other
34
+ statement runs with the user's setting. A Databricks SQL connection is not
35
+ touched: a warehouse answers these statements.
36
+
37
+ Parameters of a caller's own statement (``engine.execute_sql``). Spark 4.1 binds ``:name`` and ``?`` in its parser: the session's
38
+ parser is handed the values (``parsePlanWithParameters``) and puts them in
39
+ as literals. A session extension that wraps the parser and does not pass
40
+ the values on leaves every marker unbound, and the statement fails with
41
+ UNBOUND_SQL_PARAMETER: Delta's ``DeltaSqlParser`` in every release for
42
+ Spark 4.1 (4.1.0, 4.2.0, 4.3.1 and 4.4.1 tried; each inherits the default
43
+ of ``ParserInterface``, which drops them), and Iceberg's before
44
+ apache/iceberg#16626. On such a session
45
+ the statement is parsed with ``spark.sql.legacy.parameterSubstitution.
46
+ constantsOnly`` true, Spark's own setting for binding the values in the
47
+ analyzer as Spark 4.0 did, which no parser can drop: a value goes where a
48
+ constant goes (a comparison, LIMIT, IDENTIFIER), and a marker in a view's
49
+ body is refused by Spark. Whether a session's parser binds is asked of the
50
+ session once, with ``SELECT :p AS p``. On a classic session the analysis is
51
+ done when ``sql()`` returns, so the setting is put back then, and a
52
+ DataFrame returned keeps its values. The setting is read, set, used by
53
+ ``sql()`` and put back under one lock of this module, and the session's
54
+ parser is asked under it too, so two threads' statements through the
55
+ engine never see each other's setting: without it one thread put back the
56
+ setting while another's statement was being parsed, and that statement was
57
+ unbound (8 of 120 in the review), or the setting was left on.
58
+ A statement the user runs on the session outside the engine while the
59
+ setting is on is bound in the analyzer too. On Spark Connect the server's parser cannot be asked and
60
+ its statement is analyzed later, so its session is left as it is.
61
+
62
+ ``ansi_session=False`` on the engine leaves every session as it is, adaptive
63
+ execution and parameter binding included: the one switch says the engine
64
+ may not change the session the user gave it.
65
+ """
66
+
67
+ from __future__ import annotations
68
+
69
+ import threading
70
+ import weakref
71
+ from contextlib import contextmanager
72
+ from typing import Any, Iterator, Optional
73
+
74
+ from prograph.logging import get_backend_logger
75
+ from prograph.rules import home
76
+
77
+ SPARK_ANSI_KEY = "spark.sql.ansi.enabled"
78
+ SPARK_ADAPTIVE_KEY = "spark.sql.adaptive.enabled"
79
+ # Off, a decimal result past 38 digits keeps its exact scale and an overflow
80
+ # raises (D7, m2-functions.md 9); on (Spark's default) the scale is cut and
81
+ # the value rounded.
82
+ SPARK_PRECISION_LOSS_KEY = "spark.sql.decimalOperations.allowPrecisionLoss"
83
+ # How Spark reads a backslash in a string literal: ProGraph writes its literals for `false`, the default
84
+ # (`Dialect.render_string`: 'it\'s', 'a\\b'); with `true` every quote and backslash in them is read as
85
+ # written, and a constant such as "O'Neil" matches nothing. Pinned per statement, as ANSI mode is.
86
+ SPARK_ESCAPED_LITERALS_KEY = "spark.sql.parser.escapedStringLiterals"
87
+ DATABRICKS_ANSI_KEY = "ANSI_MODE"
88
+ SPARK_PARAMETERS_IN_ANALYZER_KEY = "spark.sql.legacy.parameterSubstitution.constantsOnly"
89
+
90
+ _logged: set = set()
91
+
92
+
93
+ def _log_once(how: str, was: Any) -> None:
94
+ """One line per way of turning it on, not one per statement."""
95
+ if how not in _logged:
96
+ _logged.add(how)
97
+ get_backend_logger().info("ansi_session_on", how=how, was=was, now="true")
98
+
99
+
100
+ def _is_on(value: Any) -> bool:
101
+ return str(value).strip().lower() == "true"
102
+
103
+
104
+ # A setting the session will not let be read (Databricks serverless and shared Spark Connect sessions refuse a
105
+ # get of a key outside their allow-list, with or without a default).
106
+ UNAVAILABLE = object()
107
+
108
+
109
+ def _refused(exc: BaseException) -> bool:
110
+ """Whether `exc` is the session refusing a setting: error class CONFIG_NOT_AVAILABLE or SQLSTATE 42K01
111
+ (pyspark's and Spark Connect's errors carry both), the message only as the last resort."""
112
+ for name in ("getCondition", "getErrorClass"): # getErrorClass is deprecated in pyspark 4
113
+ get = getattr(exc, name, None)
114
+ try:
115
+ if callable(get) and str(get() or "").startswith("CONFIG_NOT_AVAILABLE"):
116
+ return True
117
+ except Exception: # noqa: BLE001 - a getter that fails says nothing
118
+ pass
119
+ get = getattr(exc, "getSqlState", None)
120
+ try:
121
+ if callable(get) and get() == "42K01":
122
+ return True
123
+ except Exception: # noqa: BLE001
124
+ pass
125
+ return "CONFIG_NOT_AVAILABLE" in str(exc)
126
+
127
+
128
+ def _get(spark: Any, key: str, default: Any = None) -> Any:
129
+ """The session's value of `key` (`default` where unset), or `UNAVAILABLE` where the session refuses to
130
+ read it. Any other error raises."""
131
+ try:
132
+ return spark.conf.get(key, default) if default is not None or key != SPARK_ANSI_KEY \
133
+ else spark.conf.get(key)
134
+ except Exception as exc: # noqa: BLE001 - pyspark's and Spark Connect's errors differ
135
+ if not _refused(exc):
136
+ raise
137
+ if f"unreadable:{key}" not in _logged:
138
+ _logged.add(f"unreadable:{key}")
139
+ get_backend_logger().info("session_setting_unreadable", key=key)
140
+ return UNAVAILABLE
141
+
142
+
143
+ def _set(spark: Any, key: str, value: str) -> bool:
144
+ """Set `key` on the session; the precision-loss setting is best-effort:
145
+ a session that refuses it (Databricks serverless and Spark Connect allow a short list of
146
+ settings) runs without it, and a `DecimalPrecisionLossWarning` says once that decimal
147
+ arithmetic past 38 digits rounds there (M2 review B1). Any other refusal raises."""
148
+ try:
149
+ spark.conf.set(key, value)
150
+ return True
151
+ except Exception as exc: # noqa: BLE001 - pyspark's and Spark Connect's errors differ
152
+ if key != SPARK_PRECISION_LOSS_KEY:
153
+ raise
154
+ if "precision_loss_refused" not in _logged:
155
+ _logged.add("precision_loss_refused")
156
+ get_backend_logger().info("precision_loss_setting_refused", error=str(exc)[:200])
157
+ warn_precision_loss("this session refuses the setting")
158
+ return False
159
+
160
+
161
+ class DecimalPrecisionLossWarning(UserWarning):
162
+ """Spark rounds a decimal product or quotient whose exact type passes 38 digits (its
163
+ `allowPrecisionLoss` is on), where D7 keeps the digits or makes the value unbound."""
164
+
165
+
166
+ def warn_precision_loss(why: str) -> None:
167
+ """Said once per process: decimal arithmetic is not exact on this session (M2 review B1)."""
168
+ if "precision_loss_warned" in _logged:
169
+ return
170
+ _logged.add("precision_loss_warned")
171
+ import warnings
172
+ warnings.warn(
173
+ f"Spark's {SPARK_PRECISION_LOSS_KEY} stays on ({why}): a decimal product or quotient "
174
+ f"whose exact type passes 38 digits is rounded to fewer places instead of exact "
175
+ f"(1.000000000000000001 * 1.5 gives 1.500000000000000000). See sparql-choices.md D7.",
176
+ DecimalPrecisionLossWarning, stacklevel=3)
177
+
178
+
179
+ @home("statement-session", "The session one statement runs on (ANSI mode, adaptive execution, p...",
180
+ terms=("ANSI mode", "adaptive execution", "Spark session", "parameter binding", "Databricks connection"))
181
+ @contextmanager
182
+ def statement_session(spark: Any, ansi_session: bool = True,
183
+ adaptive_off: bool = False) -> Iterator[Any]:
184
+ """The session one statement of the engine runs on: ANSI mode on,
185
+ decimal precision loss off (D7: a decimal past 38 digits keeps its
186
+ scale or raises), backslash escapes in string literals read as escapes
187
+ (``spark.sql.parser.escapedStringLiterals`` false, as ProGraph writes
188
+ its literals), and adaptive execution off where ``adaptive_off`` (a
189
+ statement that holds a slice below its result).
190
+
191
+ The statement's result must be materialized inside the ``with``: on
192
+ Spark Connect the settings are put back when it ends, and a DataFrame
193
+ collected afterwards would run with the user's settings.
194
+ """
195
+ if not ansi_session:
196
+ loss = _get(spark, SPARK_PRECISION_LOSS_KEY, "true")
197
+ if loss is UNAVAILABLE or _is_on(loss):
198
+ warn_precision_loss("ansi_session=False leaves the session's settings as they are")
199
+ yield spark
200
+ return
201
+ # Each setting read through `_get`: a session that will not let one be read (serverless, shared Spark
202
+ # Connect) is not refused for it. An unreadable ANSI mode is set and not put back; an unreadable
203
+ # precision loss is left as it is, said once; an unreadable escapedStringLiterals is off (as DBSQL forces
204
+ # it); an unreadable adaptive execution is left as it is.
205
+ wanted = {}
206
+ ansi_before = _get(spark, SPARK_ANSI_KEY)
207
+ if ansi_before is UNAVAILABLE or not _is_on(ansi_before):
208
+ wanted[SPARK_ANSI_KEY] = ("true", ansi_before)
209
+ loss_before = _get(spark, SPARK_PRECISION_LOSS_KEY, None)
210
+ if loss_before is UNAVAILABLE:
211
+ warn_precision_loss("this session does not let the setting be read")
212
+ elif loss_before is None or _is_on(loss_before):
213
+ wanted[SPARK_PRECISION_LOSS_KEY] = ("false", loss_before)
214
+ escaped_before = _get(spark, SPARK_ESCAPED_LITERALS_KEY, None)
215
+ if escaped_before is not UNAVAILABLE and _is_on(escaped_before):
216
+ wanted[SPARK_ESCAPED_LITERALS_KEY] = ("false", escaped_before)
217
+ if adaptive_off:
218
+ # None where the user never set it (on by default): put back unset,
219
+ # not set to the default.
220
+ adaptive_before = _get(spark, SPARK_ADAPTIVE_KEY, None)
221
+ if adaptive_before is not UNAVAILABLE and (adaptive_before is None or _is_on(adaptive_before)):
222
+ wanted[SPARK_ADAPTIVE_KEY] = ("false", adaptive_before)
223
+ if not wanted:
224
+ yield spark
225
+ return
226
+ if SPARK_ADAPTIVE_KEY in wanted:
227
+ get_backend_logger().debug("adaptive_execution_off", reason="slice_below_the_result")
228
+ jsession = getattr(spark, '_jsparkSession', None)
229
+ if jsession is not None:
230
+ clone = type(spark)(spark.sparkContext, jsession.cloneSession())
231
+ for key, (value, _) in wanted.items():
232
+ _set(clone, key, value)
233
+ if SPARK_ANSI_KEY in wanted:
234
+ _log_once("spark_cloned_session", ansi_before)
235
+ yield clone
236
+ return
237
+ if SPARK_ANSI_KEY in wanted:
238
+ _log_once("spark_connect_set_and_restore", ansi_before)
239
+ wanted = {k: v for k, v in wanted.items() if _set(spark, k, v[0])}
240
+ try:
241
+ yield spark
242
+ finally:
243
+ for key, (_, before) in wanted.items():
244
+ if before is UNAVAILABLE:
245
+ continue # what the session would not show is not put back
246
+ if before is None:
247
+ spark.conf.unset(key)
248
+ else:
249
+ spark.conf.set(key, before)
250
+
251
+
252
+ # Whether a classic session's parser binds parameters, asked once per session
253
+ # (its extensions are fixed when it is built).
254
+ _parser_binds: "weakref.WeakKeyDictionary[Any, bool]" = weakref.WeakKeyDictionary()
255
+ # Held from reading the setting to putting it back (the module's docstring).
256
+ _parameters_lock = threading.Lock()
257
+
258
+
259
+ def _parser_binds_parameters(spark: Any) -> bool:
260
+ known = _parser_binds.get(spark)
261
+ if known is None:
262
+ try:
263
+ spark.sql("SELECT :p AS p", args={"p": 1})
264
+ known = True
265
+ except Exception as e: # pyspark's AnalysisException, by its condition
266
+ condition = getattr(e, "getCondition", None) or getattr(e, "getErrorClass", None)
267
+ if condition is None or condition() != "UNBOUND_SQL_PARAMETER":
268
+ raise
269
+ known = False
270
+ get_backend_logger().info("parameters_bound_in_analyzer",
271
+ reason="the session's parser drops parameters")
272
+ _parser_binds[spark] = known
273
+ return known
274
+
275
+
276
+ @contextmanager
277
+ def parameter_session(spark: Any, ansi_session: bool = True) -> Iterator[Any]:
278
+ """The session a caller's statement with parameters is parsed on
279
+ (the module's docstring): the session itself, with
280
+ the values bound in the analyzer where its parser drops them. The
281
+ statement must be parsed (``sql()`` called) inside the ``with``."""
282
+ if not ansi_session or getattr(spark, '_jsparkSession', None) is None:
283
+ yield spark
284
+ return
285
+ with _parameters_lock:
286
+ before = _get(spark, SPARK_PARAMETERS_IN_ANALYZER_KEY, None)
287
+ if before is UNAVAILABLE: # a session that will not show it: the statement as it is
288
+ yield spark
289
+ return
290
+ # The user's own setting binds in the analyzer already; the parser is
291
+ # asked only with it off, so the answer is the parser's.
292
+ own_parser = not _is_on(before) and _parser_binds_parameters(spark)
293
+ if _is_on(before):
294
+ yield spark
295
+ return
296
+ if not own_parser:
297
+ spark.conf.set(SPARK_PARAMETERS_IN_ANALYZER_KEY, "true")
298
+ try:
299
+ yield spark
300
+ finally:
301
+ if before is None:
302
+ spark.conf.unset(SPARK_PARAMETERS_IN_ANALYZER_KEY)
303
+ else:
304
+ spark.conf.set(SPARK_PARAMETERS_IN_ANALYZER_KEY, before)
305
+ return
306
+ # Spark's own parser binds: the statement needs no setting and no lock.
307
+ yield spark
308
+
309
+
310
+ def connection_ansi_on(connection: Any) -> Optional[str]:
311
+ """Turn ANSI mode on for a Databricks SQL connection the engine was
312
+ handed. Returns the value to put back, or None when nothing was
313
+ changed: it was on already, or its value could not be read (a setting
314
+ that cannot be put back is not touched)."""
315
+ logger = get_backend_logger()
316
+ cursor = connection.cursor()
317
+ try:
318
+ cursor.execute(f"SET {DATABRICKS_ANSI_KEY}")
319
+ row = cursor.fetchone()
320
+ before = None if row is None else str(row[-1]).strip().lower()
321
+ if before not in ("true", "false"):
322
+ logger.warning("ansi_session_skip", reason="the connection's ANSI_MODE could not be read",
323
+ read=None if row is None else str(row[-1]))
324
+ return None
325
+ if before == "true":
326
+ return None
327
+ cursor.execute(f"SET {DATABRICKS_ANSI_KEY} = true")
328
+ logger.info("ansi_session_on", how="databricks_connection", was=before, now="true")
329
+ return before
330
+ finally:
331
+ cursor.close()
332
+
333
+
334
+ def connection_ansi_restore(connection: Any, before: Optional[str]) -> None:
335
+ """Put a Databricks SQL connection's ANSI mode back. A connection the
336
+ user has closed already has no setting left to restore."""
337
+ if before is None:
338
+ return
339
+ logger = get_backend_logger()
340
+ try:
341
+ cursor = connection.cursor()
342
+ try:
343
+ cursor.execute(f"SET {DATABRICKS_ANSI_KEY} = {before}")
344
+ finally:
345
+ cursor.close()
346
+ logger.info("ansi_session_restored", how="databricks_connection", now=before)
347
+ except Exception as e:
348
+ logger.debug("ansi_session_restore_failed", error=str(e))