prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,781 @@
1
+ """Relational Algebra IR for SQL Generation.
2
+
3
+ This module defines a language-agnostic tree-based intermediate representation
4
+ for SQL queries. All graph query languages (Cypher, Gremlin, SPARQL, SQL/PGQ)
5
+ lower their execution plans into this common tree, which is then optimized
6
+ and serialized to SQL.
7
+
8
+ The tree is NOT Cypher-specific. Gremlin operations like fold(), choose(),
9
+ path(), and group() have their own node types. The tree represents what the
10
+ SQL engine needs to DO, not what the source language said.
11
+
12
+ Architecture:
13
+
14
+ Cypher ──┐
15
+ Gremlin ──┼──→ plan Operations ──→ TreeBuilder ──→ RelNode tree
16
+ SPARQL ──┘ ──→ Passes ──→ Serializer ──→ SQL
17
+
18
+ There is one TreeBuilder (prograph.plan.interpreter.tree_builder). Each
19
+ language's plan builder lowers to the same Operations first.
20
+
21
+ Design principles:
22
+
23
+ 1. Language-agnostic nodes. ColumnRef carries SQL expressions and aliases,
24
+ with optional metadata tags, not Cypher-specific origin_var/origin_prop.
25
+
26
+ 2. Structured expressions, not strings. Predicates and join conditions use
27
+ an expression algebra (Expr nodes) so optimization passes can inspect
28
+ and transform them without parsing SQL strings.
29
+
30
+ 3. Sub-plan nodes for Gremlin operations that execute full query pipelines
31
+ per input row (choose, coalesce, local, optional).
32
+
33
+ 4. Optimization passes are tree → tree functions. The tree is immutable
34
+ during serialization, all rewrites happen before the serializer runs.
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ from dataclasses import dataclass, field
40
+ from enum import Enum, auto
41
+ from typing import Any, Dict, List, Optional, Set, Tuple, Union
42
+
43
+ from .dialect import alias_name
44
+
45
+
46
+ # =============================================================================
47
+ # Expression Algebra
48
+ # =============================================================================
49
+ # Predicates (Filter.predicate, Join.on, Aggregate.having) use structured
50
+ # expressions, not raw SQL strings. This allows optimization passes to
51
+ # inspect, decompose, and rewrite predicates (e.g., predicate pushdown
52
+ # needs to know which columns a predicate references).
53
+ #
54
+ # The serializer translates Expr nodes to SQL strings.
55
+ # =============================================================================
56
+
57
+ class Expr:
58
+ """Base class for expression nodes."""
59
+
60
+ def columns_referenced(self) -> Set['ColumnId']:
61
+ """Return the ColumnId identities referenced by this expression."""
62
+ raise NotImplementedError
63
+
64
+
65
+ @dataclass(frozen=True)
66
+ class ColExpr(Expr):
67
+ """Reference to a column by its ColumnRef.
68
+
69
+ The primary expression type. Carries the full ColumnRef so the
70
+ serializer can emit either the SQL expression or the alias form.
71
+ """
72
+ ref: 'ColumnRef'
73
+
74
+ def columns_referenced(self) -> Set['ColumnId']:
75
+ return {self.ref.identity()}
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class LitExpr(Expr):
80
+ """A literal value (string, number, boolean, null)."""
81
+ value: Any
82
+ sql: str = ''
83
+ """Pre-rendered SQL form (e.g., "'Alice'", "42", "NULL")."""
84
+
85
+ def columns_referenced(self) -> Set[str]:
86
+ return set()
87
+
88
+
89
+ @dataclass(frozen=True)
90
+ class CompareExpr(Expr):
91
+ """Binary comparison: left op right."""
92
+ left: Expr
93
+ op: str # '=', '!=', '<', '>', '<=', '>='
94
+ right: Expr
95
+
96
+ def columns_referenced(self) -> Set[str]:
97
+ return self.left.columns_referenced() | self.right.columns_referenced()
98
+
99
+
100
+ @dataclass(frozen=True)
101
+ class AndExpr(Expr):
102
+ """Conjunction of predicates."""
103
+ children: Tuple[Expr, ...] = ()
104
+
105
+ def columns_referenced(self) -> Set[str]:
106
+ result: Set[str] = set()
107
+ for child in self.children:
108
+ result |= child.columns_referenced()
109
+ return result
110
+
111
+
112
+ @dataclass(frozen=True)
113
+ class OrExpr(Expr):
114
+ """Disjunction of predicates."""
115
+ children: Tuple[Expr, ...] = ()
116
+
117
+ def columns_referenced(self) -> Set[str]:
118
+ result: Set[str] = set()
119
+ for child in self.children:
120
+ result |= child.columns_referenced()
121
+ return result
122
+
123
+
124
+ @dataclass(frozen=True)
125
+ class NotExpr(Expr):
126
+ """Logical negation."""
127
+ child: Expr = None
128
+
129
+ def columns_referenced(self) -> Set[str]:
130
+ return self.child.columns_referenced() if self.child else set()
131
+
132
+
133
+ @dataclass(frozen=True)
134
+ class IsNullExpr(Expr):
135
+ """IS NULL / IS NOT NULL."""
136
+ child: Expr = None
137
+ negated: bool = False
138
+
139
+ def columns_referenced(self) -> Set[str]:
140
+ return self.child.columns_referenced() if self.child else set()
141
+
142
+
143
+ @dataclass(frozen=True)
144
+ class FuncExpr(Expr):
145
+ """Function call: name(args...)."""
146
+ name: str = ''
147
+ args: Tuple[Expr, ...] = ()
148
+
149
+ def columns_referenced(self) -> Set[str]:
150
+ result: Set[str] = set()
151
+ for arg in self.args:
152
+ result |= arg.columns_referenced()
153
+ return result
154
+
155
+
156
+ @dataclass(frozen=True)
157
+ class ExistsExpr(Expr):
158
+ """EXISTS (SELECT 1 FROM <cte> <alias> WHERE <condition>): a test that
159
+ a CTE (a subquery run on its own) has a row matching this one. The
160
+ condition reads the CTE's columns through `alias` and the outer row's
161
+ columns as usual; only the latter are references of this expression."""
162
+ cte_name: str = ''
163
+ alias: str = ''
164
+ condition: Optional[Expr] = None
165
+
166
+ def columns_referenced(self) -> Set['ColumnId']:
167
+ if self.condition is None:
168
+ return set()
169
+ return {c for c in self.condition.columns_referenced()
170
+ if getattr(c, 'scope', None) != self.alias}
171
+
172
+
173
+ # =============================================================================
174
+ # Column References
175
+ # =============================================================================
176
+
177
+ @dataclass(frozen=True)
178
+ class ColumnId:
179
+ """Stable, structured identity of a column, for matching and liveness, not
180
+ rendering.
181
+
182
+ A *base* column is identified by its producing scan alias (``scope``, e.g.
183
+ ``'o'`` or ``'_r1'``) and source column name (``name``, e.g. ``'total_due'``).
184
+ A *derived* column (a projection or aggregate output) has ``scope=None`` and
185
+ is identified by its output ``name`` (alias). Two ColumnRefs denote the same
186
+ column iff their ``ColumnId`` compare equal, replacing the ad-hoc
187
+ string-form matching (``o__total_due`` vs ``o.total_due`` vs
188
+ ``o.`total_due```) that caused wrong column references.
189
+ """
190
+ scope: Optional[str]
191
+ name: str
192
+
193
+
194
+ def identity_from_sql(sql: str, alias: Optional[str] = None) -> ColumnId:
195
+ """Derive a structured ColumnId from a column's ``sql`` (and fallback
196
+ ``alias``). ``sql`` is the form that stays consistent across the colliding
197
+ alias/dot rendering conventions, so a scan definition and a reference to the
198
+ same base column derive the same id. Functions/expressions (parentheses,
199
+ whitespace, commas) are derived columns, keyed by their output alias.
200
+ Used by :meth:`ColumnRef.identity`.
201
+ """
202
+ if '(' not in sql and ' ' not in sql and ',' not in sql:
203
+ # Split scope/column on the first '.' that is OUTSIDE backticks, so a
204
+ # backtick-quoted column name that itself contains dots, e.g.
205
+ # managed-edge FK columns rendered as ``_r0.`Person.id.1```, parses as
206
+ # (``_r0``, ``Person.id.1``) instead of collapsing to the alias. A naive
207
+ # ``replace('`','').partition('.')`` mis-splits those and breaks identity
208
+ # consistency between a scan column and references to it.
209
+ in_backtick = False
210
+ split_at = None
211
+ for i, ch in enumerate(sql):
212
+ if ch == '`':
213
+ in_backtick = not in_backtick
214
+ elif ch == '.' and not in_backtick:
215
+ split_at = i
216
+ break
217
+ if split_at is not None:
218
+ scope = alias_name(sql[:split_at].replace('`', ''))
219
+ name = sql[split_at + 1:].replace('`', '')
220
+ if scope and name:
221
+ return ColumnId(scope=scope, name=name)
222
+ else:
223
+ nb = sql.replace('`', '')
224
+ if nb:
225
+ return ColumnId(scope=None, name=nb)
226
+ return ColumnId(scope=None, name=alias or sql.replace('`', ''))
227
+
228
+
229
+ @dataclass(frozen=True)
230
+ class ColumnRef:
231
+ """A resolved column reference.
232
+
233
+ Language-agnostic: no Cypher-specific origin_var/origin_prop fields.
234
+ The metadata dict carries optional provenance information that
235
+ optimization passes may use, but the core model doesn't depend on it.
236
+
237
+ Examples:
238
+ ColumnRef(sql='p.`firstName`', alias='p.name')
239
+ ColumnRef(sql='COUNT(f.`id`)', alias='friend_count')
240
+ ColumnRef(sql='p.`id`', alias='p__id')
241
+ """
242
+ sql: str
243
+ """The SQL expression (e.g., 'p.`firstName`', 'COUNT(f.`id`)')."""
244
+
245
+ alias: Optional[str] = None
246
+ """The output column alias (e.g., 'p.name', 'friend_count')."""
247
+
248
+ metadata: Optional[Dict[str, Any]] = None
249
+ """Optional provenance tags. Not used by core tree logic.
250
+
251
+ Examples:
252
+ {'var': 'p', 'prop': 'name', 'label': 'Person'} # Cypher
253
+ {'step': 'out', 'traverser_key': '_value'} # Gremlin
254
+ {'subject': '?p', 'predicate': 'foaf:name'} # SPARQL
255
+ """
256
+
257
+ table: Optional[str] = None
258
+ """Optional structured table name for dialect-aware quoting."""
259
+
260
+ column: Optional[str] = None
261
+ """Optional structured column name for dialect-aware quoting."""
262
+
263
+ column_id: Optional['ColumnId'] = None
264
+ """Stable, *structured* identity for matching/liveness, independent of how
265
+ the column is rendered (the colliding ``alias`` / dot / backtick-``sql``
266
+ forms). Set at construction where known; otherwise :meth:`identity` derives
267
+ one. The single identity all liveness/pruning logic should compare on."""
268
+
269
+ def identity(self) -> 'ColumnId':
270
+ """Stable structured identity for *matching* (never for rendering).
271
+
272
+ Prefers an explicit ``column_id``, then the structured
273
+ ``table``/``column`` (a base column), then the output ``alias`` or a
274
+ quoting-normalized ``sql`` (a derived column). Replaces the three
275
+ colliding string forms (``o__total_due`` / ``o.total_due`` /
276
+ ``o.`total_due```) reconciled by hand today (``_all_column_forms``, the
277
+ ``_property_access`` fallback chain).
278
+ """
279
+ if self.column_id is not None:
280
+ return self.column_id
281
+ if self.table and self.column:
282
+ return ColumnId(scope=self.table, name=self.column)
283
+ return identity_from_sql(self.sql, self.alias)
284
+
285
+ # =============================================================================
286
+ # Enums
287
+ # =============================================================================
288
+
289
+ class JoinType(Enum):
290
+ INNER = auto()
291
+ LEFT = auto()
292
+ CROSS = auto()
293
+
294
+
295
+ class SortDirection(Enum):
296
+ ASC = auto()
297
+ DESC = auto()
298
+
299
+
300
+ class NullsOrder(Enum):
301
+ FIRST = auto()
302
+ LAST = auto()
303
+
304
+
305
+ class SetOp(Enum):
306
+ UNION_ALL = auto()
307
+ UNION_DISTINCT = auto()
308
+
309
+
310
+ # =============================================================================
311
+ # Sort key
312
+ # =============================================================================
313
+
314
+ @dataclass(frozen=True)
315
+ class SortKey:
316
+ """A single ORDER BY key.
317
+
318
+ References a ColumnRef from the child's output. When the child is
319
+ a Project, the serializer emits the alias form, no lookup needed.
320
+ """
321
+ column: ColumnRef
322
+ direction: SortDirection = SortDirection.ASC
323
+ nulls: Optional[NullsOrder] = None
324
+ # A text key ordered by code point (D8), written through the dialect's CODEPOINT_TEXT
325
+ # (`lower.codepoint`, rule codepoint-string-order).
326
+ codepoint: bool = False
327
+
328
+
329
+ # =============================================================================
330
+ # Node Types, Abstract Base
331
+ # =============================================================================
332
+
333
+ @dataclass
334
+ class RelNode:
335
+ """Base class for all relational algebra nodes."""
336
+
337
+ # =============================================================================
338
+ # Leaf Nodes
339
+ # =============================================================================
340
+
341
+ @dataclass
342
+ class Scan(RelNode):
343
+ """Read from a base table.
344
+
345
+ Maps to: FROM table AS alias
346
+
347
+ Carries the schema mapping so optimization passes can look up
348
+ property→column mappings, ID columns, composite keys, etc.
349
+ """
350
+ table: str = ''
351
+ """Fully qualified table name (e.g., 'catalog.schema.person')."""
352
+
353
+ alias: str = ''
354
+ """SQL alias for this table (e.g., 'p', '_r0')."""
355
+
356
+ columns: List[ColumnRef] = field(default_factory=list)
357
+ """Columns to read from this table."""
358
+
359
+ # Graph metadata (for optimization passes)
360
+ label: Optional[str] = None
361
+ """Node label (e.g., 'Person'). None for relationship tables."""
362
+
363
+ mapping: Optional[Any] = None
364
+ """NodeMapping or RelationshipMapping from the schema."""
365
+
366
+ is_relationship: bool = False
367
+ """True if this scans a relationship table."""
368
+
369
+
370
+ @dataclass
371
+ class Values(RelNode):
372
+ """Inline literal rows.
373
+
374
+ Maps to: VALUES (v1), (v2), (v3) AS alias(col)
375
+
376
+ Used for:
377
+ - Gremlin inject(): g.inject(1, 2, 3)
378
+ - Standalone UNWIND: UNWIND [1, 2, 3] AS x
379
+ - Dummy tables: (SELECT 1 AS _dummy) AS _base
380
+ """
381
+ rows: List[List[Any]] = field(default_factory=list)
382
+ """Row values. Each inner list is one row."""
383
+
384
+ alias: str = '_base'
385
+ columns: List[ColumnRef] = field(default_factory=list)
386
+
387
+
388
+ @dataclass
389
+ class CteRef(RelNode):
390
+ """Reference to a CTE defined elsewhere in the tree.
391
+
392
+ Maps to: FROM cte_name
393
+ """
394
+ cte_name: str = ''
395
+ columns: List[ColumnRef] = field(default_factory=list)
396
+
397
+ # =============================================================================
398
+ # Unary Nodes (single child)
399
+ # =============================================================================
400
+
401
+ @dataclass
402
+ class Filter(RelNode):
403
+ """Filter rows by a predicate.
404
+
405
+ Maps to: WHERE predicate
406
+
407
+ Predicate is a structured Expr, not a SQL string. This allows
408
+ optimization passes to decompose and push down predicates.
409
+ """
410
+ child: RelNode = None
411
+ predicate: Expr = None
412
+
413
+
414
+ @dataclass
415
+ class Project(RelNode):
416
+ """Select specific columns, optionally with DISTINCT.
417
+
418
+ Maps to: SELECT [DISTINCT] col1 AS alias1, col2 AS alias2, ...
419
+
420
+ This is where RETURN, WITH projections, and Gremlin select()/values()
421
+ land. Sort nodes reference the same ColumnRef objects, so alias
422
+ resolution is automatic.
423
+ """
424
+ child: RelNode = None
425
+ columns: List[ColumnRef] = field(default_factory=list)
426
+ distinct: bool = False
427
+ names_its_columns: bool = False
428
+ """A subquery whose reader already reads it by its own column names
429
+ (`_ThroughASubquery`): the serializer leaves the reader's text as it
430
+ is, where for any other subquery it re-points the reader's columns."""
431
+
432
+
433
+ @dataclass
434
+ class Aggregate(RelNode):
435
+ """Group-by aggregation.
436
+
437
+ Maps to: SELECT group_cols, agg_cols ... GROUP BY group_cols [HAVING ...]
438
+
439
+ Covers:
440
+ - Cypher: WITH p, count(f) AS cnt
441
+ - Gremlin: groupCount(), fold() (full-scan agg with no group-by)
442
+ - SPARQL: GROUP BY with aggregates
443
+ """
444
+ child: RelNode = None
445
+ group_by: List[ColumnRef] = field(default_factory=list)
446
+ """Columns to group by. Empty for full-scan aggregation (fold)."""
447
+
448
+ aggregations: List[ColumnRef] = field(default_factory=list)
449
+ """Aggregation expressions (e.g., COUNT(f.`id`) AS friend_count)."""
450
+
451
+ having: Optional[Expr] = None
452
+ """HAVING predicate."""
453
+
454
+
455
+ @dataclass
456
+ class Sort(RelNode):
457
+ """Order rows.
458
+
459
+ Maps to: ORDER BY key1 ASC, key2 DESC NULLS LAST
460
+
461
+ Sort keys reference ColumnRef objects from the child's output.
462
+ The serializer emits alias form when the child is a Project.
463
+ """
464
+ child: RelNode = None
465
+ keys: List[SortKey] = field(default_factory=list)
466
+
467
+
468
+ @dataclass
469
+ class Limit(RelNode):
470
+ """Limit (and optionally skip) rows.
471
+
472
+ Maps to: LIMIT count OFFSET offset
473
+ """
474
+ child: RelNode = None
475
+ count: Optional[int] = None
476
+ offset: Optional[int] = None
477
+
478
+
479
+ @dataclass
480
+ class LateralExplode(RelNode):
481
+ """Explode a list/array column into rows.
482
+
483
+ Maps to: LATERAL VIEW explode(list_expr) AS element_alias
484
+
485
+ Covers:
486
+ - Cypher UNWIND
487
+ - Gremlin unfold()
488
+ - SPARQL (no direct equivalent)
489
+
490
+ Produces all parent columns plus the new element column.
491
+ """
492
+ child: RelNode = None
493
+ list_expr: Expr = None
494
+ """Expression for the list to explode."""
495
+
496
+ element_alias: str = ''
497
+ """Alias for the exploded element (e.g., 'tag')."""
498
+
499
+ element_column: Optional[ColumnRef] = None
500
+ """ColumnRef for the exploded element."""
501
+
502
+ more: tuple = ()
503
+ """PostgreSQL: further arrays of the same length exploded with it, in step, (expression, alias) each, and
504
+ `ordinal_alias` the element's 1-based position (an element's parts, rule cypher-result-values)."""
505
+
506
+ ordinal_alias: str = ''
507
+
508
+ # =============================================================================
509
+ # Binary Nodes
510
+ # =============================================================================
511
+
512
+ @dataclass
513
+ class Join(RelNode):
514
+ """Join two relations.
515
+
516
+ Maps to: left ... JOIN_TYPE JOIN right ON conditions
517
+
518
+ Covers:
519
+ - Cypher relationship patterns: MATCH (a)-[:KNOWS]->(b)
520
+ - Gremlin traversal steps: out(), in(), both()
521
+ - SPARQL triple patterns: ?s :knows ?o
522
+ - Re-join after CTE boundary
523
+
524
+ Carries graph metadata for optimization (hub elimination).
525
+ """
526
+ left: RelNode = None
527
+ right: RelNode = None
528
+ join_type: JoinType = JoinType.INNER
529
+ on: List[Expr] = field(default_factory=list)
530
+ """Join conditions. Empty list for CROSS JOIN."""
531
+
532
+ # Graph metadata (for optimization passes)
533
+ rel_type: Optional[str] = None
534
+ """Relationship type being traversed (e.g., 'KNOWS')."""
535
+
536
+ rel_scan: Optional[Scan] = None
537
+ """The relationship table Scan, if this join traverses a relationship."""
538
+
539
+
540
+ @dataclass
541
+ class AntiJoin(RelNode):
542
+ """Anti-join: filter rows where a pattern does NOT exist.
543
+
544
+ Maps to: LEFT ANTI JOIN rel ON conditions
545
+
546
+ Modeled as a distinct node type because:
547
+ 1. Native LEFT ANTI JOIN avoids materializing right-side columns
548
+ 2. AntiJoinDeferral pass needs to identify these
549
+ 3. Serialization emits Spark's native LeftAnti physical operator
550
+ """
551
+ child: RelNode = None
552
+ right: Scan = None
553
+ """The relationship table to anti-join against."""
554
+
555
+ on: List[Expr] = field(default_factory=list)
556
+ null_check: Expr = None
557
+ """IS NULL expression on the right side's join column."""
558
+
559
+ # For undirected relationships: second anti-join leg
560
+ second_right: Optional[Scan] = None
561
+ second_on: List[Expr] = field(default_factory=list)
562
+ second_null_check: Optional[Expr] = None
563
+
564
+
565
+ @dataclass
566
+ class SemiJoin(RelNode):
567
+ """Semi-join: filter rows where a matching row EXISTS on the right.
568
+
569
+ Maps to: LEFT SEMI JOIN rel ON conditions
570
+
571
+ Created by the JoinElimination optimization when an INNER JOIN's
572
+ right-side columns are unused downstream, the join serves only as
573
+ an existence check. Spark's native LeftSemi physical operator
574
+ (BroadcastHash or SortMerge) avoids materializing right-side
575
+ columns through the shuffle, unlike INNER JOIN which carries all
576
+ columns from both sides.
577
+ """
578
+ child: RelNode = None
579
+ right: Scan = None
580
+ """The table to semi-join against (existence check)."""
581
+
582
+ on: List[Expr] = field(default_factory=list)
583
+ """Join conditions using structured Expr nodes."""
584
+
585
+ # =============================================================================
586
+ # Set Operations
587
+ # =============================================================================
588
+
589
+ @dataclass
590
+ class SetOperation(RelNode):
591
+ """UNION ALL / UNION DISTINCT of multiple branches.
592
+
593
+ Maps to: (branch1) UNION ALL (branch2) ...
594
+
595
+ Covers:
596
+ - Cypher UNION / UNION ALL
597
+ - Gremlin union()
598
+ - Multi-label node scans: MATCH (m) with no label
599
+ - SPARQL UNION
600
+ """
601
+ branches: List[RelNode] = field(default_factory=list)
602
+ op: SetOp = SetOp.UNION_ALL
603
+ alias: Optional[str] = None
604
+
605
+ # =============================================================================
606
+ # CTE (Common Table Expression)
607
+ # =============================================================================
608
+
609
+ @dataclass
610
+ class WithClause(RelNode):
611
+ """One or more CTEs followed by a body query.
612
+
613
+ Maps to: WITH cte1 AS (...), cte2 AS (...) SELECT ... FROM ...
614
+
615
+ Flat list of CTEs rather than nested CTE nodes. This is closer
616
+ to how SQL actually works and simpler to serialize. Each CTE
617
+ definition can reference earlier CTEs in the list.
618
+ """
619
+ ctes: List[CTEDef] = field(default_factory=list)
620
+ """CTE definitions, in order."""
621
+
622
+ body: RelNode = None
623
+ """The query body that reads from the CTEs."""
624
+
625
+
626
+ @dataclass
627
+ class CTEDef:
628
+ """A single CTE definition within a WithClause."""
629
+ name: str = ''
630
+ definition: RelNode = None
631
+ """The subtree that defines this CTE's contents."""
632
+
633
+ columns: List[ColumnRef] = field(default_factory=list)
634
+ """Output columns of this CTE (for CteRef nodes to reference)."""
635
+
636
+ recursive: bool = False
637
+ """True for recursive CTEs (e.g., variable-length path traversal).
638
+
639
+ When any CTEDef in a WithClause is recursive, the serializer emits
640
+ ``WITH RECURSIVE`` instead of ``WITH``.
641
+ """
642
+
643
+ cycle_columns: List[str] = field(default_factory=list)
644
+ """Column names for CYCLE detection (PostgreSQL 14+).
645
+
646
+ When non-empty, the serializer appends a ``CYCLE col1, col2 SET
647
+ "is_cycle" USING "_path"`` clause to the CTE definition. This
648
+ prevents infinite recursion on cyclic graphs.
649
+ """
650
+
651
+
652
+ # =============================================================================
653
+ # Sub-Plan Nodes (Gremlin branching/scoped operations)
654
+ # =============================================================================
655
+ #
656
+ # These nodes execute a full sub-plan (RelNode subtree) per input row.
657
+ # They map to LATERAL JOIN in SQL, or correlated subqueries.
658
+ #
659
+ # This is the fundamental concept that Gremlin has and Cypher doesn't:
660
+ # per-traverser sub-plan execution. CHOOSE, COALESCE, LOCAL, and
661
+ # Gremlin's OPTIONAL all share this pattern.
662
+ # =============================================================================
663
+
664
+ @dataclass
665
+ class LateralJoin(RelNode):
666
+ """Execute a sub-plan per input row and join the results.
667
+
668
+ Maps to: child CROSS JOIN LATERAL (sub_plan parameterized by child row)
669
+
670
+ This is the SQL equivalent of Gremlin's per-traverser execution model.
671
+ The sub_plan can reference columns from the child (correlated).
672
+
673
+ Covers:
674
+ - Gremlin local(): per-row scoped execution
675
+ - Gremlin optional(): try sub-plan, fallback to input row
676
+ - Correlated subqueries in general
677
+
678
+ For optional semantics (keep input row when sub-plan is empty):
679
+ use join_type=LEFT.
680
+ """
681
+ child: RelNode = None
682
+ sub_plan: RelNode = None
683
+ """Sub-plan to execute per input row. May reference child columns."""
684
+
685
+ join_type: JoinType = JoinType.CROSS
686
+ """CROSS for mandatory, LEFT for optional (fallback to input)."""
687
+
688
+ # =============================================================================
689
+ # Path Tracking (Gremlin path operations)
690
+ # =============================================================================
691
+
692
+ # =============================================================================
693
+ # Constant/Inject
694
+ # =============================================================================
695
+
696
+ # =============================================================================
697
+ # Optimization Pass Interface
698
+ # =============================================================================
699
+
700
+ # =============================================================================
701
+ # Serializer Interface
702
+ # =============================================================================
703
+
704
+ class SQLSerializer:
705
+ """Serialize a RelNode tree to a SQL string.
706
+
707
+ Single read-only tree walk. No mutation, no state accumulation.
708
+ Each node type has a corresponding serialization method.
709
+
710
+ Handles Spark SQL dialect specifics:
711
+ - BROADCAST hints after SELECT
712
+ - DISTINCT before column list
713
+ - LATERAL VIEW wrapping when combined with JOINs
714
+ - WithClause flattening into WITH ... SELECT
715
+ - Alias quoting with backticks
716
+ - LATERAL subqueries for sub-plan nodes
717
+
718
+ Structured Expr predicates are serialized to SQL strings during
719
+ this walk, the tree itself never contains raw SQL strings
720
+ (docs/architecture/rules.md, "SQL text from the tree").
721
+ """
722
+
723
+ def serialize(self, tree: RelNode) -> str:
724
+ """Convert a relational algebra tree to a SQL string."""
725
+ raise NotImplementedError
726
+
727
+
728
+ # =============================================================================
729
+ # Node catalog and SQL translation
730
+ # =============================================================================
731
+ #
732
+ # Node Type Source Languages SQL
733
+ # ──────────────────────────────────────────────────────
734
+ # Scan All trivial
735
+ # Values Cypher, Gremlin trivial
736
+ # CteRef All trivial
737
+ # Filter All trivial
738
+ # Project All trivial
739
+ # Aggregate All standard
740
+ # Sort All standard
741
+ # Limit All standard
742
+ # LateralExplode Cypher, Gremlin LATERAL VIEW EXPLODE
743
+ # Join All standard
744
+ # AntiJoin Cypher LEFT JOIN + IS NULL
745
+ # SetOperation All UNION ALL
746
+ # WithClause All WITH ... AS
747
+ # Constant Gremlin SELECT literal FROM child
748
+ # LateralJoin Gremlin complex: LATERAL subquery (Spark 3.4+)
749
+ #
750
+ # plain entries: straightforward SQL generation
751
+ # complex: feasible but complex; may fall back to the Python executor
752
+
753
+
754
+ def holds_a_subquery_expression(tree: Any) -> bool:
755
+ """Whether a subquery stands inside an expression anywhere in the
756
+ statement `tree` is: an `ExistsExpr`. Wherever it sits: a
757
+ projection, a WHERE, a HAVING, a sort key, a join's condition, a CTE,
758
+ a branch of a UNION. Decided by the kind of the node the serializer
759
+ writes, not by the query language's construct or its clause.
760
+
761
+ The statement is walked whole, every field of every node, column and
762
+ expression, so no place an expression can stand is left out: one
763
+ that this missed would be a sorted slice left to Spark's optimizer
764
+ (docs/architecture/rules.md, "A CTE that holds a slice")."""
765
+ import dataclasses
766
+ seen, stack = {}, [tree] # id -> the object, held
767
+ while stack:
768
+ value = stack.pop()
769
+ if isinstance(value, (list, tuple, set, frozenset)):
770
+ stack.extend(value)
771
+ continue
772
+ if isinstance(value, dict):
773
+ stack.extend(value.values())
774
+ continue
775
+ if not dataclasses.is_dataclass(value) or isinstance(value, type) or id(value) in seen:
776
+ continue
777
+ seen[id(value)] = value
778
+ if isinstance(value, ExistsExpr):
779
+ return True
780
+ stack.extend(getattr(value, f.name, None) for f in dataclasses.fields(value))
781
+ return False