prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,935 @@
1
+ """Cypher's lists on the one plan (rule cypher-lists; designs 41-cypher-lists, 41-cypher-mixed-values): a
2
+ list is the engine's array (DuckDB LIST, Spark and Databricks ARRAY, PostgreSQL arrays) of one element
3
+ type, elements possibly null; its slot `slot.list_slot`, the element type recursive (`list_type`,
4
+ `cypher_maps.map_type`): lists of lists and of maps nest natively. A list of several element types
5
+ (`[1, 2.0]` among them: 1 is no 1.0) is built only where it is a literal read element by element; a list
6
+ of nodes or relationships is an array of their values (`cypher_answer.element_type`).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import itertools
12
+ from dataclasses import replace
13
+ from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional
14
+
15
+ from prograph.plan.ir import expr as E
16
+ from prograph.plan.sql.dialect import COLLECT_ORDERED_BY, COLLECT_ORDERED_MAX_KEYS
17
+ from prograph.plan.ir.facts import NotBuilt, expr_slot
18
+ from prograph.plan.sql import rel_algebra as R
19
+ from prograph.plan.ir.slot import LIST, LITERAL, MAP, MIXED, NOTHING, UNKNOWN, VALUE, Bound, Slot, list_slot, value_slot
20
+ from prograph.rules import module_home
21
+ from prograph.schema import xsd
22
+
23
+ if TYPE_CHECKING:
24
+ from prograph.plan.ir import lower as L
25
+
26
+ module_home("cypher-lists", "Cypher's lists, in SQL",
27
+ terms=("list", "list literal", "size", "collect", "index", "slice", "engine array", "Cypher list",
28
+ "reverse"))
29
+
30
+ LIST_CALLS = frozenset({"LIST", "SIZE", "INDEX", "SLICE", "SLICE_FROM", "SLICE_TO", "SLICE_FROM_TO", "IN",
31
+ "RANGE", "ALL", "ANY", "NONE", "SINGLE", "COMPREHENSION", "REVERSE"})
32
+ QUANTIFIERS = ("ALL", "ANY", "NONE", "SINGLE")
33
+
34
+
35
+ LIST_OF = "urn:prograph:cypher:list:"
36
+ """A list's element type when its elements are lists: this prefix and theirs (empty: of no type)."""
37
+
38
+
39
+ def list_type(element: Optional[str]) -> str:
40
+ """The element type of a list whose elements are lists of `element` (None: of no type yet)."""
41
+ return LIST_OF + (element or "")
42
+
43
+
44
+ def column_list_type(kind: Any) -> Optional[str]:
45
+ """The Cypher value type (`list_type`) of a column whose recorded type (`GraphSchema.column_type`) is the
46
+ list type `kind`: an engine array is a Cypher list of its element type, a list of lists nested; None for
47
+ a column of no list type or of elements of no known type."""
48
+ from prograph.schema import xsd
49
+ if getattr(kind, "kind", None) != "list":
50
+ return None
51
+ element = getattr(kind, "element", None)
52
+ inner = column_list_type(element) if getattr(element, "kind", None) == "list" else xsd.datatype_of_type(element)
53
+ return None if inner is None else list_type(inner)
54
+
55
+
56
+ def nested(datatype: Optional[str]) -> bool:
57
+ """Whether a list's element type is itself a list's."""
58
+ return datatype is not None and datatype.startswith(LIST_OF)
59
+
60
+
61
+ def element_of(datatype: str) -> Optional[str]:
62
+ """The element type of the lists a `list_type` names."""
63
+ return datatype[len(LIST_OF):] or None
64
+
65
+
66
+ def joined(a: Optional[str], b: Optional[str]) -> Optional[str]:
67
+ """Two element types as one: equal ones, or a list of no type yet with any list; None where they
68
+ differ (a mixed list)."""
69
+ if a == b:
70
+ return a
71
+ from prograph.plan.ir.cypher_answer import element_base, joined_elements
72
+ if element_base(a) is not None and element_base(b) is not None: # nodes or relationships
73
+ return joined_elements(a, b)
74
+ from prograph.plan.ir.cypher_answer import joined_paths, path_parts
75
+ if path_parts(a) is not None and path_parts(b) is not None: # paths
76
+ return joined_paths(a, b)
77
+ from prograph.plan.ir.cypher_maps import is_map_type, map_type, type_fields
78
+ if is_map_type(a) and is_map_type(b):
79
+ fa, fb = type_fields(a), type_fields(b)
80
+ if set(fa) != set(fb):
81
+ return None
82
+ out = {}
83
+ for k in fa:
84
+ if fa[k] is None or fb[k] is None:
85
+ out[k] = fa[k] or fb[k]
86
+ else:
87
+ out[k] = joined(fa[k], fb[k])
88
+ if out[k] is None:
89
+ return None
90
+ return map_type(out)
91
+ if nested(a) and nested(b):
92
+ ea, eb = element_of(a), element_of(b)
93
+ if ea is None or eb is None:
94
+ return list_type(ea or eb)
95
+ inner = joined(ea, eb)
96
+ return None if inner is None else list_type(inner)
97
+ return None
98
+
99
+
100
+ def native_nesting(datatype: Optional[str]) -> None:
101
+ """A list of lists is the engine's nested array (DuckDB, Spark, Databricks). PostgreSQL's arrays are
102
+ rectangular (`[[1, 2], [3]]` has no int[][]); its lists of lists, held as jsonb (design
103
+ 41-cypher-mixed-values), are not built yet."""
104
+ from prograph.plan.ir.lower import DIALECT
105
+ from prograph.plan.ir.cypher_mixed import engine, has_mixed
106
+ if has_mixed(datatype):
107
+ engine()
108
+ if nested(datatype) and DIALECT.get() in ("postgresql", "postgres"):
109
+ raise NotBuilt("a list of lists on PostgreSQL (its arrays are rectangular)")
110
+
111
+
112
+ def element_slot(lst: Slot) -> Slot:
113
+ """The slot of a list's elements (UNWIND's, an index's, a lambda variable's)."""
114
+ return _element(lst)
115
+
116
+
117
+ def _element(lst: Slot) -> Slot:
118
+ """The slot of a list's elements: its type, possibly null."""
119
+ if not lst.datatypes:
120
+ return NOTHING
121
+ return typed_slot(lst.datatype, Bound.MAYBE)
122
+
123
+
124
+ def typed_slot(datatype: str, null: Bound) -> Slot:
125
+ """The slot of a Cypher value of a `value_type`: a list's, a map's or a plain value's."""
126
+ if nested(datatype):
127
+ return list_slot(element_of(datatype), null)
128
+ from prograph.plan.ir.cypher_mixed import is_mixed, mixed_slot, tags_of
129
+ if is_mixed(datatype):
130
+ return mixed_slot(tags_of(datatype), null)
131
+ from prograph.plan.ir.cypher_maps import is_map_type, map_slot, type_fields
132
+ if is_map_type(datatype):
133
+ return map_slot(type_fields(datatype), null)
134
+ from prograph.plan.ir.cypher_answer import element_base, element_slot
135
+ if element_base(datatype) is not None: # a node or relationship value (rule cypher-result-values)
136
+ return element_slot(datatype, null)
137
+ return value_slot(datatype, null)
138
+ _INTEGERS = frozenset({xsd.INTEGER}) | xsd.INTEGER_DATATYPES
139
+
140
+
141
+ def _integer(s: Slot) -> bool:
142
+ return s.present is Bound.NEVER or (s.kinds == {VALUE} and len(s.datatypes) == 1 and
143
+ s.datatype in _INTEGERS)
144
+
145
+
146
+ def _list(s: Slot, what: str, arg: int = 0) -> None:
147
+ """The slot is a list (or null); a value known to be none is an error where a row holds it."""
148
+ if s.present is not Bound.NEVER and s.kinds != {LIST}:
149
+ from prograph.plan.ir.cypher_runtime_errors import RuntimeTypeError, known_kind
150
+ if known_kind(s):
151
+ raise RuntimeTypeError(arg, "InvalidArgumentType", f"{what} of a value that is not a list")
152
+ raise NotBuilt(f"{what} of a value that is not a list")
153
+
154
+
155
+ def _not_integer(s: Slot, what: str, arg: int) -> None:
156
+ """The slot is an integer or null; a value known to be of another kind is an error where a row holds it."""
157
+ if not _integer(s):
158
+ from prograph.plan.ir.cypher_runtime_errors import RuntimeTypeError, known_kind
159
+ if known_kind(s):
160
+ raise RuntimeTypeError(arg, "InvalidArgumentType", f"{what} that is not an integer")
161
+ raise NotBuilt(f"{what} that is not an integer")
162
+
163
+
164
+ def concat_slot(a: Slot, b: Slot) -> Slot:
165
+ """`+` with a list: list + list, list + element, element + list; null where an operand is null."""
166
+ if a.present is Bound.NEVER or b.present is Bound.NEVER:
167
+ return NOTHING
168
+ sides = []
169
+ for s in (a, b):
170
+ if s.kinds == {LIST}:
171
+ sides.append(s.datatype if s.datatypes else None)
172
+ else:
173
+ sides.append(element_type([s], "a list's element", ))
174
+ types = [t for t in sides if t is not None]
175
+ if len(types) > 1:
176
+ from prograph.plan.ir.cypher_mixed import widened
177
+ both = widened(*types)
178
+ if both is None:
179
+ raise NotBuilt("+ of a list and values of another type (a list or map among scalars)")
180
+ return list_slot(both, present([a, b]))
181
+ return list_slot(types[0] if types else None, present([a, b]))
182
+
183
+
184
+ def element_type(slots: List[Slot], what: str) -> Optional[str]:
185
+ """The one type of `slots`' values, None where every one is null; NotBuilt for several."""
186
+ out: Optional[str] = None
187
+ seen = False
188
+ for s in slots:
189
+ if s.present is Bound.NEVER:
190
+ continue
191
+ t = value_type(s)
192
+ if t is None:
193
+ raise NotBuilt(f"{what} of a value that is not one plain type (a list, map, node or mixed)")
194
+ if seen and joined(out, t) is None:
195
+ raise NotBuilt(f"{what} of values of several types (a mixed list)")
196
+ out, seen = (joined(out, t) if seen else t), True
197
+ return out
198
+
199
+
200
+ def value_type(s: Slot) -> Optional[str]:
201
+ """The one type of a Cypher value's slot, None where it has no one plain type. A condition
202
+ (`a = b`, `x IS NULL`) is a boolean value: its cell the engine's boolean, null where unknown. A list
203
+ is a value of its `list_type`."""
204
+ if s.kinds == {LIST}:
205
+ return list_type(s.datatype if s.datatypes else None)
206
+ if s.kinds == {MIXED}:
207
+ from prograph.plan.ir.cypher_mixed import mixed_type, slot_tags
208
+ return mixed_type(slot_tags(s))
209
+ if s.kinds == {MAP}:
210
+ from prograph.plan.ir.cypher_maps import fields_of, map_type
211
+ return map_type(fields_of(s))
212
+ if s.kinds == {VALUE} and len(s.datatypes) == 1 and UNKNOWN not in s.datatypes:
213
+ return s.datatype
214
+ if s.kinds == {LITERAL} and s.datatypes == {xsd.BOOLEAN} and s.native == xsd.BOOLEAN:
215
+ return xsd.BOOLEAN
216
+ return None
217
+
218
+
219
+ def present(slots: List[Slot]) -> Bound:
220
+ if any(s.present is Bound.NEVER for s in slots):
221
+ return Bound.ALWAYS # null: no value
222
+ return Bound.NEVER if all(s.present is Bound.ALWAYS for s in slots) else Bound.MAYBE
223
+
224
+
225
+ def _substituted(x, name: str, value: E.Expr):
226
+ """`x` with the lambda variable `name` read as `value` (the front's lambda names are fresh)."""
227
+ import dataclasses
228
+ if isinstance(x, E.Col):
229
+ return value if x.name == name else x
230
+ if isinstance(x, tuple):
231
+ return tuple(_substituted(y, name, value) for y in x)
232
+ if isinstance(x, E.Expr) and dataclasses.is_dataclass(x):
233
+ return dataclasses.replace(x, **{f.name: _substituted(getattr(x, f.name), name, value)
234
+ for f in dataclasses.fields(x)})
235
+ return x
236
+
237
+
238
+ def _unrolled(e: E.Call, slots: Mapping[str, Slot]) -> Optional[E.Expr]:
239
+ """A quantifier or a comprehension over a list literal whose elements are of several types (`[1,
240
+ 'a', null]`), each element's condition or result worked out on its own: `q(x IN [e1, e2] WHERE p)` is
241
+ `q(x IN [p(e1), p(e2)] WHERE x)`, `[x IN [e1, e2] | m]` is `[m(e1), m(e2)]` (openCypher evaluates them
242
+ element by element; a literal's elements are known at planning). None where the list is of one type
243
+ or no literal; a list of several types read any other way is not built (design 41-cypher-mixed-values)."""
244
+ lst = e.args[0]
245
+ if not (isinstance(lst, E.Call) and lst.name == "LIST"):
246
+ return None
247
+ try:
248
+ element_type([expr_slot(a, slots) for a in lst.args], "a list")
249
+ return None
250
+ except NotBuilt:
251
+ pass
252
+ from prograph.plan.ir.fresh import holds
253
+ if any(holds(x) for x in e.args[2:]): # each element's copy of rand() would be one draw
254
+ raise NotBuilt("rand() over a list literal of several types")
255
+ var = e.args[1].value
256
+ if e.name in QUANTIFIERS:
257
+ tests = tuple(_substituted(e.args[2], var, a) for a in lst.args)
258
+ if any(value_type(t) != xsd.BOOLEAN for t in (expr_slot(x, slots) for x in tests)
259
+ if t.present is not Bound.NEVER):
260
+ raise NotBuilt(f"{e.name.lower()}() whose test is not a boolean (a type error)")
261
+ if all(expr_slot(x, slots).present is Bound.NEVER for x in tests):
262
+ return E.Const(None) # every element's test null: any, all, none and single are null
263
+ return E.Call(e.name, (E.Call("LIST", tests, E.CYPHER), e.args[1], E.Col(var)), E.CYPHER)
264
+ if isinstance(e.args[2], E.Const) and e.args[2].value is True:
265
+ return E.Call("LIST", tuple(_substituted(e.args[3], var, a) for a in lst.args), E.CYPHER)
266
+ return None
267
+
268
+
269
+ def _map_key(e: E.Call, slots: Mapping[str, Slot]) -> Optional[E.Expr]:
270
+ """`m['k']` of a map with a key written as a string literal: `m.k` (openCypher 9, dynamic property
271
+ access; rule cypher-maps); null where the map is null (`null['k']`) or the key is (`m[null]`)."""
272
+ if e.name != "INDEX" or not isinstance(e.args[1], E.Const):
273
+ return None
274
+ key, base = e.args[1].value, expr_slot(e.args[0], slots)
275
+ if isinstance(key, str) and base.present is Bound.NEVER or key is None and base.kinds == {MAP}:
276
+ return E.Const(None)
277
+ if not isinstance(key, str) or base.kinds != {MAP}:
278
+ return None
279
+ return E.Call("FIELD", e.args, E.CYPHER)
280
+
281
+
282
+ def list_call_slot(e: E.Call, slots: Mapping[str, Slot]) -> Slot:
283
+ keyed = _map_key(e, slots)
284
+ if keyed is not None:
285
+ return expr_slot(keyed, slots)
286
+ if e.name in QUANTIFIERS or e.name == "COMPREHENSION":
287
+ unrolled = _unrolled(e, slots)
288
+ if unrolled is not None:
289
+ return expr_slot(unrolled, slots)
290
+ if e.name in QUANTIFIERS or e.name == "COMPREHENSION":
291
+ lst = expr_slot(e.args[0], slots)
292
+ _list(lst, e.name.lower())
293
+ inner = {**slots, e.args[1].value: _element(lst)}
294
+ expr_slot(e.args[2], inner)
295
+ if e.name in QUANTIFIERS:
296
+ from prograph.plan.ir.slot import BOOLEAN
297
+ return BOOLEAN
298
+ if lst.present is Bound.NEVER:
299
+ return NOTHING
300
+ mapped = expr_slot(e.args[3], inner)
301
+ return list_slot(element_type([mapped], "a list comprehension") if lst.datatypes else None,
302
+ present([lst]))
303
+ args = [expr_slot(a, slots) for a in e.args]
304
+ if e.name == "LIST":
305
+ from prograph.plan.ir.cypher_mixed import element_type as mixed_element_type
306
+ return list_slot(mixed_element_type(args, "a list"), Bound.NEVER)
307
+ if e.name == "INDEX":
308
+ lst, i = args
309
+ if lst.kinds == {MAP} and lst.present is not Bound.NEVER and i.present is not Bound.NEVER:
310
+ if i.kinds == {VALUE} and i.datatype is not None and i.datatype != xsd.STRING:
311
+ from prograph.plan.ir.cypher_runtime_errors import RuntimeTypeError
312
+ raise RuntimeTypeError(1, "MapElementAccessByNonString", "a map indexed by a value that is not a string")
313
+ raise NotBuilt("a map indexed by a key read from a value")
314
+ _list(lst, "an index")
315
+ _not_integer(i, "an index", 1)
316
+ if lst.present is Bound.NEVER or i.present is Bound.NEVER or not lst.datatypes:
317
+ return NOTHING
318
+ return _element(lst)
319
+ if e.name.startswith("SLICE"):
320
+ lst, bounds = args[0], args[1:]
321
+ _list(lst, "a slice")
322
+ for n, b in enumerate(bounds, 1):
323
+ _not_integer(b, "a slice bound", n)
324
+ if lst.present is Bound.NEVER or any(b.present is Bound.NEVER for b in bounds):
325
+ return NOTHING
326
+ return list_slot(lst.datatype if lst.datatypes else None, present([lst, *bounds]))
327
+ if e.name == "IN":
328
+ from prograph.plan.ir.slot import BOOLEAN
329
+ _list(args[1], "IN", 1)
330
+ return BOOLEAN
331
+ if e.name == "RANGE":
332
+ for n, a in enumerate(args):
333
+ _not_integer(a, "range() of a value", n)
334
+ if any(a.present is Bound.NEVER for a in args):
335
+ raise NotBuilt("range() of null") # openCypher: an error
336
+ return list_slot(xsd.INTEGER, present(args))
337
+ if e.name == "REVERSE": # openCypher 9: a list's elements or a string's characters, last first
338
+ (arg,) = args
339
+ if arg.present is Bound.NEVER:
340
+ return NOTHING
341
+ if arg.kinds == {LIST}:
342
+ return arg
343
+ if arg.kinds == {VALUE} and arg.datatypes == {xsd.STRING}:
344
+ return value_slot(xsd.STRING, present([arg]))
345
+ from prograph.plan.ir.cypher_runtime_errors import RuntimeTypeError, known_kind
346
+ if known_kind(arg):
347
+ raise RuntimeTypeError(0, "InvalidArgumentType", "reverse() of a value that is not a list or a string")
348
+ raise NotBuilt("reverse() of a value that is not a list or a string")
349
+ if e.name == "SIZE":
350
+ (arg,) = args
351
+ if arg.present is Bound.NEVER:
352
+ return NOTHING
353
+ if arg.kinds == {LIST}:
354
+ return value_slot(xsd.INTEGER, present([arg]))
355
+ if arg.kinds == {VALUE} and arg.datatypes == {xsd.STRING}:
356
+ return value_slot(xsd.INTEGER, present([arg]))
357
+ from prograph.plan.ir.cypher_runtime_errors import RuntimeTypeError, known_kind
358
+ if known_kind(arg):
359
+ raise RuntimeTypeError(0, "InvalidArgumentType", "size() of a value that is not a list or a string")
360
+ raise NotBuilt("size() of a value that is not a list or a string")
361
+ raise NotBuilt(f"the list function {e.name}")
362
+
363
+
364
+ def list_call_cell(e: E.Call, cells: Dict[str, "L.Cell"]) -> "L.Cell":
365
+ """A list literal: the engine's array of its elements, in order, nulls kept; size(): a list's
366
+ length, a string's in characters; null where its argument is null."""
367
+ from prograph.plan.ir import lower as L
368
+ if e.name == "LIST":
369
+ # Each element lowered once and the list's slot from theirs (a slot worked out again at each
370
+ # level of a nested literal is exponential in its depth).
371
+ elements = [L.expr_cell(a, cells) for a in e.args]
372
+ from prograph.plan.ir import cypher_mixed as M
373
+ slot = list_slot(M.element_type([c.slot for c in elements], "a list"), Bound.NEVER)
374
+ target = slot.datatype if slot.datatypes else None
375
+ native_nesting(target)
376
+ return L.Cell(slot, L.fn("ARRAY", *(M.converted_cell(c, target, L) for c in elements)),
377
+ parts=_literal_parts(elements, slot, L))
378
+ keyed = _map_key(e, {n: c.slot for n, c in cells.items()})
379
+ if keyed is not None:
380
+ return L.expr_cell(keyed, cells)
381
+ slot = list_call_slot(e, {n: c.slot for n, c in cells.items()})
382
+ if e.name in QUANTIFIERS or e.name == "COMPREHENSION":
383
+ unrolled = _unrolled(e, {n: c.slot for n, c in cells.items()})
384
+ if unrolled is not None:
385
+ return L.expr_cell(unrolled, cells)
386
+ if slot.kinds == {LIST}:
387
+ native_nesting(slot.datatype if slot.datatypes else None)
388
+ return L.Cell(slot, _over_elements(e, cells, L))
389
+ if slot.present is Bound.NEVER:
390
+ return L.nothing()
391
+ args = [L.expr_cell(a, cells) for a in e.args]
392
+ if e.name == "LIST":
393
+ native_nesting(slot.datatype if slot.datatypes else None)
394
+ return L.Cell(slot, L.fn("ARRAY", *(a.value for a in args)))
395
+ if e.name == "INDEX":
396
+ lst, i = (a.value for a in args)
397
+ n = L.fn("SIZE", lst)
398
+ at = L.case(_lt(i, L.integer(0)), L.fn("+", n, i), i)
399
+ inside = L.all_of([_ge(at, L.integer(0)), _lt(at, n)])
400
+ cell = L.Cell(slot, L.case(inside, L.fn("CYPHER_ELEMENT_AT", lst, L.fn("+", at, L.integer(1)))))
401
+ from prograph.plan.ir.cypher_answer import element_at, element_shape
402
+ listed = args[0]
403
+ shape = element_shape(listed.slot, L) if listed.slot.present is not Bound.NEVER else None
404
+ if shape is not None and shape.many and listed.parts and shape.kind in ("node", "rel", "props"):
405
+ # An element taken out of a list of elements holds its parts (PostgreSQL's answer).
406
+ from dataclasses import replace
407
+ position = L.fn("+", at, L.integer(1))
408
+ return replace(cell, parts=tuple(L.case(inside, x) for x in element_at(listed, position, L)))
409
+ return cell
410
+ if e.name.startswith("SLICE"):
411
+ lst = args[0].value
412
+ n = L.fn("SIZE", lst)
413
+ rest = [a.value for a in args[1:]]
414
+ start = _clamped(rest.pop(0), n, L) if "_FROM" in e.name else L.integer(0)
415
+ end = _clamped(rest.pop(0), n, L) if "_TO" in e.name else n
416
+ length = L.fn("GREATEST", L.fn("-", end, start), L.integer(0))
417
+ out = L.fn("SLICE", lst, L.fn("+", start, L.integer(1)), length)
418
+ nulls = [L.is_null(a.value) for a in args[1:]] # a null bound: null
419
+ return L.Cell(slot, L.case(L.any_of(nulls), L.NULL, out) if nulls else out)
420
+ if e.name == "IN":
421
+ return L.Cell(slot, _within(args[0], args[1], L))
422
+ if e.name == "RANGE":
423
+ return L.Cell(slot, L.fn("CYPHER_RANGE", *(a.value for a in args)))
424
+ (arg,) = args
425
+ if e.name == "REVERSE":
426
+ return L.Cell(slot, L.fn("LIST_REVERSE" if arg.slot.kinds == {LIST} else "STR_REVERSE", arg.value))
427
+ if arg.slot.kinds == {LIST}:
428
+ return L.Cell(slot, L.fn("SIZE", arg.value))
429
+ return L.Cell(slot, L.fn("STR_LEN", arg.value))
430
+
431
+
432
+ def _literal_parts(elements, slot: Slot, L) -> tuple:
433
+ """The parts of a list literal of nodes or relationships that each hold theirs (PostgreSQL's answer,
434
+ rule cypher-result-values): each part an array of the elements'; none where one element holds none or
435
+ holds them in another layout than the list's (the list is refused where it is answered); a null element null
436
+ parts."""
437
+ from prograph.plan.ir.cypher_answer import element_shape, list_parts
438
+ shape = element_shape(slot, L)
439
+ if shape is None or not shape.many or not elements:
440
+ return ()
441
+ flat = shape.flat()
442
+ if shape.kind == "map":
443
+ # Maps of one key set whose fields' types may differ in a null field: each part by key, null where an
444
+ # element's field is null.
445
+ from prograph.plan.ir.cypher_answer import map_list_parts
446
+ return map_list_parts(elements, shape, L)
447
+ # A null element (`[n, null, n]`): null parts, so a null id, which the reader makes a null element.
448
+ nulls = tuple(typed_empty(xsd.STRING, L) if p.role in ("labels", "slots") else L.NULL for p in flat.parts(True))
449
+ elements = [replace(c, parts=nulls) if c.slot.present is Bound.NEVER else c for c in elements]
450
+ if any(not c.parts or (c.parts is not nulls and element_shape(c.slot, L) != flat) for c in elements):
451
+ return ()
452
+ return list_parts(elements, shape, L)
453
+
454
+
455
+ def collect_slot(arg: Slot) -> Slot:
456
+ """collect(x): the list of the group's non-null values, [] over none."""
457
+ return list_slot(element_type([arg], "collect()"), Bound.NEVER)
458
+
459
+
460
+ def collect_value(arg: "L.Cell", order=(), distinct: bool = False) -> "R.Expr":
461
+ """collect(x): the engine's array aggregate of the non-null values; in the order of the WITH just
462
+ before it where it ordered: the dialects' CYPHER_COLLECT_ORDERED[_DESC] for one key and
463
+ CYPHER_COLLECT_ORDERED_BY_<directions> for several, null keys last ascending and first descending
464
+ (cypher-order), strings by code point."""
465
+ from prograph.plan.ir import lower as L
466
+ native_nesting(value_type(arg.slot))
467
+ if distinct:
468
+ from prograph.plan.ir.cypher_mixed import refuse_identity
469
+ refuse_identity(arg.slot, "collect(DISTINCT ...)")
470
+ from prograph.plan.ir.cypher_answer import holds_elements
471
+ if holds_elements(arg.slot) and L.DIALECT.get() in L.LEVEL_DIALECTS:
472
+ raise NotBuilt("collect(DISTINCT ...) of nodes or relationships on Spark or Databricks (their set "
473
+ "aggregate takes no map)")
474
+ if arg.slot.present is Bound.NEVER:
475
+ # Still an aggregate, so a query of no rows has its one row: [].
476
+ return L.case(R.CompareExpr(left=L.fn("COUNT", L.integer(1)), op=">=", right=L.integer(0)), L.fn("ARRAY"))
477
+ # The engine's array aggregate skips nulls (`COLLECT_LIST`); over no value it is null, so [] typed
478
+ # as the elements (PostgreSQL's untyped ARRAY[] is text[]).
479
+ if order and distinct:
480
+ raise NotBuilt("collect(DISTINCT ...) in the order of a WITH")
481
+ if order:
482
+ keys = collect_keys(order, L)
483
+ if len(order) == 1:
484
+ name = "CYPHER_COLLECT_ORDERED_DESC" if order[0][1] else "CYPHER_COLLECT_ORDERED"
485
+ elif len(order) <= COLLECT_ORDERED_MAX_KEYS:
486
+ # Several keys: each in its direction, ties broken by the next (`dialect._collect_ordered_by`).
487
+ name = COLLECT_ORDERED_BY + "".join("D" if desc else "A" for _, desc in order)
488
+ else:
489
+ raise NotBuilt(f"collect() in the order of more than {COLLECT_ORDERED_MAX_KEYS} keys")
490
+ listed = L.fn(name, arg.value, *keys)
491
+ return L.fn("COALESCE", listed, empty_of(arg.slot, L))
492
+ empty = empty_of(arg.slot, L)
493
+ # DISTINCT: each value once (the engines' set aggregate, nulls skipped).
494
+ return L.fn("COALESCE", L.fn("COLLECT_SET" if distinct else "COLLECT_LIST", arg.value), empty)
495
+
496
+
497
+ def collect_keys(order, L) -> list:
498
+ """The SQL keys of a collect's order (`order`: (key cell, descending) each), as the aggregate sorts by them:
499
+ a string by code point, a list or mixed value by its orderability key (`cypher_mixed.sort_key`; TCK
500
+ WithOrderBy1 [45]); a key of no one plain kind is refused."""
501
+ from prograph.plan.ir.lower import _cypher_family
502
+ keys = []
503
+ for key, desc in order:
504
+ family = _cypher_family(key.slot)
505
+ from prograph.plan.ir.cypher_mixed import needs_key, sort_key
506
+ if family is None and needs_key(key.slot):
507
+ keys.append(sort_key(key))
508
+ continue
509
+ if family is None or family in ("urn:prograph:cypher:time", "urn:prograph:cypher:datetime",
510
+ "urn:prograph:cypher:duration"):
511
+ raise NotBuilt("collect() ordered by a key of no one plain kind")
512
+ keys.append(L.codepoint(key.value) if family == "string" else key.value)
513
+ return keys
514
+
515
+
516
+ def collect_cell(agg: E.Agg, cells: Dict[str, "L.Cell"], slot: Slot) -> "L.Cell":
517
+ """collect(x) as a cell. Where x is a node or relationship that holds its own parts (PostgreSQL's answer,
518
+ rule cypher-result-values) the list holds its parts too, one aggregate each in ONE order: the WITH's keys,
519
+ then the element's id, so a tie falls the same way in every part and in the jsonb value."""
520
+ from prograph.plan.ir import lower as L
521
+ from prograph.plan.ir.cypher_answer import collected_parts, element_shape
522
+ if L.DIALECT.get() not in ("postgresql", "postgres"):
523
+ return L.Cell(slot, L.cypher_aggregate(agg, cells, slot))
524
+ arg = L.expr_cell(agg.arg, cells)
525
+ shape = element_shape(arg.slot, L) if arg.slot.present is not Bound.NEVER else None
526
+ if shape is None or shape.many or shape.kind not in ("node", "rel", "props", "map", "path") or not arg.parts:
527
+ return L.Cell(slot, L.cypher_aggregate(agg, cells, slot))
528
+ from prograph.plan.ir.cypher_answer import PartsRefused
529
+ if shape.kind == "map" and any(p.depth > 0 for p in shape.parts(True)):
530
+ raise PartsRefused("collect() of maps holding a list, on PostgreSQL (the lists would be ragged arrays)")
531
+ if agg.distinct:
532
+ raise PartsRefused("collect(DISTINCT ...) of nodes or relationships, on PostgreSQL")
533
+ if len(agg.order) + 1 > COLLECT_ORDERED_MAX_KEYS:
534
+ raise PartsRefused(f"collect() of nodes or relationships in the order of more than "
535
+ f"{COLLECT_ORDERED_MAX_KEYS - 1} keys, on PostgreSQL")
536
+ from prograph.plan.ir.slot import value_slot
537
+ order = [(L.expr_cell(k, cells), d) for k, d in agg.order] + [(L.Cell(value_slot(xsd.INTEGER if shape.kind == "map" else xsd.STRING, Bound.NEVER),
538
+ L.fn("CYPHER_ARRAY_KEY", arg.parts[0]) if shape.kind == "path" else arg.parts[0]), False)]
539
+ listed = collect_value(arg, order)
540
+ keys = collect_keys(order, L)
541
+ return L.Cell(slot, listed, parts=collected_parts(arg, keys, [d for _, d in order], L))
542
+
543
+
544
+ def _lt(a, b):
545
+ from prograph.plan.sql import rel_algebra as R
546
+ return R.CompareExpr(left=a, op="<", right=b)
547
+
548
+
549
+ def _ge(a, b):
550
+ from prograph.plan.sql import rel_algebra as R
551
+ return R.CompareExpr(left=a, op=">=", right=b)
552
+
553
+
554
+ def _clamped(bound, n, L):
555
+ """A slice bound as a 0-based position within [0, n]: negative from the end."""
556
+ return L.case(_lt(bound, L.integer(0)), L.fn("GREATEST", L.fn("+", n, bound), L.integer(0)),
557
+ L.fn("LEAST", bound, n))
558
+
559
+
560
+ def _within(x: "L.Cell", lst: "L.Cell", L) -> "R.Expr":
561
+ """`x IN list` (openCypher 9): null for a null list, false for the empty one; true where an element
562
+ equals x; else null where x or an element is null, else false."""
563
+ from prograph.plan.sql import rel_algebra as R
564
+ if lst.slot.present is Bound.NEVER:
565
+ return L.NULL_BOOLEAN
566
+ l, n = lst.value, L.fn("SIZE", lst.value)
567
+ var = R.LitExpr(value="_el", sql="_el")
568
+ has_null = R.CompareExpr(left=n, op="<>", right=L.fn("SIZE", L.fn("LIST_FILTER", l, var, L.is_null(var, negated=True))))
569
+ empty = R.CompareExpr(left=n, op="=", right=L.integer(0))
570
+ if x.slot.present is Bound.NEVER:
571
+ return L.case(L.is_null(l), L.NULL_BOOLEAN, empty, L.FALSE, L.NULL_BOOLEAN)
572
+ element = lst.slot.datatype if lst.slot.datatypes else None
573
+ xt = value_type(x.slot)
574
+ from prograph.plan.ir.cypher_answer import element_base, identity_text
575
+ from prograph.schema.source_map import BLANK, IRI
576
+ held = element_base(element) is not None
577
+ if held and identity_text(x, L) is None or not held and (element_base(xt) is not None or x.slot.kinds & {IRI, BLANK}):
578
+ raise NotBuilt("IN of a node or relationship against a list of other values (or of a value against a "
579
+ "list of nodes or relationships)")
580
+ from prograph.plan.ir.cypher_maps import is_map_type
581
+ from prograph.plan.ir.cypher_mixed import is_mixed
582
+ if held or nested(element) or nested(xt) or is_map_type(element) or is_map_type(xt) or is_mixed(element) or \
583
+ is_mixed(xt):
584
+ # A node or relationship in a list of them: each element by identity (`cypher_answer.held_equality`).
585
+ # A list or map among lists or maps, or one against values: each element by Cypher's `=` (true where one
586
+ # is equal; else null where one is null; else false).
587
+ e = R.LitExpr(value="_el", sql=f"_el{next(_LAMBDAS)}")
588
+ found, unknown = _counts(l, e, L.cypher_compare("=", x, L.Cell(_element(lst.slot), e)), L)
589
+ zero = L.integer(0)
590
+ return L.case(L.is_null(l), L.NULL_BOOLEAN, empty, L.FALSE, L.is_null(x.value), L.NULL_BOOLEAN,
591
+ R.CompareExpr(left=found, op=">", right=zero), L.TRUE,
592
+ R.CompareExpr(left=unknown, op=">", right=zero), L.NULL_BOOLEAN, L.FALSE)
593
+ if xt is None:
594
+ raise NotBuilt("IN of a value that is not one plain type")
595
+ numbers = {"i": _INTEGERS, "f": {xsd.DOUBLE, xsd.FLOAT}}
596
+ same = element is not None and (element == xt or (element in _INTEGERS and xt in _INTEGERS))
597
+ if element is not None and not same and xt in numbers["i"] and element in numbers["f"]:
598
+ same, x = True, L.Cell(x.slot, L.fn("CAST_DOUBLE", x.value))
599
+ elif element is not None and not same and xt in numbers["f"] and element in numbers["i"]:
600
+ raise NotBuilt("IN of a float in a list of integers")
601
+ found = L.fn("CYPHER_LIST_CONTAINS", l, x.value) if same else L.FALSE
602
+ return L.case(L.is_null(l), L.NULL_BOOLEAN, empty, L.FALSE, L.is_null(x.value), L.NULL_BOOLEAN,
603
+ found, L.TRUE, has_null, L.NULL_BOOLEAN, L.FALSE)
604
+
605
+
606
+ def concat_cell(e: E.Call, cells: Dict[str, "L.Cell"]) -> "L.Cell":
607
+ """list + list their concatenation; list + x appends x, x + list prepends it; null where an operand
608
+ is null; the empty list concatenated is the other operand."""
609
+ from prograph.plan.ir import lower as L
610
+ from prograph.plan.ir import cypher_mixed as M
611
+ a, b = (L.expr_cell(x, cells) for x in e.args)
612
+ slot = concat_slot(a.slot, b.slot)
613
+ if slot.present is Bound.NEVER:
614
+ return L.nothing()
615
+ native_nesting(slot.datatype if slot.datatypes else None)
616
+ sides = []
617
+ for x, c in zip(e.args, (a, b)):
618
+ if c.slot.kinds == {LIST} and not c.slot.datatypes:
619
+ # A list of no element type: `[]` adds nothing; a literal of nulls (`[null]`) keeps its nulls,
620
+ # typed as the other side's elements; a zero-length trail's empty lists of nodes or relationships
621
+ # (`EMPTY_ELEMENTS`, typed as the engine holds those; a trail's hidden column) are concatenated as
622
+ # they are; any other such list is not built.
623
+ from prograph.plan.ir.cypher_answer import element_base
624
+ if not (isinstance(x, E.Call) and x.name == "LIST"):
625
+ if isinstance(x, E.Col) and "#" in x.name and slot.datatypes and element_base(slot.datatype) is not None:
626
+ sides.append(c.value)
627
+ continue
628
+ raise NotBuilt("+ of a list whose element type is unknown")
629
+ if x.args:
630
+ nulls = [typed_null(slot.datatype, L) for _ in x.args] if slot.datatypes else None
631
+ if nulls is None:
632
+ raise NotBuilt("+ of lists of nulls only")
633
+ sides.append(L.fn("ARRAY", *nulls))
634
+ else:
635
+ sides.append(None)
636
+ elif c.slot.kinds == {LIST}:
637
+ side = c.slot.datatype
638
+ sides.append(M.convert_elements(c.value, side, slot.datatype, L) if side != slot.datatype and M.has_mixed(slot.datatype)
639
+ else c.value)
640
+ else:
641
+ sides.append(L.fn("ARRAY", M.converted_cell(c, slot.datatype, L)))
642
+ kept = [v for v in sides if v is not None]
643
+ out = kept[0] if len(kept) == 1 else L.fn("LIST_CONCAT", *kept) if kept else L.fn("ARRAY")
644
+ cell = L.Cell(slot, L.case(L.any_of([L.is_null(a.value), L.is_null(b.value)]), L.NULL, out))
645
+ parts = _concat_parts(e, (a, b), slot, L)
646
+ if not parts:
647
+ return cell
648
+ from dataclasses import replace
649
+ return replace(cell, parts=parts)
650
+
651
+
652
+ def _concat_parts(e: E.Call, sides, slot: Slot, L) -> tuple:
653
+ """The parts of `a + b` of lists of nodes or relationships (PostgreSQL's answer): each side's parts joined
654
+ part by part, the list's own for a side that is one element; a side that is `[]` adds none. A null side
655
+ makes the list null: its `ids` part is."""
656
+ from prograph.plan.ir.cypher_answer import concat_parts, element_shape, list_parts
657
+ shape = element_shape(slot, L)
658
+ if shape is None or not shape.many:
659
+ return ()
660
+ flat = shape.flat()
661
+ parts = []
662
+ for x, c in zip(e.args, sides):
663
+ if c.slot.kinds == {LIST} and not c.slot.datatypes:
664
+ if isinstance(x, E.Call) and x.name == "LIST" and not x.args:
665
+ continue # `[]` adds nothing
666
+ return ()
667
+ if c.slot.kinds == {LIST}:
668
+ if not c.parts or element_shape(c.slot, L) != shape:
669
+ return ()
670
+ parts.append(c.parts)
671
+ else:
672
+ if not c.parts or element_shape(c.slot, L) != flat:
673
+ return ()
674
+ parts.append(list_parts([c], shape, L))
675
+ if not parts:
676
+ return ()
677
+ joined = concat_parts(parts, L)
678
+ null = L.any_of([L.is_null(c.value) for c in sides])
679
+ return tuple(L.case(null, L.NULL, x) for x in joined) # a list with a null side is null, in every part
680
+
681
+
682
+ def _over_elements(e: E.Call, cells: Dict[str, "L.Cell"], L) -> "R.Expr":
683
+ """all/any/none/single(x IN l WHERE p) and [x IN l WHERE p | e] (openCypher 9): the element a lambda
684
+ variable of the dialects' LIST_FILTER and LIST_TRANSFORM (an ordered unnest on PostgreSQL). A
685
+ quantifier from the counts of elements where p is true and where it is null: any true if one is true,
686
+ else null if one is null, else false; all true if all are, else false if one is false, else null;
687
+ none the negation of any; single true where exactly one is true and none null, false where two are
688
+ true or none is and none null, else null. A null list gives null."""
689
+ from prograph.plan.sql import rel_algebra as R
690
+ lst = L.expr_cell(e.args[0], cells)
691
+ if lst.slot.present is Bound.NEVER:
692
+ return L.NULL_BOOLEAN if e.name in QUANTIFIERS else L.NULL
693
+ fresh = e.args[1].value
694
+ var = R.LitExpr(value=fresh, sql="_" + fresh.lstrip("#"))
695
+ l, n = lst.value, L.fn("SIZE", lst.value)
696
+ from prograph.plan.ir import fresh as F
697
+ if any(F.holds(x) for x in e.args[2:]):
698
+ # rand() per element (rule fresh-values): the lambda runs over the elements' positions, the
699
+ # element read at its position, and each draw a function of the solution's row and the position.
700
+ return _over_positions(e, lst, var, cells, L)
701
+ inner = {**cells, fresh: L.Cell(_element(lst.slot), var)}
702
+ p = L.boolean(e.args[2], inner)
703
+ if e.name == "COMPREHENSION":
704
+ kept = l if isinstance(e.args[2], E.Const) and e.args[2].value is True else \
705
+ L.fn("LIST_FILTER", l, var, L.fn("COALESCE", p, L.FALSE))
706
+ mapped = e.args[3]
707
+ if not (isinstance(mapped, E.Col) and mapped.name == fresh):
708
+ out = L.expr_cell(mapped, inner)
709
+ # Null for every element (`| null`, a property no element has): typed nulls, as Spark's array
710
+ # of untyped nulls does not reach the answer (Arrow refuses it).
711
+ kept = L.fn("LIST_TRANSFORM", kept, var,
712
+ L.fn("CAST_STRING", L.NULL) if out.slot.present is Bound.NEVER else out.value)
713
+ return L.case(L.is_null(l), L.NULL, kept)
714
+ true = L.fn("SIZE", L.fn("LIST_FILTER", l, var, L.fn("COALESCE", p, L.FALSE)))
715
+ null = L.fn("SIZE", L.fn("LIST_FILTER", l, var, L.is_null(p)))
716
+ return _quantified(e.name, true, null, n, l, L)
717
+
718
+
719
+ def _over_positions(e: E.Call, lst: "L.Cell", var: "R.Expr", cells: Dict[str, "L.Cell"], L) -> "R.Expr":
720
+ """`_over_elements` where the predicate or the mapped expression draws a fresh value: over the list's
721
+ positions 1..size (`range`), the element variable read as the element at its position, so equal
722
+ elements of one list draw apart and one element's draw read twice is one value (`fresh.per_element`)."""
723
+ from prograph.plan.ir import fresh as F
724
+ l, n = lst.value, L.fn("SIZE", lst.value)
725
+ positions = L.fn("CYPHER_RANGE", L.integer(1), n, L.integer(1))
726
+ inner = {**cells, e.args[1].value: L.Cell(_element(lst.slot), L.fn("CYPHER_ELEMENT_AT", l, var))}
727
+ with F.per_element(var):
728
+ p = L.boolean(e.args[2], inner)
729
+ mapped = L.expr_cell(e.args[3], inner) if e.name == "COMPREHENSION" else None
730
+ if e.name == "COMPREHENSION":
731
+ kept = positions if isinstance(e.args[2], E.Const) and e.args[2].value is True else \
732
+ L.fn("LIST_FILTER", positions, var, L.fn("COALESCE", p, L.FALSE))
733
+ out = L.fn("CAST_STRING", L.NULL) if mapped.slot.present is Bound.NEVER else mapped.value
734
+ return L.case(L.is_null(l), L.NULL, L.fn("LIST_TRANSFORM", kept, var, out))
735
+ true = L.fn("SIZE", L.fn("LIST_FILTER", positions, var, L.fn("COALESCE", p, L.FALSE)))
736
+ null = L.fn("SIZE", L.fn("LIST_FILTER", positions, var, L.is_null(p)))
737
+ return _quantified(e.name, true, null, n, l, L)
738
+
739
+
740
+ def _quantified(name: str, true: "R.Expr", null: "R.Expr", n: "R.Expr", l: "R.Expr", L) -> "R.Expr":
741
+ """A quantifier's value from how many elements make its test true and how many null (`_over_elements`)."""
742
+ from prograph.plan.sql import rel_algebra as R
743
+
744
+ def cmp(a, op, b):
745
+ return R.CompareExpr(left=a, op=op, right=b)
746
+ zero = L.integer(0)
747
+ if name == "ANY":
748
+ out = L.case(cmp(true, ">", zero), L.TRUE, cmp(null, ">", zero), L.NULL_BOOLEAN, L.FALSE)
749
+ elif name == "NONE":
750
+ out = L.case(cmp(true, ">", zero), L.FALSE, cmp(null, ">", zero), L.NULL_BOOLEAN, L.TRUE)
751
+ elif name == "ALL":
752
+ out = L.case(cmp(true, "=", n), L.TRUE, cmp(L.fn("+", true, null), "<", n), L.FALSE, L.NULL_BOOLEAN)
753
+ else: # SINGLE
754
+ out = L.case(cmp(true, ">", L.integer(1)), L.FALSE, cmp(null, ">", zero), L.NULL_BOOLEAN,
755
+ cmp(true, "=", L.integer(1)), L.TRUE, L.FALSE)
756
+ return L.case(L.is_null(l), L.NULL_BOOLEAN, out)
757
+
758
+
759
+ _CASTS = {xsd.DOUBLE: "CAST_DOUBLE", xsd.FLOAT: "CAST_DOUBLE", xsd.STRING: "CAST_STRING",
760
+ xsd.BOOLEAN: "CAST_BOOLEAN"}
761
+
762
+
763
+ def typed_null(datatype: Optional[str], L) -> "R.Expr":
764
+ """A null of the SQL type a Cypher value of `datatype` is held as; a list's the empty list's
765
+ type under a false CASE (no cast names an engine's array type)."""
766
+ from prograph.plan.ir.cypher_maps import is_map_type, type_fields
767
+ from prograph.plan.ir.cypher_mixed import is_mixed, null_struct
768
+ if is_mixed(datatype):
769
+ return null_struct(L)
770
+ if is_map_type(datatype):
771
+ parts = []
772
+ for k, t in type_fields(datatype).items():
773
+ parts += [L.text(k), typed_null(t, L) if t is not None else L.NULL]
774
+ return L.case(L.FALSE, L.fn("NAMED_STRUCT", *parts))
775
+ if nested(datatype):
776
+ inner = element_of(datatype)
777
+ return L.case(L.FALSE, typed_empty(inner, L) if inner is not None else L.fn("ARRAY"))
778
+ if datatype in _INTEGERS:
779
+ return L.fn("CAST_LONG", L.NULL)
780
+ if datatype in _CASTS:
781
+ return L.fn(_CASTS[datatype], L.NULL)
782
+ if datatype == xsd.DECIMAL and L.DIALECT.get() in ("postgresql", "postgres"):
783
+ # PostgreSQL's numeric takes any precision: a null of it beside a DECIMAL(p, s) list (its bare [] is text[]);
784
+ # elsewhere the engine's own [] takes the other list's type.
785
+ return L.fn("TRY_DECIMAL_TEXT", L.NULL)
786
+ from prograph.plan.ir.cypher_answer import NODE_TYPE, PATH_TYPE, REL_TYPE, element_base
787
+ kind = {NODE_TYPE: "nodes", REL_TYPE: "relationships"}.get(element_base(datatype)) or \
788
+ ("paths" if datatype and datatype.startswith(PATH_TYPE) else None)
789
+ if kind is not None:
790
+ raise NotBuilt(f"a null beside {kind} in one list (`collect(n) + [null]`), or a null written where "
791
+ f"{kind} are")
792
+ raise NotBuilt("a null beside values of this type in one list, or a null written where they are (dates, "
793
+ "times, durations and the like)")
794
+
795
+
796
+ def empty_of(element: Slot, L) -> "R.Expr":
797
+ """[] as a list of `element`'s values: `typed_empty` where a null of their type is written; else (dates,
798
+ nodes, paths and the like) the engine's own [], which takes the other list's type beside it, PostgreSQL's
799
+ a jsonb[] for the values it answers as jsonb (its bare [] is text[])."""
800
+ try:
801
+ return typed_empty(value_type(element), L)
802
+ except NotBuilt:
803
+ from prograph.plan.ir.cypher_answer import PATH_TYPE, PROPS_TYPE, element_base
804
+ if L.DIALECT.get() in ("postgresql", "postgres") and element.kinds == {L.VALUE} \
805
+ and any(element_base(d) is not None or d.startswith(PROPS_TYPE) or d.startswith(PATH_TYPE)
806
+ for d in element.datatypes):
807
+ return L.fn("JSONB_EMPTY_ARRAY") # PostgreSQL answers these as jsonb: its array of them is jsonb[]
808
+ return L.fn("ARRAY")
809
+
810
+
811
+ def typed_empty(datatype: Optional[str], L) -> "R.Expr":
812
+ """[] as a list of `datatype`'s SQL type on every engine: a one-null list filtered to nothing."""
813
+ from prograph.plan.sql import rel_algebra as R
814
+ var = R.LitExpr(value="_e", sql="_e")
815
+ return L.fn("LIST_FILTER", L.fn("ARRAY", typed_null(datatype, L)), var, L.FALSE)
816
+
817
+
818
+ _LAMBDAS = itertools.count()
819
+
820
+
821
+ def _counts(lst: "R.Expr", var: "R.Expr", test: "R.Expr", L) -> "tuple":
822
+ """How many elements of `lst` make `test` (of the lambda variable `var`) true, and how many null."""
823
+ true = L.fn("SIZE", L.fn("LIST_FILTER", lst, var, L.fn("COALESCE", test, L.FALSE)))
824
+ null = L.fn("SIZE", L.fn("LIST_FILTER", lst, var, L.is_null(test)))
825
+ return true, null
826
+
827
+
828
+ def _pairwise(a: "L.Cell", b: "L.Cell", L) -> "R.Expr":
829
+ """`a = b` of two lists whose elements are lists (openCypher 9 equality, element by element, each
830
+ pair by Cypher's own `=`): null where a list is null; false for two sizes or a pair unequal; else
831
+ null where a pair's equality is null; else true."""
832
+ from prograph.plan.sql import rel_algebra as R
833
+ x, y = a.value, b.value
834
+ i = R.LitExpr(value="_q", sql=f"_q{next(_LAMBDAS)}")
835
+ pair = L.cypher_compare("=", L.Cell(_element(a.slot), L.fn("CYPHER_ELEMENT_AT", x, i)),
836
+ L.Cell(_element(b.slot), L.fn("CYPHER_ELEMENT_AT", y, i)))
837
+ positions = L.fn("CYPHER_RANGE", L.integer(1), L.fn("SIZE", x), L.integer(1))
838
+ unequal, null = _counts(positions, i, R.NotExpr(child=pair), L)
839
+ zero = L.integer(0)
840
+ return L.case(L.any_of([L.is_null(x), L.is_null(y)]), L.NULL_BOOLEAN,
841
+ R.CompareExpr(left=L.fn("SIZE", x), op="<>", right=L.fn("SIZE", y)), L.FALSE,
842
+ R.CompareExpr(left=unequal, op=">", right=zero), L.FALSE,
843
+ R.CompareExpr(left=null, op=">", right=zero), L.NULL_BOOLEAN, L.TRUE)
844
+
845
+
846
+ def list_equality(op: str, a: "L.Cell", b: "L.Cell") -> "R.Expr":
847
+ """`=` and `<>` of two lists (openCypher 9 equality): null where a list is null; false for two
848
+ sizes; false where a pair of elements differs (two non-null elements unequal: a definite mismatch
849
+ wins over a null); else null where a pair holds a null; else true. Lists of nodes or relationships pair
850
+ by identity. A list against a non-list is not
851
+ equal. `<`, `>` on lists: `list_order`."""
852
+ from prograph.plan.ir import lower as L
853
+ from prograph.plan.sql import rel_algebra as R
854
+ if a.slot.kinds != {LIST} or b.slot.kinds != {LIST}:
855
+ if a.slot.present is Bound.NEVER or b.slot.present is Bound.NEVER:
856
+ return L.NULL_BOOLEAN
857
+ # A list and a value of another kind: unequal (= false, <> true), not ordered (null); null where
858
+ # either is null.
859
+ differ = {"=": L.FALSE, "!=": L.TRUE}.get(op, L.NULL_BOOLEAN)
860
+ return L.case(L.any_of([L.is_null(a.value), L.is_null(b.value)]), L.NULL_BOOLEAN, differ)
861
+ if op not in ("=", "!="):
862
+ return list_order(op, a, b)
863
+ if a.slot.present is Bound.NEVER or b.slot.present is Bound.NEVER:
864
+ return L.NULL_BOOLEAN
865
+ ta = a.slot.datatype if a.slot.datatypes else None
866
+ tb = b.slot.datatype if b.slot.datatypes else None
867
+ from prograph.plan.ir.cypher_answer import element_base
868
+ from prograph.plan.ir.cypher_temporal import STRUCTS
869
+ from prograph.plan.ir.cypher_maps import is_map_type
870
+ from prograph.plan.ir.cypher_mixed import is_mixed
871
+ # Lists of nodes or relationships pair by pair too, each pair by identity (`cypher_answer.held_equality`).
872
+ held = element_base(ta) is not None or element_base(tb) is not None
873
+ if held or nested(ta) or nested(tb) or is_map_type(ta) or is_map_type(tb) or is_mixed(ta) or is_mixed(tb):
874
+ eq = _pairwise(a, b, L)
875
+ return eq if op == "=" else R.NotExpr(child=eq)
876
+ if ta in STRUCTS or tb in STRUCTS:
877
+ raise NotBuilt("equality of lists of times, date-times or durations")
878
+ numbers = _INTEGERS | {xsd.DOUBLE, xsd.FLOAT}
879
+ comparable = ta is None or tb is None or ta == tb or (ta in numbers and tb in numbers)
880
+ x, y = a.value, b.value
881
+ size = R.CompareExpr(left=L.fn("SIZE", x), op="<>", right=L.fn("SIZE", y))
882
+ nulls = L.fn("CYPHER_LIST_NULL_PAIRS", x, y)
883
+ if ta is None or tb is None:
884
+ differ = L.integer(0) # a list of nulls only has no definite mismatch (PostgreSQL types it text[])
885
+ else:
886
+ differ = L.fn("CYPHER_LIST_MISMATCHES", x, y) if comparable else L.fn("CYPHER_LIST_VALUE_PAIRS", x, y)
887
+ zero = L.integer(0)
888
+ eq = L.case(L.any_of([L.is_null(x), L.is_null(y)]), L.NULL_BOOLEAN, size, L.FALSE,
889
+ R.CompareExpr(left=differ, op=">", right=zero), L.FALSE,
890
+ R.CompareExpr(left=nulls, op=">", right=zero), L.NULL_BOOLEAN, L.TRUE)
891
+ return eq if op == "=" else R.NotExpr(child=eq)
892
+
893
+
894
+ def list_order(op: str, a: "L.Cell", b: "L.Cell") -> "R.Expr":
895
+ """`<`, `<=`, `>`, `>=` of two lists (openCypher 9 comparability, TCK Comparison2 [4]): lexicographic.
896
+ The first position where the elements are not equal (their `=` false or null) decides, by the elements'
897
+ own comparison (null where one is null); where every position the shorter list has is equal, the
898
+ shorter is the smaller. Null for a null list. Built for two lists of one element type, integers, strings
899
+ or booleans (either side may be a list of nulls only); two lists of different element types, and lists of
900
+ floats (NaN's place among lists), temporals, lists or maps are refused."""
901
+ from prograph.plan.ir import lower as L
902
+ from prograph.plan.sql import rel_algebra as R
903
+ if a.slot.present is Bound.NEVER or b.slot.present is Bound.NEVER:
904
+ return L.NULL_BOOLEAN
905
+ plain = _INTEGERS | {xsd.STRING, xsd.BOOLEAN}
906
+ types = [x.slot.datatype if x.slot.datatypes else None for x in (a, b)]
907
+ from prograph.plan.ir.cypher_mixed import is_mixed, tags_of
908
+ mixed = any(is_mixed(t) for t in types)
909
+ if mixed and all(t is None or t in plain or (is_mixed(t) and "f" not in tags_of(t)) for t in types):
910
+ # A list of integers, strings and booleans in different places (TCK Comparison2 [4]): the elements
911
+ # compared as mixed values are (`cypher_mixed.compare`: two kinds unequal and unordered, so the
912
+ # first such pair decides null); a float among them is refused as below.
913
+ pass
914
+ elif any(t is not None and t not in plain for t in types):
915
+ raise NotBuilt("ordering lists of floats, temporals, lists, maps or values of several types (<, > on lists)")
916
+ families = {"integer" if t in _INTEGERS else t for t in types if t is not None and not is_mixed(t)}
917
+ if len(families) > 1 and not mixed:
918
+ # openCypher answers null at the first pair of two kinds (and compares two empty lists); the engines'
919
+ # arrays of two element types meet in no one SQL type (PostgreSQL's CASE refuses them).
920
+ raise NotBuilt("ordering two lists of different element types (<, > on lists)")
921
+ x, y = a.value, b.value
922
+ i = R.LitExpr(value="_q", sql=f"_q{next(_LAMBDAS)}")
923
+ pair = L.cypher_compare("=", L.Cell(_element(a.slot), L.fn("CYPHER_ELEMENT_AT", x, i)),
924
+ L.Cell(_element(b.slot), L.fn("CYPHER_ELEMENT_AT", y, i)))
925
+ sx, sy = L.fn("SIZE", x), L.fn("SIZE", y)
926
+ shorter = L.case(R.CompareExpr(left=sx, op="<", right=sy), sx, sy)
927
+ positions = L.fn("CYPHER_RANGE", L.integer(1), shorter, L.integer(1))
928
+ differing = L.fn("LIST_FILTER", positions, i, R.NotExpr(child=L.fn("COALESCE", pair, L.FALSE)))
929
+ k = L.fn("CYPHER_ELEMENT_AT", differing, L.integer(1))
930
+ first = L.cypher_compare(op, L.Cell(_element(a.slot), L.fn("CYPHER_ELEMENT_AT", x, k)),
931
+ L.Cell(_element(b.slot), L.fn("CYPHER_ELEMENT_AT", y, k)))
932
+ sql_op = {"<": "<", "<=": "<=", ">": ">", ">=": ">="}[op]
933
+ return L.case(L.any_of([L.is_null(x), L.is_null(y)]), L.NULL_BOOLEAN,
934
+ R.CompareExpr(left=L.fn("SIZE", differing), op="=", right=L.integer(0)),
935
+ R.CompareExpr(left=sx, op=sql_op, right=sy), first)