pytrilogy 0.0.2.1__py3-none-any.whl → 0.0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pytrilogy might be problematic. Click here for more details.
- {pytrilogy-0.0.2.1.dist-info → pytrilogy-0.0.2.3.dist-info}/METADATA +3 -3
- {pytrilogy-0.0.2.1.dist-info → pytrilogy-0.0.2.3.dist-info}/RECORD +25 -25
- {pytrilogy-0.0.2.1.dist-info → pytrilogy-0.0.2.3.dist-info}/WHEEL +1 -1
- trilogy/__init__.py +1 -1
- trilogy/core/enums.py +1 -0
- trilogy/core/env_processor.py +5 -0
- trilogy/core/functions.py +30 -5
- trilogy/core/models.py +71 -8
- trilogy/core/optimization.py +46 -31
- trilogy/core/optimizations/predicate_pushdown.py +74 -8
- trilogy/core/processing/concept_strategies_v3.py +9 -0
- trilogy/core/processing/node_generators/basic_node.py +1 -1
- trilogy/core/processing/node_generators/node_merge_node.py +46 -108
- trilogy/core/processing/node_generators/select_node.py +2 -3
- trilogy/core/processing/utility.py +10 -32
- trilogy/core/query_processor.py +3 -1
- trilogy/dialect/base.py +32 -83
- trilogy/dialect/presto.py +1 -0
- trilogy/parsing/common.py +4 -1
- trilogy/parsing/parse_engine.py +25 -5
- trilogy/parsing/render.py +7 -3
- trilogy/parsing/trilogy.lark +6 -3
- {pytrilogy-0.0.2.1.dist-info → pytrilogy-0.0.2.3.dist-info}/LICENSE.md +0 -0
- {pytrilogy-0.0.2.1.dist-info → pytrilogy-0.0.2.3.dist-info}/entry_points.txt +0 -0
- {pytrilogy-0.0.2.1.dist-info → pytrilogy-0.0.2.3.dist-info}/top_level.txt +0 -0
|
@@ -4,7 +4,6 @@ from trilogy.core.models import Concept, Environment, Conditional
|
|
|
4
4
|
from trilogy.core.processing.nodes import MergeNode, History, StrategyNode
|
|
5
5
|
import networkx as nx
|
|
6
6
|
from trilogy.core.graph_models import concept_to_node
|
|
7
|
-
from trilogy.core.processing.utility import PathInfo
|
|
8
7
|
from trilogy.constants import logger
|
|
9
8
|
from trilogy.utility import unique
|
|
10
9
|
from trilogy.core.exceptions import AmbiguousRelationshipResolutionException
|
|
@@ -63,7 +62,9 @@ def extract_ds_components(g: nx.DiGraph, nodelist: list[str]) -> list[list[str]]
|
|
|
63
62
|
if not str(x).startswith("ds~")
|
|
64
63
|
]
|
|
65
64
|
)
|
|
66
|
-
|
|
65
|
+
# if we had no ego graphs, return all concepts
|
|
66
|
+
if not graphs:
|
|
67
|
+
return [[extract_address(node) for node in nodelist]]
|
|
67
68
|
graphs = filter_unique_graphs(graphs)
|
|
68
69
|
for node in nodelist:
|
|
69
70
|
parsed = extract_address(node)
|
|
@@ -82,6 +83,7 @@ def determine_induced_minimal_nodes(
|
|
|
82
83
|
H: nx.Graph = nx.to_undirected(G).copy()
|
|
83
84
|
nodes_to_remove = []
|
|
84
85
|
concepts = nx.get_node_attributes(G, "concept")
|
|
86
|
+
|
|
85
87
|
for node in G.nodes:
|
|
86
88
|
if concepts.get(node):
|
|
87
89
|
lookup = concepts[node]
|
|
@@ -107,9 +109,11 @@ def determine_induced_minimal_nodes(
|
|
|
107
109
|
paths = nx.multi_source_dijkstra_path(H, nodelist)
|
|
108
110
|
except nx.exception.NodeNotFound:
|
|
109
111
|
return None
|
|
112
|
+
|
|
110
113
|
H.remove_nodes_from(list(x for x in H.nodes if x not in paths))
|
|
111
114
|
sG: nx.Graph = ax.steinertree.steiner_tree(H, nodelist).copy()
|
|
112
115
|
final: nx.DiGraph = nx.subgraph(G, sG.nodes).copy()
|
|
116
|
+
|
|
113
117
|
for edge in G.edges:
|
|
114
118
|
if edge[1] in final.nodes and edge[0].startswith("ds~"):
|
|
115
119
|
ds_name = extract_address(edge[0])
|
|
@@ -125,6 +129,7 @@ def determine_induced_minimal_nodes(
|
|
|
125
129
|
[final.in_degree(node) > 0 for node in final.nodes if node.startswith("c~")]
|
|
126
130
|
):
|
|
127
131
|
return None
|
|
132
|
+
|
|
128
133
|
if not all([node in final.nodes for node in nodelist]):
|
|
129
134
|
return None
|
|
130
135
|
return final
|
|
@@ -308,111 +313,44 @@ def gen_merge_node(
|
|
|
308
313
|
history: History | None = None,
|
|
309
314
|
conditions: Conditional | None = None,
|
|
310
315
|
) -> Optional[MergeNode]:
|
|
311
|
-
join_candidates: List[PathInfo] = []
|
|
312
|
-
|
|
313
|
-
# inject new concepts into search, and identify if two dses can reach there
|
|
314
|
-
if not join_candidates:
|
|
315
|
-
for filter_downstream in [True, False]:
|
|
316
|
-
weak_resolve = resolve_weak_components(
|
|
317
|
-
all_concepts,
|
|
318
|
-
environment,
|
|
319
|
-
g,
|
|
320
|
-
filter_downstream=filter_downstream,
|
|
321
|
-
accept_partial=accept_partial,
|
|
322
|
-
)
|
|
323
|
-
if weak_resolve:
|
|
324
|
-
log_graph = [[y.address for y in x] for x in weak_resolve]
|
|
325
|
-
logger.info(
|
|
326
|
-
f"{padding(depth)}{LOGGER_PREFIX} Was able to resolve graph through weak component resolution - final graph {log_graph}"
|
|
327
|
-
)
|
|
328
|
-
return subgraphs_to_merge_node(
|
|
329
|
-
weak_resolve,
|
|
330
|
-
depth=depth,
|
|
331
|
-
all_concepts=all_concepts,
|
|
332
|
-
environment=environment,
|
|
333
|
-
g=g,
|
|
334
|
-
source_concepts=source_concepts,
|
|
335
|
-
history=history,
|
|
336
|
-
conditions=conditions,
|
|
337
|
-
)
|
|
338
|
-
if not join_candidates:
|
|
339
|
-
return None
|
|
340
|
-
join_additions: list[set[str]] = []
|
|
341
|
-
for candidate in join_candidates:
|
|
342
|
-
join_additions.append(candidate.reduced_concepts)
|
|
343
|
-
|
|
344
|
-
common: set[str] = set()
|
|
345
|
-
final_candidates: list[set[str]] = []
|
|
346
|
-
# find all values that show up in every join_additions
|
|
347
|
-
for ja in join_additions:
|
|
348
|
-
if not common:
|
|
349
|
-
common = ja
|
|
350
|
-
else:
|
|
351
|
-
common = common.intersection(ja)
|
|
352
|
-
if all(ja.issubset(y) for y in join_additions):
|
|
353
|
-
final_candidates.append(ja)
|
|
354
316
|
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
logger.info(
|
|
363
|
-
f"{padding(depth)}{LOGGER_PREFIX} No additional join candidates could be found"
|
|
317
|
+
for filter_downstream in [True, False]:
|
|
318
|
+
weak_resolve = resolve_weak_components(
|
|
319
|
+
all_concepts,
|
|
320
|
+
environment,
|
|
321
|
+
g,
|
|
322
|
+
filter_downstream=filter_downstream,
|
|
323
|
+
accept_partial=accept_partial,
|
|
364
324
|
)
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
)
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
# )
|
|
398
|
-
# return None
|
|
399
|
-
# logger.info(
|
|
400
|
-
# f"{padding(depth)}{LOGGER_PREFIX} finished subgraph fetch for {[c.address for c in graph]}, have parent {type(parent)}"
|
|
401
|
-
# )
|
|
402
|
-
# parents.append(parent)
|
|
403
|
-
|
|
404
|
-
# return MergeNode(
|
|
405
|
-
# input_concepts=[
|
|
406
|
-
# environment.concepts[x]
|
|
407
|
-
# for x in shortest.reduced_concepts
|
|
408
|
-
# if environment.concepts[x].derivation != PurposeLineage.MERGE
|
|
409
|
-
# ],
|
|
410
|
-
# output_concepts=[
|
|
411
|
-
# x for x in all_concepts if x.derivation != PurposeLineage.MERGE
|
|
412
|
-
# ],
|
|
413
|
-
# environment=environment,
|
|
414
|
-
# g=g,
|
|
415
|
-
# parents=parents,
|
|
416
|
-
# depth=depth,
|
|
417
|
-
# conditions=conditions,
|
|
418
|
-
# )
|
|
325
|
+
if weak_resolve:
|
|
326
|
+
log_graph = [[y.address for y in x] for x in weak_resolve]
|
|
327
|
+
logger.info(
|
|
328
|
+
f"{padding(depth)}{LOGGER_PREFIX} Was able to resolve graph through weak component resolution - final graph {log_graph}"
|
|
329
|
+
)
|
|
330
|
+
return subgraphs_to_merge_node(
|
|
331
|
+
weak_resolve,
|
|
332
|
+
depth=depth,
|
|
333
|
+
all_concepts=all_concepts,
|
|
334
|
+
environment=environment,
|
|
335
|
+
g=g,
|
|
336
|
+
source_concepts=source_concepts,
|
|
337
|
+
history=history,
|
|
338
|
+
conditions=conditions,
|
|
339
|
+
)
|
|
340
|
+
# one concept handling may need to be kicked to alias
|
|
341
|
+
if len(all_concepts) == 1:
|
|
342
|
+
concept = all_concepts[0]
|
|
343
|
+
for k, v in concept.pseudonyms.items():
|
|
344
|
+
test = subgraphs_to_merge_node(
|
|
345
|
+
[[concept, v]],
|
|
346
|
+
g=g,
|
|
347
|
+
all_concepts=[concept],
|
|
348
|
+
environment=environment,
|
|
349
|
+
depth=depth,
|
|
350
|
+
source_concepts=source_concepts,
|
|
351
|
+
history=history,
|
|
352
|
+
conditions=conditions,
|
|
353
|
+
)
|
|
354
|
+
if test:
|
|
355
|
+
return test
|
|
356
|
+
return None
|
|
@@ -89,6 +89,7 @@ def dm_to_strategy_node(
|
|
|
89
89
|
accept_partial=accept_partial,
|
|
90
90
|
datasource=datasource,
|
|
91
91
|
grain=datasource.grain,
|
|
92
|
+
conditions=datasource.where.conditional if datasource.where else None,
|
|
92
93
|
)
|
|
93
94
|
# we need to nest the group node one further
|
|
94
95
|
if force_group is True:
|
|
@@ -295,9 +296,6 @@ def gen_select_node_from_table(
|
|
|
295
296
|
g.nodes[ncandidate]
|
|
296
297
|
except KeyError:
|
|
297
298
|
raise nx.exception.NetworkXNoPath
|
|
298
|
-
raise SyntaxError(
|
|
299
|
-
f"Could not find node {ncandidate}, have {list(g.nodes())}"
|
|
300
|
-
)
|
|
301
299
|
raise e
|
|
302
300
|
except nx.exception.NetworkXNoPath:
|
|
303
301
|
all_found = False
|
|
@@ -372,6 +370,7 @@ def gen_select_node_from_table(
|
|
|
372
370
|
accept_partial=accept_partial,
|
|
373
371
|
datasource=datasource,
|
|
374
372
|
grain=Grain(components=all_concepts),
|
|
373
|
+
conditions=datasource.where.conditional if datasource.where else None,
|
|
375
374
|
)
|
|
376
375
|
# we need to nest the group node one further
|
|
377
376
|
if force_group is True:
|
|
@@ -124,15 +124,20 @@ def resolve_join_order(joins: List[BaseJoin]) -> List[BaseJoin]:
|
|
|
124
124
|
return final_joins
|
|
125
125
|
|
|
126
126
|
|
|
127
|
-
def add_node_join_concept(
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
127
|
+
def add_node_join_concept(
|
|
128
|
+
graph: nx.DiGraph,
|
|
129
|
+
concept: Concept,
|
|
130
|
+
datasource: Datasource | QueryDatasource,
|
|
131
|
+
concepts: List[Concept],
|
|
132
|
+
):
|
|
133
|
+
|
|
131
134
|
concepts.append(concept)
|
|
132
135
|
|
|
133
136
|
graph.add_node(concept.address, type=NodeType.CONCEPT)
|
|
134
137
|
graph.add_edge(datasource.identifier, concept.address)
|
|
135
|
-
for
|
|
138
|
+
for _, v in concept.pseudonyms.items():
|
|
139
|
+
if v in concepts:
|
|
140
|
+
continue
|
|
136
141
|
if v.address != concept.address:
|
|
137
142
|
add_node_join_concept(graph, v, datasource, concepts)
|
|
138
143
|
|
|
@@ -149,13 +154,6 @@ def get_node_joins(
|
|
|
149
154
|
graph.add_node(datasource.identifier, type=NodeType.NODE)
|
|
150
155
|
for concept in datasource.output_concepts:
|
|
151
156
|
add_node_join_concept(graph, concept, datasource, concepts)
|
|
152
|
-
# we don't need to join on a concept if all of the keys exist in the grain
|
|
153
|
-
# if concept.keys and all([x in grain for x in concept.keys]):
|
|
154
|
-
# continue
|
|
155
|
-
# concepts.append(concept)
|
|
156
|
-
|
|
157
|
-
# graph.add_node(concept.address, type=NodeType.CONCEPT)
|
|
158
|
-
# graph.add_edge(datasource.identifier, concept.address)
|
|
159
157
|
|
|
160
158
|
# add edges for every constant to every datasource
|
|
161
159
|
for datasource in datasources:
|
|
@@ -195,26 +193,6 @@ def get_node_joins(
|
|
|
195
193
|
),
|
|
196
194
|
)
|
|
197
195
|
|
|
198
|
-
node_map = {
|
|
199
|
-
x[0:20]: len(
|
|
200
|
-
[
|
|
201
|
-
partial
|
|
202
|
-
for partial in identifier_map[x].partial_concepts
|
|
203
|
-
if partial in grain
|
|
204
|
-
]
|
|
205
|
-
+ [
|
|
206
|
-
output
|
|
207
|
-
for output in identifier_map[x].output_concepts
|
|
208
|
-
if output.address in grain_pseudonyms
|
|
209
|
-
]
|
|
210
|
-
)
|
|
211
|
-
for x in node_list
|
|
212
|
-
}
|
|
213
|
-
print("NODE MAP")
|
|
214
|
-
print(node_map)
|
|
215
|
-
print([x.address for x in grain])
|
|
216
|
-
print(grain_pseudonyms)
|
|
217
|
-
|
|
218
196
|
for left in node_list:
|
|
219
197
|
# the constant dataset is a special case
|
|
220
198
|
# and can never be on the left of a join
|
trilogy/core/query_processor.py
CHANGED
|
@@ -467,9 +467,11 @@ def process_query(
|
|
|
467
467
|
for cte in raw_ctes:
|
|
468
468
|
cte.parent_ctes = [seen[x.name] for x in cte.parent_ctes]
|
|
469
469
|
deduped_ctes: List[CTE] = list(seen.values())
|
|
470
|
+
root_cte.order_by = statement.order_by
|
|
471
|
+
root_cte.limit = statement.limit
|
|
472
|
+
root_cte.hidden_concepts = [x for x in statement.hidden_components]
|
|
470
473
|
|
|
471
474
|
final_ctes = optimize_ctes(deduped_ctes, root_cte, statement)
|
|
472
|
-
|
|
473
475
|
return ProcessedQuery(
|
|
474
476
|
order_by=statement.order_by,
|
|
475
477
|
grain=statement.grain,
|
trilogy/dialect/base.py
CHANGED
|
@@ -8,7 +8,6 @@ from trilogy.core.enums import (
|
|
|
8
8
|
FunctionType,
|
|
9
9
|
WindowType,
|
|
10
10
|
DatePart,
|
|
11
|
-
PurposeLineage,
|
|
12
11
|
ComparisonOperator,
|
|
13
12
|
)
|
|
14
13
|
from trilogy.core.models import (
|
|
@@ -36,6 +35,7 @@ from trilogy.core.models import (
|
|
|
36
35
|
Environment,
|
|
37
36
|
RawColumnExpr,
|
|
38
37
|
ListWrapper,
|
|
38
|
+
MapWrapper,
|
|
39
39
|
ShowStatement,
|
|
40
40
|
RowsetItem,
|
|
41
41
|
MultiSelectStatement,
|
|
@@ -45,6 +45,7 @@ from trilogy.core.models import (
|
|
|
45
45
|
RawSQLStatement,
|
|
46
46
|
ProcessedRawSQLStatement,
|
|
47
47
|
NumericType,
|
|
48
|
+
MapType,
|
|
48
49
|
MergeStatementV2,
|
|
49
50
|
)
|
|
50
51
|
from trilogy.core.query_processor import process_query, process_persist
|
|
@@ -97,6 +98,7 @@ DATATYPE_MAP = {
|
|
|
97
98
|
DataType.FLOAT: "float",
|
|
98
99
|
DataType.BOOL: "bool",
|
|
99
100
|
DataType.NUMERIC: "numeric",
|
|
101
|
+
DataType.MAP: "map",
|
|
100
102
|
}
|
|
101
103
|
|
|
102
104
|
|
|
@@ -116,6 +118,7 @@ FUNCTION_MAP = {
|
|
|
116
118
|
FunctionType.IS_NULL: lambda x: f"isnull({x[0]})",
|
|
117
119
|
# complex
|
|
118
120
|
FunctionType.INDEX_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
|
|
121
|
+
FunctionType.MAP_ACCESS: lambda x: f"{x[0]}[{x[1]}][1]",
|
|
119
122
|
FunctionType.UNNEST: lambda x: f"unnest({x[0]})",
|
|
120
123
|
# math
|
|
121
124
|
FunctionType.ADD: lambda x: f"{x[0]} + {x[1]}",
|
|
@@ -189,14 +192,13 @@ TOP {{ limit }}{% endif %}
|
|
|
189
192
|
\t{{ select }}{% if not loop.last %},{% endif %}{% endfor %}
|
|
190
193
|
{% if base %}FROM
|
|
191
194
|
\t{{ base }}{% endif %}{% if joins %}{% for join in joins %}
|
|
192
|
-
\t{{ join }}{% endfor %}{% endif %}
|
|
193
|
-
|
|
194
|
-
\t{{ where }}
|
|
195
|
-
|
|
196
|
-
\t{{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
|
|
197
|
-
{
|
|
198
|
-
|
|
199
|
-
{{ order }}{% if not loop.last %},{% endif %}{% endfor %}
|
|
195
|
+
\t{{ join }}{% endfor %}{% endif %}{% if where %}
|
|
196
|
+
WHERE
|
|
197
|
+
\t{{ where }}{% endif %}{%- if group_by %}
|
|
198
|
+
GROUP BY {% for group in group_by %}
|
|
199
|
+
\t{{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{%- if order_by %}
|
|
200
|
+
ORDER BY{% for order in order_by %}
|
|
201
|
+
\t{{ order }}{% if not loop.last %},{% endif %}{% endfor %}
|
|
200
202
|
{% endif %}{% endif %}
|
|
201
203
|
"""
|
|
202
204
|
)
|
|
@@ -214,7 +216,13 @@ def safe_get_cte_value(coalesce, cte: CTE, c: Concept, quote_char: str):
|
|
|
214
216
|
raw = cte.source_map.get(address, None)
|
|
215
217
|
|
|
216
218
|
if not raw:
|
|
217
|
-
|
|
219
|
+
for k, v in c.pseudonyms.items():
|
|
220
|
+
if cte.source_map.get(k):
|
|
221
|
+
c = v
|
|
222
|
+
raw = cte.source_map[k]
|
|
223
|
+
break
|
|
224
|
+
if not raw:
|
|
225
|
+
return INVALID_REFERENCE_STRING("Missing source reference")
|
|
218
226
|
if isinstance(raw, str):
|
|
219
227
|
rendered = cte.get_alias(c, raw)
|
|
220
228
|
return f"{raw}.{safe_quote(rendered, quote_char)}"
|
|
@@ -291,6 +299,7 @@ class BaseDialect:
|
|
|
291
299
|
self.render_expr(v, cte) # , alias=False)
|
|
292
300
|
for v in c.lineage.arguments
|
|
293
301
|
]
|
|
302
|
+
|
|
294
303
|
if cte.group_to_grain:
|
|
295
304
|
rval = f"{self.FUNCTION_MAP[c.lineage.operator](args)}"
|
|
296
305
|
else:
|
|
@@ -335,11 +344,11 @@ class BaseDialect:
|
|
|
335
344
|
Parenthetical,
|
|
336
345
|
AggregateWrapper,
|
|
337
346
|
MagicConstants,
|
|
347
|
+
MapWrapper[Any, Any],
|
|
348
|
+
MapType,
|
|
338
349
|
NumericType,
|
|
339
350
|
ListType,
|
|
340
|
-
ListWrapper[
|
|
341
|
-
ListWrapper[str],
|
|
342
|
-
ListWrapper[float],
|
|
351
|
+
ListWrapper[Any],
|
|
343
352
|
DatePart,
|
|
344
353
|
CaseWhen,
|
|
345
354
|
CaseElse,
|
|
@@ -435,6 +444,8 @@ class BaseDialect:
|
|
|
435
444
|
return str(e)
|
|
436
445
|
elif isinstance(e, ListWrapper):
|
|
437
446
|
return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}]"
|
|
447
|
+
elif isinstance(e, MapWrapper):
|
|
448
|
+
return f"MAP {{{','.join([f'{self.render_expr(k, cte=cte, cte_map=cte_map)}:{self.render_expr(v, cte=cte, cte_map=cte_map)}' for k, v in e.items()])}}}"
|
|
438
449
|
elif isinstance(e, list):
|
|
439
450
|
return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}]"
|
|
440
451
|
elif isinstance(e, DataType):
|
|
@@ -643,82 +654,20 @@ class BaseDialect:
|
|
|
643
654
|
f" {selected}"
|
|
644
655
|
)
|
|
645
656
|
|
|
646
|
-
# where assignment
|
|
647
|
-
output_where = False
|
|
648
|
-
if query.where_clause:
|
|
649
|
-
# found = False
|
|
650
|
-
filter = set(
|
|
651
|
-
[
|
|
652
|
-
str(x.address)
|
|
653
|
-
for x in query.where_clause.row_arguments
|
|
654
|
-
if not x.derivation == PurposeLineage.CONSTANT
|
|
655
|
-
]
|
|
656
|
-
)
|
|
657
|
-
query_output = set([str(z.address) for z in query.output_columns])
|
|
658
|
-
# if it wasn't an output
|
|
659
|
-
# we would have forced it up earlier and we don't need to render at this point
|
|
660
|
-
if filter.issubset(query_output):
|
|
661
|
-
output_where = True
|
|
662
|
-
for ex_set in query.where_clause.existence_arguments:
|
|
663
|
-
for c in ex_set:
|
|
664
|
-
if c.address not in cte_output_map:
|
|
665
|
-
cts = [
|
|
666
|
-
ct
|
|
667
|
-
for ct in query.ctes
|
|
668
|
-
if ct.name in query.base.existence_source_map[c.address]
|
|
669
|
-
]
|
|
670
|
-
if not cts:
|
|
671
|
-
raise ValueError(query.base.existence_source_map[c.address])
|
|
672
|
-
cte_output_map[c.address] = cts[0]
|
|
673
|
-
|
|
674
657
|
compiled_ctes = self.generate_ctes(query)
|
|
675
658
|
|
|
676
659
|
# restort selections by the order they were written in
|
|
677
660
|
sorted_select: List[str] = []
|
|
678
661
|
for output_c in output_addresses:
|
|
679
662
|
sorted_select.append(select_columns[output_c])
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
ctes=compiled_ctes[:-1],
|
|
689
|
-
)
|
|
690
|
-
else:
|
|
691
|
-
final = self.SQL_TEMPLATE.render(
|
|
692
|
-
output=(
|
|
693
|
-
query.output_to
|
|
694
|
-
if isinstance(query, ProcessedQueryPersist)
|
|
695
|
-
else None
|
|
696
|
-
),
|
|
697
|
-
select_columns=sorted_select,
|
|
698
|
-
base=query.base.name,
|
|
699
|
-
joins=[
|
|
700
|
-
render_join(join, self.QUOTE_CHARACTER, None)
|
|
701
|
-
for join in query.joins
|
|
702
|
-
],
|
|
703
|
-
ctes=compiled_ctes,
|
|
704
|
-
limit=query.limit,
|
|
705
|
-
# move up to CTEs
|
|
706
|
-
where=(
|
|
707
|
-
self.render_expr(
|
|
708
|
-
query.where_clause.conditional, cte_map=cte_output_map
|
|
709
|
-
)
|
|
710
|
-
if query.where_clause and output_where
|
|
711
|
-
else None
|
|
712
|
-
),
|
|
713
|
-
order_by=(
|
|
714
|
-
[
|
|
715
|
-
self.render_order_item(i, query.base, final=True)
|
|
716
|
-
for i in query.order_by.items
|
|
717
|
-
]
|
|
718
|
-
if query.order_by
|
|
719
|
-
else None
|
|
720
|
-
),
|
|
721
|
-
)
|
|
663
|
+
|
|
664
|
+
final = self.SQL_TEMPLATE.render(
|
|
665
|
+
output=(
|
|
666
|
+
query.output_to if isinstance(query, ProcessedQueryPersist) else None
|
|
667
|
+
),
|
|
668
|
+
full_select=compiled_ctes[-1].statement,
|
|
669
|
+
ctes=compiled_ctes[:-1],
|
|
670
|
+
)
|
|
722
671
|
|
|
723
672
|
if CONFIG.strict_mode and INVALID_REFERENCE_STRING(1) in final:
|
|
724
673
|
raise ValueError(
|
trilogy/dialect/presto.py
CHANGED
|
@@ -15,6 +15,7 @@ FUNCTION_MAP = {
|
|
|
15
15
|
FunctionType.LENGTH: lambda x: f"length({x[0]})",
|
|
16
16
|
FunctionType.AVG: lambda x: f"avg({x[0]})",
|
|
17
17
|
FunctionType.INDEX_ACCESS: lambda x: f"element_at({x[0]},{x[1]})",
|
|
18
|
+
FunctionType.MAP_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
|
|
18
19
|
FunctionType.LIKE: lambda x: (
|
|
19
20
|
f" CASE WHEN {x[0]} like {x[1]} THEN True ELSE False END"
|
|
20
21
|
),
|
trilogy/parsing/common.py
CHANGED
|
@@ -7,6 +7,7 @@ from trilogy.core.models import (
|
|
|
7
7
|
Metadata,
|
|
8
8
|
FilterItem,
|
|
9
9
|
ListWrapper,
|
|
10
|
+
MapWrapper,
|
|
10
11
|
WindowItem,
|
|
11
12
|
)
|
|
12
13
|
from typing import List, Tuple
|
|
@@ -41,7 +42,7 @@ def concept_list_to_keys(concepts: Tuple[Concept, ...]) -> Tuple[Concept, ...]:
|
|
|
41
42
|
|
|
42
43
|
|
|
43
44
|
def constant_to_concept(
|
|
44
|
-
parent: ListWrapper | list | int | float | str,
|
|
45
|
+
parent: ListWrapper | MapWrapper | list | int | float | str,
|
|
45
46
|
name: str,
|
|
46
47
|
namespace: str,
|
|
47
48
|
purpose: Purpose | None = None,
|
|
@@ -53,6 +54,7 @@ def constant_to_concept(
|
|
|
53
54
|
output_purpose=Purpose.CONSTANT,
|
|
54
55
|
arguments=[parent],
|
|
55
56
|
)
|
|
57
|
+
assert const_function.arguments[0] == parent, const_function.arguments[0]
|
|
56
58
|
fmetadata = metadata or Metadata()
|
|
57
59
|
return Concept(
|
|
58
60
|
name=name,
|
|
@@ -186,6 +188,7 @@ def arbitrary_to_concept(
|
|
|
186
188
|
| FilterItem
|
|
187
189
|
| Function
|
|
188
190
|
| ListWrapper
|
|
191
|
+
| MapWrapper
|
|
189
192
|
| int
|
|
190
193
|
| float
|
|
191
194
|
| str
|
trilogy/parsing/parse_engine.py
CHANGED
|
@@ -43,6 +43,7 @@ from trilogy.core.functions import (
|
|
|
43
43
|
Min,
|
|
44
44
|
Split,
|
|
45
45
|
IndexAccess,
|
|
46
|
+
MapAccess,
|
|
46
47
|
AttrAccess,
|
|
47
48
|
Abs,
|
|
48
49
|
Unnest,
|
|
@@ -94,6 +95,7 @@ from trilogy.core.models import (
|
|
|
94
95
|
RawColumnExpr,
|
|
95
96
|
arg_to_datatype,
|
|
96
97
|
ListWrapper,
|
|
98
|
+
MapWrapper,
|
|
97
99
|
MapType,
|
|
98
100
|
ShowStatement,
|
|
99
101
|
DataType,
|
|
@@ -104,6 +106,7 @@ from trilogy.core.models import (
|
|
|
104
106
|
RowsetDerivationStatement,
|
|
105
107
|
LooseConceptList,
|
|
106
108
|
list_to_wrapper,
|
|
109
|
+
dict_to_map_wrapper,
|
|
107
110
|
NumericType,
|
|
108
111
|
)
|
|
109
112
|
from trilogy.parsing.exceptions import ParseError
|
|
@@ -117,7 +120,7 @@ from trilogy.parsing.common import (
|
|
|
117
120
|
arbitrary_to_concept,
|
|
118
121
|
)
|
|
119
122
|
|
|
120
|
-
CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper)
|
|
123
|
+
CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper, MapWrapper)
|
|
121
124
|
|
|
122
125
|
with open(join(dirname(__file__), "trilogy.lark"), "r") as f:
|
|
123
126
|
PARSER = Lark(
|
|
@@ -253,7 +256,7 @@ class ParseToObjects(Transformer):
|
|
|
253
256
|
self.environment.add_concept(concept, meta=meta)
|
|
254
257
|
final.append(concept)
|
|
255
258
|
elif isinstance(
|
|
256
|
-
arg, (FilterItem, WindowItem, AggregateWrapper, ListWrapper)
|
|
259
|
+
arg, (FilterItem, WindowItem, AggregateWrapper, ListWrapper, MapWrapper)
|
|
257
260
|
):
|
|
258
261
|
id_hash = string_to_hash(str(arg))
|
|
259
262
|
concept = arbitrary_to_concept(
|
|
@@ -330,7 +333,12 @@ class ParseToObjects(Transformer):
|
|
|
330
333
|
def numeric_type(self, args) -> NumericType:
|
|
331
334
|
return NumericType(precision=args[0], scale=args[1])
|
|
332
335
|
|
|
333
|
-
def
|
|
336
|
+
def map_type(self, args) -> MapType:
|
|
337
|
+
return MapType(key_type=args[0], value_type=args[1])
|
|
338
|
+
|
|
339
|
+
def data_type(
|
|
340
|
+
self, args
|
|
341
|
+
) -> DataType | ListType | StructType | MapType | NumericType:
|
|
334
342
|
resolved = args[0]
|
|
335
343
|
if isinstance(resolved, StructType):
|
|
336
344
|
return resolved
|
|
@@ -338,6 +346,8 @@ class ParseToObjects(Transformer):
|
|
|
338
346
|
return resolved
|
|
339
347
|
elif isinstance(resolved, NumericType):
|
|
340
348
|
return resolved
|
|
349
|
+
elif isinstance(resolved, MapType):
|
|
350
|
+
return resolved
|
|
341
351
|
return DataType(args[0].lower())
|
|
342
352
|
|
|
343
353
|
def array_comparison(self, args) -> ComparisonOperator:
|
|
@@ -490,7 +500,6 @@ class ParseToObjects(Transformer):
|
|
|
490
500
|
# we need to strip off every parenthetical to see what is being assigned.
|
|
491
501
|
while isinstance(source_value, Parenthetical):
|
|
492
502
|
source_value = source_value.content
|
|
493
|
-
|
|
494
503
|
if isinstance(
|
|
495
504
|
source_value, (FilterItem, WindowItem, AggregateWrapper, Function)
|
|
496
505
|
):
|
|
@@ -549,7 +558,7 @@ class ParseToObjects(Transformer):
|
|
|
549
558
|
else:
|
|
550
559
|
metadata = None
|
|
551
560
|
name = args[1]
|
|
552
|
-
constant: Union[str, float, int, bool] = args[2]
|
|
561
|
+
constant: Union[str, float, int, bool, MapWrapper, ListWrapper] = args[2]
|
|
553
562
|
lookup, namespace, name, parent = parse_concept_reference(
|
|
554
563
|
name, self.environment
|
|
555
564
|
)
|
|
@@ -605,6 +614,7 @@ class ParseToObjects(Transformer):
|
|
|
605
614
|
columns: List[ColumnAssignment] = args[1]
|
|
606
615
|
grain: Optional[Grain] = None
|
|
607
616
|
address: Optional[Address] = None
|
|
617
|
+
where: Optional[WhereClause] = None
|
|
608
618
|
for val in args[1:]:
|
|
609
619
|
if isinstance(val, Address):
|
|
610
620
|
address = val
|
|
@@ -612,6 +622,8 @@ class ParseToObjects(Transformer):
|
|
|
612
622
|
grain = val
|
|
613
623
|
elif isinstance(val, Query):
|
|
614
624
|
address = Address(location=f"({val.text})", is_query=True)
|
|
625
|
+
elif isinstance(val, WhereClause):
|
|
626
|
+
where = val
|
|
615
627
|
if not address:
|
|
616
628
|
raise ValueError(
|
|
617
629
|
"Malformed datasource, missing address or query declaration"
|
|
@@ -624,6 +636,7 @@ class ParseToObjects(Transformer):
|
|
|
624
636
|
grain=grain, # type: ignore
|
|
625
637
|
address=address,
|
|
626
638
|
namespace=self.environment.namespace,
|
|
639
|
+
where=where,
|
|
627
640
|
)
|
|
628
641
|
for column in columns:
|
|
629
642
|
column.concept = column.concept.with_grain(datasource.grain)
|
|
@@ -1003,6 +1016,11 @@ class ParseToObjects(Transformer):
|
|
|
1003
1016
|
def array_lit(self, args):
|
|
1004
1017
|
return list_to_wrapper(args)
|
|
1005
1018
|
|
|
1019
|
+
def map_lit(self, args):
|
|
1020
|
+
parsed = dict(zip(args[::2], args[1::2]))
|
|
1021
|
+
wrapped = dict_to_map_wrapper(parsed)
|
|
1022
|
+
return wrapped
|
|
1023
|
+
|
|
1006
1024
|
def literal(self, args):
|
|
1007
1025
|
return args[0]
|
|
1008
1026
|
|
|
@@ -1140,6 +1158,8 @@ class ParseToObjects(Transformer):
|
|
|
1140
1158
|
@v_args(meta=True)
|
|
1141
1159
|
def index_access(self, meta, args):
|
|
1142
1160
|
args = self.process_function_args(args, meta=meta)
|
|
1161
|
+
if args[0].datatype == DataType.MAP or isinstance(args[0].datatype, MapType):
|
|
1162
|
+
return MapAccess(args)
|
|
1143
1163
|
return IndexAccess(args)
|
|
1144
1164
|
|
|
1145
1165
|
@v_args(meta=True)
|