pytrilogy 0.0.2.1__py3-none-any.whl → 0.0.2.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pytrilogy might be problematic. Click here for more details.

@@ -44,7 +44,7 @@ def gen_basic_node(
44
44
  list(optional_set) + [concept],
45
45
  )
46
46
  )
47
-
47
+ # check for the concept by itself
48
48
  for attempt, basic_output in reversed(attempts):
49
49
  partials = []
50
50
  attempt = unique(attempt, "address")
@@ -4,7 +4,6 @@ from trilogy.core.models import Concept, Environment, Conditional
4
4
  from trilogy.core.processing.nodes import MergeNode, History, StrategyNode
5
5
  import networkx as nx
6
6
  from trilogy.core.graph_models import concept_to_node
7
- from trilogy.core.processing.utility import PathInfo
8
7
  from trilogy.constants import logger
9
8
  from trilogy.utility import unique
10
9
  from trilogy.core.exceptions import AmbiguousRelationshipResolutionException
@@ -63,7 +62,9 @@ def extract_ds_components(g: nx.DiGraph, nodelist: list[str]) -> list[list[str]]
63
62
  if not str(x).startswith("ds~")
64
63
  ]
65
64
  )
66
-
65
+ # if we had no ego graphs, return all concepts
66
+ if not graphs:
67
+ return [[extract_address(node) for node in nodelist]]
67
68
  graphs = filter_unique_graphs(graphs)
68
69
  for node in nodelist:
69
70
  parsed = extract_address(node)
@@ -82,6 +83,7 @@ def determine_induced_minimal_nodes(
82
83
  H: nx.Graph = nx.to_undirected(G).copy()
83
84
  nodes_to_remove = []
84
85
  concepts = nx.get_node_attributes(G, "concept")
86
+
85
87
  for node in G.nodes:
86
88
  if concepts.get(node):
87
89
  lookup = concepts[node]
@@ -107,9 +109,11 @@ def determine_induced_minimal_nodes(
107
109
  paths = nx.multi_source_dijkstra_path(H, nodelist)
108
110
  except nx.exception.NodeNotFound:
109
111
  return None
112
+
110
113
  H.remove_nodes_from(list(x for x in H.nodes if x not in paths))
111
114
  sG: nx.Graph = ax.steinertree.steiner_tree(H, nodelist).copy()
112
115
  final: nx.DiGraph = nx.subgraph(G, sG.nodes).copy()
116
+
113
117
  for edge in G.edges:
114
118
  if edge[1] in final.nodes and edge[0].startswith("ds~"):
115
119
  ds_name = extract_address(edge[0])
@@ -125,6 +129,7 @@ def determine_induced_minimal_nodes(
125
129
  [final.in_degree(node) > 0 for node in final.nodes if node.startswith("c~")]
126
130
  ):
127
131
  return None
132
+
128
133
  if not all([node in final.nodes for node in nodelist]):
129
134
  return None
130
135
  return final
@@ -308,111 +313,44 @@ def gen_merge_node(
308
313
  history: History | None = None,
309
314
  conditions: Conditional | None = None,
310
315
  ) -> Optional[MergeNode]:
311
- join_candidates: List[PathInfo] = []
312
-
313
- # inject new concepts into search, and identify if two dses can reach there
314
- if not join_candidates:
315
- for filter_downstream in [True, False]:
316
- weak_resolve = resolve_weak_components(
317
- all_concepts,
318
- environment,
319
- g,
320
- filter_downstream=filter_downstream,
321
- accept_partial=accept_partial,
322
- )
323
- if weak_resolve:
324
- log_graph = [[y.address for y in x] for x in weak_resolve]
325
- logger.info(
326
- f"{padding(depth)}{LOGGER_PREFIX} Was able to resolve graph through weak component resolution - final graph {log_graph}"
327
- )
328
- return subgraphs_to_merge_node(
329
- weak_resolve,
330
- depth=depth,
331
- all_concepts=all_concepts,
332
- environment=environment,
333
- g=g,
334
- source_concepts=source_concepts,
335
- history=history,
336
- conditions=conditions,
337
- )
338
- if not join_candidates:
339
- return None
340
- join_additions: list[set[str]] = []
341
- for candidate in join_candidates:
342
- join_additions.append(candidate.reduced_concepts)
343
-
344
- common: set[str] = set()
345
- final_candidates: list[set[str]] = []
346
- # find all values that show up in every join_additions
347
- for ja in join_additions:
348
- if not common:
349
- common = ja
350
- else:
351
- common = common.intersection(ja)
352
- if all(ja.issubset(y) for y in join_additions):
353
- final_candidates.append(ja)
354
316
 
355
- if not final_candidates:
356
- filtered_paths = [x.difference(common) for x in join_additions]
357
- raise AmbiguousRelationshipResolutionException(
358
- f"Ambiguous concept join resolution fetching {[x.address for x in all_concepts]} - unique values in possible paths = {filtered_paths}. Include an additional concept to disambiguate",
359
- join_additions,
360
- )
361
- if not join_candidates:
362
- logger.info(
363
- f"{padding(depth)}{LOGGER_PREFIX} No additional join candidates could be found"
317
+ for filter_downstream in [True, False]:
318
+ weak_resolve = resolve_weak_components(
319
+ all_concepts,
320
+ environment,
321
+ g,
322
+ filter_downstream=filter_downstream,
323
+ accept_partial=accept_partial,
364
324
  )
365
- return None
366
- shortest: PathInfo = sorted(
367
- [x for x in join_candidates if x.reduced_concepts in final_candidates],
368
- key=lambda x: len(x.reduced_concepts),
369
- )[0]
370
- logger.info(f"{padding(depth)}{LOGGER_PREFIX} final path is {shortest.paths}")
371
-
372
- return subgraphs_to_merge_node(
373
- shortest.concept_subgraphs,
374
- depth=depth,
375
- all_concepts=all_concepts,
376
- environment=environment,
377
- g=g,
378
- source_concepts=source_concepts,
379
- history=history,
380
- conditions=conditions,
381
- )
382
- # parents = []
383
- # for graph in shortest.concept_subgraphs:
384
- # logger.info(
385
- # f"{padding(depth)}{LOGGER_PREFIX} fetching subgraph {[c.address for c in graph]}"
386
- # )
387
- # parent = source_concepts(
388
- # mandatory_list=graph,
389
- # environment=environment,
390
- # g=g,
391
- # depth=depth + 1,
392
- # history=history,
393
- # )
394
- # if not parent:
395
- # logger.info(
396
- # f"{padding(depth)}{LOGGER_PREFIX} Unable to instantiate target subgraph"
397
- # )
398
- # return None
399
- # logger.info(
400
- # f"{padding(depth)}{LOGGER_PREFIX} finished subgraph fetch for {[c.address for c in graph]}, have parent {type(parent)}"
401
- # )
402
- # parents.append(parent)
403
-
404
- # return MergeNode(
405
- # input_concepts=[
406
- # environment.concepts[x]
407
- # for x in shortest.reduced_concepts
408
- # if environment.concepts[x].derivation != PurposeLineage.MERGE
409
- # ],
410
- # output_concepts=[
411
- # x for x in all_concepts if x.derivation != PurposeLineage.MERGE
412
- # ],
413
- # environment=environment,
414
- # g=g,
415
- # parents=parents,
416
- # depth=depth,
417
- # conditions=conditions,
418
- # )
325
+ if weak_resolve:
326
+ log_graph = [[y.address for y in x] for x in weak_resolve]
327
+ logger.info(
328
+ f"{padding(depth)}{LOGGER_PREFIX} Was able to resolve graph through weak component resolution - final graph {log_graph}"
329
+ )
330
+ return subgraphs_to_merge_node(
331
+ weak_resolve,
332
+ depth=depth,
333
+ all_concepts=all_concepts,
334
+ environment=environment,
335
+ g=g,
336
+ source_concepts=source_concepts,
337
+ history=history,
338
+ conditions=conditions,
339
+ )
340
+ # one concept handling may need to be kicked to alias
341
+ if len(all_concepts) == 1:
342
+ concept = all_concepts[0]
343
+ for k, v in concept.pseudonyms.items():
344
+ test = subgraphs_to_merge_node(
345
+ [[concept, v]],
346
+ g=g,
347
+ all_concepts=[concept],
348
+ environment=environment,
349
+ depth=depth,
350
+ source_concepts=source_concepts,
351
+ history=history,
352
+ conditions=conditions,
353
+ )
354
+ if test:
355
+ return test
356
+ return None
@@ -89,6 +89,7 @@ def dm_to_strategy_node(
89
89
  accept_partial=accept_partial,
90
90
  datasource=datasource,
91
91
  grain=datasource.grain,
92
+ conditions=datasource.where.conditional if datasource.where else None,
92
93
  )
93
94
  # we need to nest the group node one further
94
95
  if force_group is True:
@@ -295,9 +296,6 @@ def gen_select_node_from_table(
295
296
  g.nodes[ncandidate]
296
297
  except KeyError:
297
298
  raise nx.exception.NetworkXNoPath
298
- raise SyntaxError(
299
- f"Could not find node {ncandidate}, have {list(g.nodes())}"
300
- )
301
299
  raise e
302
300
  except nx.exception.NetworkXNoPath:
303
301
  all_found = False
@@ -372,6 +370,7 @@ def gen_select_node_from_table(
372
370
  accept_partial=accept_partial,
373
371
  datasource=datasource,
374
372
  grain=Grain(components=all_concepts),
373
+ conditions=datasource.where.conditional if datasource.where else None,
375
374
  )
376
375
  # we need to nest the group node one further
377
376
  if force_group is True:
@@ -124,15 +124,20 @@ def resolve_join_order(joins: List[BaseJoin]) -> List[BaseJoin]:
124
124
  return final_joins
125
125
 
126
126
 
127
- def add_node_join_concept(graph, concept, datasource, concepts):
128
- # we don't need to join on a concept if all of the keys exist in the grain
129
- # if concept.keys and all([x in grain for x in concept.keys]):
130
- # continue
127
+ def add_node_join_concept(
128
+ graph: nx.DiGraph,
129
+ concept: Concept,
130
+ datasource: Datasource | QueryDatasource,
131
+ concepts: List[Concept],
132
+ ):
133
+
131
134
  concepts.append(concept)
132
135
 
133
136
  graph.add_node(concept.address, type=NodeType.CONCEPT)
134
137
  graph.add_edge(datasource.identifier, concept.address)
135
- for k, v in concept.pseudonyms.items():
138
+ for _, v in concept.pseudonyms.items():
139
+ if v in concepts:
140
+ continue
136
141
  if v.address != concept.address:
137
142
  add_node_join_concept(graph, v, datasource, concepts)
138
143
 
@@ -149,13 +154,6 @@ def get_node_joins(
149
154
  graph.add_node(datasource.identifier, type=NodeType.NODE)
150
155
  for concept in datasource.output_concepts:
151
156
  add_node_join_concept(graph, concept, datasource, concepts)
152
- # we don't need to join on a concept if all of the keys exist in the grain
153
- # if concept.keys and all([x in grain for x in concept.keys]):
154
- # continue
155
- # concepts.append(concept)
156
-
157
- # graph.add_node(concept.address, type=NodeType.CONCEPT)
158
- # graph.add_edge(datasource.identifier, concept.address)
159
157
 
160
158
  # add edges for every constant to every datasource
161
159
  for datasource in datasources:
@@ -195,26 +193,6 @@ def get_node_joins(
195
193
  ),
196
194
  )
197
195
 
198
- node_map = {
199
- x[0:20]: len(
200
- [
201
- partial
202
- for partial in identifier_map[x].partial_concepts
203
- if partial in grain
204
- ]
205
- + [
206
- output
207
- for output in identifier_map[x].output_concepts
208
- if output.address in grain_pseudonyms
209
- ]
210
- )
211
- for x in node_list
212
- }
213
- print("NODE MAP")
214
- print(node_map)
215
- print([x.address for x in grain])
216
- print(grain_pseudonyms)
217
-
218
196
  for left in node_list:
219
197
  # the constant dataset is a special case
220
198
  # and can never be on the left of a join
@@ -467,9 +467,11 @@ def process_query(
467
467
  for cte in raw_ctes:
468
468
  cte.parent_ctes = [seen[x.name] for x in cte.parent_ctes]
469
469
  deduped_ctes: List[CTE] = list(seen.values())
470
+ root_cte.order_by = statement.order_by
471
+ root_cte.limit = statement.limit
472
+ root_cte.hidden_concepts = [x for x in statement.hidden_components]
470
473
 
471
474
  final_ctes = optimize_ctes(deduped_ctes, root_cte, statement)
472
-
473
475
  return ProcessedQuery(
474
476
  order_by=statement.order_by,
475
477
  grain=statement.grain,
trilogy/dialect/base.py CHANGED
@@ -8,7 +8,6 @@ from trilogy.core.enums import (
8
8
  FunctionType,
9
9
  WindowType,
10
10
  DatePart,
11
- PurposeLineage,
12
11
  ComparisonOperator,
13
12
  )
14
13
  from trilogy.core.models import (
@@ -36,6 +35,7 @@ from trilogy.core.models import (
36
35
  Environment,
37
36
  RawColumnExpr,
38
37
  ListWrapper,
38
+ MapWrapper,
39
39
  ShowStatement,
40
40
  RowsetItem,
41
41
  MultiSelectStatement,
@@ -45,6 +45,7 @@ from trilogy.core.models import (
45
45
  RawSQLStatement,
46
46
  ProcessedRawSQLStatement,
47
47
  NumericType,
48
+ MapType,
48
49
  MergeStatementV2,
49
50
  )
50
51
  from trilogy.core.query_processor import process_query, process_persist
@@ -97,6 +98,7 @@ DATATYPE_MAP = {
97
98
  DataType.FLOAT: "float",
98
99
  DataType.BOOL: "bool",
99
100
  DataType.NUMERIC: "numeric",
101
+ DataType.MAP: "map",
100
102
  }
101
103
 
102
104
 
@@ -116,6 +118,7 @@ FUNCTION_MAP = {
116
118
  FunctionType.IS_NULL: lambda x: f"isnull({x[0]})",
117
119
  # complex
118
120
  FunctionType.INDEX_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
121
+ FunctionType.MAP_ACCESS: lambda x: f"{x[0]}[{x[1]}][1]",
119
122
  FunctionType.UNNEST: lambda x: f"unnest({x[0]})",
120
123
  # math
121
124
  FunctionType.ADD: lambda x: f"{x[0]} + {x[1]}",
@@ -189,14 +192,13 @@ TOP {{ limit }}{% endif %}
189
192
  \t{{ select }}{% if not loop.last %},{% endif %}{% endfor %}
190
193
  {% if base %}FROM
191
194
  \t{{ base }}{% endif %}{% if joins %}{% for join in joins %}
192
- \t{{ join }}{% endfor %}{% endif %}
193
- {% if where %}WHERE
194
- \t{{ where }}
195
- {% endif %}{%- if group_by %}GROUP BY {% for group in group_by %}
196
- \t{{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
197
- {%- if order_by %}
198
- ORDER BY {% for order in order_by %}
199
- {{ order }}{% if not loop.last %},{% endif %}{% endfor %}
195
+ \t{{ join }}{% endfor %}{% endif %}{% if where %}
196
+ WHERE
197
+ \t{{ where }}{% endif %}{%- if group_by %}
198
+ GROUP BY {% for group in group_by %}
199
+ \t{{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{%- if order_by %}
200
+ ORDER BY{% for order in order_by %}
201
+ \t{{ order }}{% if not loop.last %},{% endif %}{% endfor %}
200
202
  {% endif %}{% endif %}
201
203
  """
202
204
  )
@@ -214,7 +216,13 @@ def safe_get_cte_value(coalesce, cte: CTE, c: Concept, quote_char: str):
214
216
  raw = cte.source_map.get(address, None)
215
217
 
216
218
  if not raw:
217
- return INVALID_REFERENCE_STRING("Missing source reference")
219
+ for k, v in c.pseudonyms.items():
220
+ if cte.source_map.get(k):
221
+ c = v
222
+ raw = cte.source_map[k]
223
+ break
224
+ if not raw:
225
+ return INVALID_REFERENCE_STRING("Missing source reference")
218
226
  if isinstance(raw, str):
219
227
  rendered = cte.get_alias(c, raw)
220
228
  return f"{raw}.{safe_quote(rendered, quote_char)}"
@@ -291,6 +299,7 @@ class BaseDialect:
291
299
  self.render_expr(v, cte) # , alias=False)
292
300
  for v in c.lineage.arguments
293
301
  ]
302
+
294
303
  if cte.group_to_grain:
295
304
  rval = f"{self.FUNCTION_MAP[c.lineage.operator](args)}"
296
305
  else:
@@ -335,11 +344,11 @@ class BaseDialect:
335
344
  Parenthetical,
336
345
  AggregateWrapper,
337
346
  MagicConstants,
347
+ MapWrapper[Any, Any],
348
+ MapType,
338
349
  NumericType,
339
350
  ListType,
340
- ListWrapper[int],
341
- ListWrapper[str],
342
- ListWrapper[float],
351
+ ListWrapper[Any],
343
352
  DatePart,
344
353
  CaseWhen,
345
354
  CaseElse,
@@ -435,6 +444,8 @@ class BaseDialect:
435
444
  return str(e)
436
445
  elif isinstance(e, ListWrapper):
437
446
  return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}]"
447
+ elif isinstance(e, MapWrapper):
448
+ return f"MAP {{{','.join([f'{self.render_expr(k, cte=cte, cte_map=cte_map)}:{self.render_expr(v, cte=cte, cte_map=cte_map)}' for k, v in e.items()])}}}"
438
449
  elif isinstance(e, list):
439
450
  return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}]"
440
451
  elif isinstance(e, DataType):
@@ -643,82 +654,20 @@ class BaseDialect:
643
654
  f" {selected}"
644
655
  )
645
656
 
646
- # where assignment
647
- output_where = False
648
- if query.where_clause:
649
- # found = False
650
- filter = set(
651
- [
652
- str(x.address)
653
- for x in query.where_clause.row_arguments
654
- if not x.derivation == PurposeLineage.CONSTANT
655
- ]
656
- )
657
- query_output = set([str(z.address) for z in query.output_columns])
658
- # if it wasn't an output
659
- # we would have forced it up earlier and we don't need to render at this point
660
- if filter.issubset(query_output):
661
- output_where = True
662
- for ex_set in query.where_clause.existence_arguments:
663
- for c in ex_set:
664
- if c.address not in cte_output_map:
665
- cts = [
666
- ct
667
- for ct in query.ctes
668
- if ct.name in query.base.existence_source_map[c.address]
669
- ]
670
- if not cts:
671
- raise ValueError(query.base.existence_source_map[c.address])
672
- cte_output_map[c.address] = cts[0]
673
-
674
657
  compiled_ctes = self.generate_ctes(query)
675
658
 
676
659
  # restort selections by the order they were written in
677
660
  sorted_select: List[str] = []
678
661
  for output_c in output_addresses:
679
662
  sorted_select.append(select_columns[output_c])
680
- if not query.base.requires_nesting:
681
- final = self.SQL_TEMPLATE.render(
682
- output=(
683
- query.output_to
684
- if isinstance(query, ProcessedQueryPersist)
685
- else None
686
- ),
687
- full_select=compiled_ctes[-1].statement,
688
- ctes=compiled_ctes[:-1],
689
- )
690
- else:
691
- final = self.SQL_TEMPLATE.render(
692
- output=(
693
- query.output_to
694
- if isinstance(query, ProcessedQueryPersist)
695
- else None
696
- ),
697
- select_columns=sorted_select,
698
- base=query.base.name,
699
- joins=[
700
- render_join(join, self.QUOTE_CHARACTER, None)
701
- for join in query.joins
702
- ],
703
- ctes=compiled_ctes,
704
- limit=query.limit,
705
- # move up to CTEs
706
- where=(
707
- self.render_expr(
708
- query.where_clause.conditional, cte_map=cte_output_map
709
- )
710
- if query.where_clause and output_where
711
- else None
712
- ),
713
- order_by=(
714
- [
715
- self.render_order_item(i, query.base, final=True)
716
- for i in query.order_by.items
717
- ]
718
- if query.order_by
719
- else None
720
- ),
721
- )
663
+
664
+ final = self.SQL_TEMPLATE.render(
665
+ output=(
666
+ query.output_to if isinstance(query, ProcessedQueryPersist) else None
667
+ ),
668
+ full_select=compiled_ctes[-1].statement,
669
+ ctes=compiled_ctes[:-1],
670
+ )
722
671
 
723
672
  if CONFIG.strict_mode and INVALID_REFERENCE_STRING(1) in final:
724
673
  raise ValueError(
trilogy/dialect/presto.py CHANGED
@@ -15,6 +15,7 @@ FUNCTION_MAP = {
15
15
  FunctionType.LENGTH: lambda x: f"length({x[0]})",
16
16
  FunctionType.AVG: lambda x: f"avg({x[0]})",
17
17
  FunctionType.INDEX_ACCESS: lambda x: f"element_at({x[0]},{x[1]})",
18
+ FunctionType.MAP_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
18
19
  FunctionType.LIKE: lambda x: (
19
20
  f" CASE WHEN {x[0]} like {x[1]} THEN True ELSE False END"
20
21
  ),
trilogy/parsing/common.py CHANGED
@@ -7,6 +7,7 @@ from trilogy.core.models import (
7
7
  Metadata,
8
8
  FilterItem,
9
9
  ListWrapper,
10
+ MapWrapper,
10
11
  WindowItem,
11
12
  )
12
13
  from typing import List, Tuple
@@ -41,7 +42,7 @@ def concept_list_to_keys(concepts: Tuple[Concept, ...]) -> Tuple[Concept, ...]:
41
42
 
42
43
 
43
44
  def constant_to_concept(
44
- parent: ListWrapper | list | int | float | str,
45
+ parent: ListWrapper | MapWrapper | list | int | float | str,
45
46
  name: str,
46
47
  namespace: str,
47
48
  purpose: Purpose | None = None,
@@ -53,6 +54,7 @@ def constant_to_concept(
53
54
  output_purpose=Purpose.CONSTANT,
54
55
  arguments=[parent],
55
56
  )
57
+ assert const_function.arguments[0] == parent, const_function.arguments[0]
56
58
  fmetadata = metadata or Metadata()
57
59
  return Concept(
58
60
  name=name,
@@ -186,6 +188,7 @@ def arbitrary_to_concept(
186
188
  | FilterItem
187
189
  | Function
188
190
  | ListWrapper
191
+ | MapWrapper
189
192
  | int
190
193
  | float
191
194
  | str
@@ -43,6 +43,7 @@ from trilogy.core.functions import (
43
43
  Min,
44
44
  Split,
45
45
  IndexAccess,
46
+ MapAccess,
46
47
  AttrAccess,
47
48
  Abs,
48
49
  Unnest,
@@ -94,6 +95,7 @@ from trilogy.core.models import (
94
95
  RawColumnExpr,
95
96
  arg_to_datatype,
96
97
  ListWrapper,
98
+ MapWrapper,
97
99
  MapType,
98
100
  ShowStatement,
99
101
  DataType,
@@ -104,6 +106,7 @@ from trilogy.core.models import (
104
106
  RowsetDerivationStatement,
105
107
  LooseConceptList,
106
108
  list_to_wrapper,
109
+ dict_to_map_wrapper,
107
110
  NumericType,
108
111
  )
109
112
  from trilogy.parsing.exceptions import ParseError
@@ -117,7 +120,7 @@ from trilogy.parsing.common import (
117
120
  arbitrary_to_concept,
118
121
  )
119
122
 
120
- CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper)
123
+ CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper, MapWrapper)
121
124
 
122
125
  with open(join(dirname(__file__), "trilogy.lark"), "r") as f:
123
126
  PARSER = Lark(
@@ -253,7 +256,7 @@ class ParseToObjects(Transformer):
253
256
  self.environment.add_concept(concept, meta=meta)
254
257
  final.append(concept)
255
258
  elif isinstance(
256
- arg, (FilterItem, WindowItem, AggregateWrapper, ListWrapper)
259
+ arg, (FilterItem, WindowItem, AggregateWrapper, ListWrapper, MapWrapper)
257
260
  ):
258
261
  id_hash = string_to_hash(str(arg))
259
262
  concept = arbitrary_to_concept(
@@ -330,7 +333,12 @@ class ParseToObjects(Transformer):
330
333
  def numeric_type(self, args) -> NumericType:
331
334
  return NumericType(precision=args[0], scale=args[1])
332
335
 
333
- def data_type(self, args) -> DataType | ListType | StructType | NumericType:
336
+ def map_type(self, args) -> MapType:
337
+ return MapType(key_type=args[0], value_type=args[1])
338
+
339
+ def data_type(
340
+ self, args
341
+ ) -> DataType | ListType | StructType | MapType | NumericType:
334
342
  resolved = args[0]
335
343
  if isinstance(resolved, StructType):
336
344
  return resolved
@@ -338,6 +346,8 @@ class ParseToObjects(Transformer):
338
346
  return resolved
339
347
  elif isinstance(resolved, NumericType):
340
348
  return resolved
349
+ elif isinstance(resolved, MapType):
350
+ return resolved
341
351
  return DataType(args[0].lower())
342
352
 
343
353
  def array_comparison(self, args) -> ComparisonOperator:
@@ -490,7 +500,6 @@ class ParseToObjects(Transformer):
490
500
  # we need to strip off every parenthetical to see what is being assigned.
491
501
  while isinstance(source_value, Parenthetical):
492
502
  source_value = source_value.content
493
-
494
503
  if isinstance(
495
504
  source_value, (FilterItem, WindowItem, AggregateWrapper, Function)
496
505
  ):
@@ -549,7 +558,7 @@ class ParseToObjects(Transformer):
549
558
  else:
550
559
  metadata = None
551
560
  name = args[1]
552
- constant: Union[str, float, int, bool] = args[2]
561
+ constant: Union[str, float, int, bool, MapWrapper, ListWrapper] = args[2]
553
562
  lookup, namespace, name, parent = parse_concept_reference(
554
563
  name, self.environment
555
564
  )
@@ -605,6 +614,7 @@ class ParseToObjects(Transformer):
605
614
  columns: List[ColumnAssignment] = args[1]
606
615
  grain: Optional[Grain] = None
607
616
  address: Optional[Address] = None
617
+ where: Optional[WhereClause] = None
608
618
  for val in args[1:]:
609
619
  if isinstance(val, Address):
610
620
  address = val
@@ -612,6 +622,8 @@ class ParseToObjects(Transformer):
612
622
  grain = val
613
623
  elif isinstance(val, Query):
614
624
  address = Address(location=f"({val.text})", is_query=True)
625
+ elif isinstance(val, WhereClause):
626
+ where = val
615
627
  if not address:
616
628
  raise ValueError(
617
629
  "Malformed datasource, missing address or query declaration"
@@ -624,6 +636,7 @@ class ParseToObjects(Transformer):
624
636
  grain=grain, # type: ignore
625
637
  address=address,
626
638
  namespace=self.environment.namespace,
639
+ where=where,
627
640
  )
628
641
  for column in columns:
629
642
  column.concept = column.concept.with_grain(datasource.grain)
@@ -1003,6 +1016,11 @@ class ParseToObjects(Transformer):
1003
1016
  def array_lit(self, args):
1004
1017
  return list_to_wrapper(args)
1005
1018
 
1019
+ def map_lit(self, args):
1020
+ parsed = dict(zip(args[::2], args[1::2]))
1021
+ wrapped = dict_to_map_wrapper(parsed)
1022
+ return wrapped
1023
+
1006
1024
  def literal(self, args):
1007
1025
  return args[0]
1008
1026
 
@@ -1140,6 +1158,8 @@ class ParseToObjects(Transformer):
1140
1158
  @v_args(meta=True)
1141
1159
  def index_access(self, meta, args):
1142
1160
  args = self.process_function_args(args, meta=meta)
1161
+ if args[0].datatype == DataType.MAP or isinstance(args[0].datatype, MapType):
1162
+ return MapAccess(args)
1143
1163
  return IndexAccess(args)
1144
1164
 
1145
1165
  @v_args(meta=True)