pytrilogy 0.0.2.2__py3-none-any.whl → 0.0.2.3__py3-none-any.whl

{pytrilogy-0.0.2.2.dist-info → pytrilogy-0.0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.2
+Version: 0.0.2.3
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -247,7 +247,7 @@ N/A, only supports default auth. In python you can pass in a custom client.
 
 
 > [!TIP]
-> The CLI can also be used for formatting. PreQL has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
+> The CLI can also be used for formatting. Trilogy has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
 
 
 ## More Examples
@@ -284,7 +284,7 @@ but all are worth checking out. Please open PRs/comment for anything missed!
 
 #### CONCEPT
 
-Types: `string | int | float | bool | date | datetime | time | timestamp | interval`;
+Types: `string | int | float | bool | date | datetime | time | numeric(scale, precision) | timestamp | interval`;
 
 Key:
 `key <name> <type>;`

{pytrilogy-0.0.2.2.dist-info → pytrilogy-0.0.2.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-trilogy/__init__.py,sha256=mSeU_j02wb5aKd5vDcHQJdZaoG1AB96bCjUn2tIVpwk,290
+trilogy/__init__.py,sha256=P6eUqLnB3qFOCWULAL337Zc9648DuD0E0pGbTqzHChw,290
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=KIvi-cgU4R9urNgDdGiCsRkCrzjAfM4xGHhZb1SVy2w,881
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
@@ -8,34 +8,34 @@ trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=DWKPuShM_DbyAenjNFOAxcIH2l0QC9WgIwqZKPQqk_o,5850
-trilogy/core/env_processor.py,sha256=Wpy-iiduBwHntTsQTYWBO1O0i3Ij9_VoL3d4IWDEoj4,2126
+trilogy/core/enums.py,sha256=zf5VvxkXB9wz-cIMwGB4d00KXOTYfBhkbIp6Ff45YVs,5880
+trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
 trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
 trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
-trilogy/core/functions.py,sha256=xISGHMiUR9hFyoZe2l3VLTMO6UEtyGHQcMuovzlVRqw,9492
+trilogy/core/functions.py,sha256=MSNe-OLAEvj72qn5mif3ce9ncRh3ZATpX3V300yAm98,10125
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=4iwlTfHzzmbDhm9vt7zgExMqY1xKyQL-fVG5Nrm-YTU,134722
-trilogy/core/optimization.py,sha256=iX8BCh2BIEPJ6cYlcn7piDKFfig47azphhyqkwkYJM4,4291
-trilogy/core/query_processor.py,sha256=PROehLEOMOuKy1EAUeIgdfGq1LrJsX0N7Z2qDED6S30,17580
+trilogy/core/models.py,sha256=eZKTOD4mXGCH78YE0xNlHex15t1Ur4d6CGAZ1QNcBCQ,136597
+trilogy/core/optimization.py,sha256=A8S9C9H5RcQcFSQLYtEEBnm-r1CW_e9GEWlLK7q3MqA,4930
+trilogy/core/query_processor.py,sha256=BrP4x96Nbzfq6MfxYzxnOL6zYCLIoHLEii65MBT-Daw,17731
 trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
 trilogy/core/optimizations/inline_datasource.py,sha256=KiwZ4fnRnSOVK8zzdwMAkafszo3fNn5LY8xgb0X1CbI,3194
-trilogy/core/optimizations/predicate_pushdown.py,sha256=mduJn4zzKmLVNUja60ZlY6LlNp389OwjEbq2Rj8K6co,5389
+trilogy/core/optimizations/predicate_pushdown.py,sha256=HWHHG0hejBoIv985hVlsh3INPTrx8EGwCRWeeiPEHsc,7374
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=hmbck3a23N61HfMwC2z3lxA6Yms9TP7Lm3Aw7GiDcMA,23765
+trilogy/core/processing/concept_strategies_v3.py,sha256=Ssekd9sA7W-JgAif-Bp7ID1pZujaKJzlcP_KuoH2h4I,23938
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=hgfPfz5FlghN8edIVGTQ21eDUof5EMvnl3Vu64hjfqY,13289
+trilogy/core/processing/utility.py,sha256=L92nMZQqVSb_aOtYGi2rkvDD10e8WWg5-oQ7ApJBlI4,12374
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
-trilogy/core/processing/node_generators/basic_node.py,sha256=uD6noJOgrtmhjb-1aa-vD8sQ97s9Tya6wbso7V_AdyE,2819
+trilogy/core/processing/node_generators/basic_node.py,sha256=4242PNGTCm2tklqMIkqVu5Iv4m_IeTnOYXxDveuCDZM,2856
 trilogy/core/processing/node_generators/common.py,sha256=liZDth7mvhkF_sUFXK7JitJsiaKD132w3ySLbF7l-nE,8956
 trilogy/core/processing/node_generators/filter_node.py,sha256=5B7UCK84A9lGgJ7EjCiC7YGWWM1xPyQGHkBwZgYypFM,4585
 trilogy/core/processing/node_generators/group_node.py,sha256=xWI1xNIXEOj6jlRGD9hcv2_vVNvY6lpzJl6pQ8HuFBE,2988
 trilogy/core/processing/node_generators/group_to_node.py,sha256=BzPdYwzoo8gRMH7BDffTTXq4z-mjfCEzvfB5I-P0_nw,2941
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=YivTVrTEUJuQbQAHFVB2lUD4mJYQ-kTVVpwY4oT9A-Y,14664
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=JDQEIs43BJCgz0yofztcjd7J20Ybk4W40IJG_6blQJI,12168
 trilogy/core/processing/node_generators/rowset_node.py,sha256=2ROd2c1_o4h5sbnDdaiL1o9UiiwyYw-B6OKyfHYNe0A,6188
 trilogy/core/processing/node_generators/select_node.py,sha256=XGcz74XsWv5POWhV2gGC_jbi7T9g7sY-VtP3sjlppFc,19057
 trilogy/core/processing/node_generators/unnest_node.py,sha256=6CH66eGwpadNX7TzUhWZ8aqIisOtQeHINbLV6X3QBUk,1779
@@ -49,14 +49,14 @@ trilogy/core/processing/nodes/select_node_v2.py,sha256=COu-WPuyabGAc3HTkJB-_7eNz
 trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=EZCP3xU8-B4xpU1pFFew4CGtkNWignbyPll6OTENFLg,29832
+trilogy/dialect/base.py,sha256=bjoDwVAARpCVLjJvnfNpr1JtvoamkAAYZxw-G5QRV_A,27845
 trilogy/dialect/bigquery.py,sha256=BAN2o0SOuLvzLYLV7Sc11woapS5L7Cc36aMBeaphW5k,2905
 trilogy/dialect/common.py,sha256=5jdOHWIj3Xv8F8y5mnyWHLjxD_we2fncM-ZnNetJP7U,2781
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
 trilogy/dialect/duckdb.py,sha256=JQPnR5F39iDQXfTccKhbq7xWYGzYc0dPpn5vo0Qd_Vk,3076
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
 trilogy/dialect/postgres.py,sha256=r47xbCA7nfEYENofiVfLZ-SnReNfDmUmW4OSHVkkP4E,3206
-trilogy/dialect/presto.py,sha256=ouSVNbs7e3eEc20emLfxKyRjdINSibPGUFL4X9OwboA,3163
+trilogy/dialect/presto.py,sha256=evb7Tq77l4iBrZnBeSPfEdmdObgrca-p5H72CIrEips,3221
 trilogy/dialect/snowflake.py,sha256=N3HknYgN-fjD7BLX1Ucj-ss_ku2Ox8DgLsF3BIHutHo,2941
 trilogy/dialect/sql_server.py,sha256=HX68vNTrcDaTnOxe6Zbx_PBgrO42e2VuThxO6CYQ2cY,3026
 trilogy/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -65,18 +65,18 @@ trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2
 trilogy/hooks/query_debugger.py,sha256=NDChfkPmmW-KINa4TaQmDe_adGiwsKFdGLDSYpbodeU,4282
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=8MDRPccWymtVkHVQMzESjL5tOH79flWU5jo6Ys-C5UQ,5963
+trilogy/parsing/common.py,sha256=UZdG51pHNMGokvHjUxv7Zt5Xrh2snxfZuVOZVyg9yZ8,6091
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=rjTDs8AhJ6NjPCugfZCTdu7d_bUOSC9M8MAl9KmURNw,57763
+trilogy/parsing/parse_engine.py,sha256=-dXf-W6cOj-l2hDvgb7TWJKLZZEHAj0kwaw-DT0YKNU,58336
 trilogy/parsing/render.py,sha256=Gy_6wVYPwYLf35Iota08sbqveuWILtUhI8MYStcvtJM,12174
-trilogy/parsing/trilogy.lark,sha256=Zn5UHqiNoVKa76r1af9KND0TNdsLIeHpyG3hJjUOOTQ,11000
+trilogy/parsing/trilogy.lark,sha256=DiJe_XQZ9lXVUW5AF9eZl531lPAw7mm9MMeHCGkw7kc,11148
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.2.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.2.dist-info/METADATA,sha256=rjwj8b_CWxQc82n5HJfPTBYlhtVKNUwIhAkWLusfA7g,7876
-pytrilogy-0.0.2.2.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-pytrilogy-0.0.2.2.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.2.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.2.dist-info/RECORD,,
+pytrilogy-0.0.2.3.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.3.dist-info/METADATA,sha256=9Q2murSlwgsOWkr6A3FkEWWtlPKoI-VTmkONLbcnNPU,7906
+pytrilogy-0.0.2.3.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+pytrilogy-0.0.2.3.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.3.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.3.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.2"
+__version__ = "0.0.2.3"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/enums.py

@@ -119,6 +119,7 @@ class FunctionType(Enum):
 
     # COMPLEX
     INDEX_ACCESS = "index_access"
+    MAP_ACCESS = "map_access"
     ATTR_ACCESS = "attr_access"
 
     # TEXT AND MAYBE MORE

trilogy/core/env_processor.py

@@ -17,6 +17,11 @@ def add_concept(concept: Concept, g: ReferenceGraph):
     for _, pseudonym in concept.pseudonyms.items():
         pseudonym = pseudonym.with_default_grain()
         pseudonym_node = concept_to_node(pseudonym)
+        if (pseudonym_node, node_name) in g.edges and (
+            node_name,
+            pseudonym_node,
+        ) in g.edges:
+            continue
         if pseudonym_node.split("@")[0] == node_name.split("@")[0]:
             continue
         g.add_edge(pseudonym_node, node_name, pseudonym=True)

trilogy/core/functions.py

@@ -202,15 +202,21 @@ def Split(args: list[Concept]) -> Function:
     )
 
 
+def get_index_output_type(
+    arg: Concept,
+) -> DataType | StructType | MapType | ListType | NumericType:
+    if isinstance(arg.datatype, ListType):
+        return arg.datatype.value_data_type
+    elif isinstance(arg.datatype, MapType):
+        return arg.datatype.value_data_type
+    return arg.datatype
+
+
 def IndexAccess(args: list[Concept]):
     return Function(
         operator=FunctionType.INDEX_ACCESS,
         arguments=args,
-        output_datatype=(
-            args[0].datatype.value_data_type
-            if isinstance(args[0].datatype, ListType)
-            else args[0].datatype
-        ),
+        output_datatype=get_index_output_type(args[0]),
         output_purpose=Purpose.PROPERTY,
         valid_inputs=[
             {
@@ -226,6 +232,25 @@ def IndexAccess(args: list[Concept]):
     )
 
 
+def MapAccess(args: list[Concept]):
+    return Function(
+        operator=FunctionType.MAP_ACCESS,
+        arguments=args,
+        output_datatype=get_index_output_type(args[0]),
+        output_purpose=Purpose.PROPERTY,
+        valid_inputs=[
+            {
+                DataType.MAP,
+            },
+            {
+                DataType.INTEGER,
+                DataType.STRING,
+            },
+        ],
+        arg_count=2,
+    )
+
+
 def AttrAccess(args: list[Concept]):
     return Function(
         operator=FunctionType.ATTR_ACCESS,

trilogy/core/models.py

@@ -67,7 +67,7 @@ from trilogy.core.enums import (
 )
 from trilogy.core.exceptions import UndefinedConceptException, InvalidSyntaxException
 from trilogy.utility import unique
-from collections import UserList
+from collections import UserList, UserDict
 from functools import cached_property
 from abc import ABC
 
@@ -267,7 +267,7 @@ class ListType(BaseModel):
 
 class MapType(BaseModel):
     key_type: DataType
-    content_type: ALL_TYPES
+    value_type: ALL_TYPES
 
     @property
     def data_type(self):
@@ -277,6 +277,22 @@ class MapType(BaseModel):
     def value(self):
         return self.data_type.value
 
+    @property
+    def value_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
+        if isinstance(self.value_type, Concept):
+            return self.value_type.datatype
+        return self.value_type
+
+    @property
+    def key_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
+        if isinstance(self.key_type, Concept):
+            return self.key_type.datatype
+        return self.key_type
+
 
 class StructType(BaseModel):
     fields: List[ALL_TYPES]
@@ -314,6 +330,34 @@ class ListWrapper(Generic[VT], UserList):
         return cls(v, type=arg_to_datatype(v[0]))
 
 
+class MapWrapper(Generic[KT, VT], UserDict):
+    """Used to distinguish parsed map objects from other dicts"""
+
+    def __init__(self, *args, key_type: DataType, value_type: DataType, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.key_type = key_type
+        self.value_type = value_type
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: Callable[[Any], core_schema.CoreSchema]
+    ) -> core_schema.CoreSchema:
+        args = get_args(source_type)
+        if args:
+            schema = handler(Dict[args])  # type: ignore
+        else:
+            schema = handler(Dict)
+        return core_schema.no_info_after_validator_function(cls.validate, schema)
+
+    @classmethod
+    def validate(cls, v):
+        return cls(
+            v,
+            key_type=arg_to_datatype(list(v.keys()).pop()),
+            value_type=arg_to_datatype(list(v.values()).pop()),
+        )
+
+
 class Metadata(BaseModel):
     """Metadata container object.
     TODO: support arbitrary tags"""
@@ -949,8 +993,8 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
     output_purpose: Purpose
     valid_inputs: Optional[
         Union[
-            Set[DataType | ListType | StructType | NumericType],
-            List[Set[DataType | ListType | StructType] | NumericType],
+            Set[DataType | ListType | StructType | MapType | NumericType],
+            List[Set[DataType | ListType | StructType | MapType | NumericType]],
         ]
     ] = None
     arguments: Sequence[
@@ -961,17 +1005,17 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
             int,
             float,
             str,
+            MapWrapper[Any, Any],
             DataType,
             ListType,
+            MapType,
             NumericType,
             DatePart,
             "Parenthetical",
             CaseWhen,
             "CaseElse",
             list,
-            ListWrapper[int],
-            ListWrapper[str],
-            ListWrapper[float],
+            ListWrapper[Any],
         ]
     ]
 
@@ -2342,7 +2386,6 @@ class CTE(BaseModel):
     hidden_concepts: List[Concept] = Field(default_factory=list)
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
-    requires_nesting: bool = True
     base_name_override: Optional[str] = None
     base_alias_override: Optional[str] = None
 
@@ -3141,6 +3184,7 @@ class Environment(BaseModel):
                 v.pseudonyms[source.address] = source
             if v.address == source.address:
                 replacements[k] = target
+                v.pseudonyms[target.address] = target
         self.concepts.update(replacements)
 
         for k, ds in self.datasources.items():
@@ -4115,6 +4159,15 @@ def list_to_wrapper(args):
     return ListWrapper(args, type=types[0])
 
 
+def dict_to_map_wrapper(arg):
+    key_types = [arg_to_datatype(arg) for arg in arg.keys()]
+
+    value_types = [arg_to_datatype(arg) for arg in arg.values()]
+    assert len(set(key_types)) == 1
+    assert len(set(key_types)) == 1
+    return MapWrapper(arg, key_type=key_types[0], value_type=value_types[0])
+
+
 def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | NumericType:
     if isinstance(arg, Function):
         return arg.output_datatype
@@ -4143,5 +4196,7 @@ def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | Numeric
     elif isinstance(arg, list):
         wrapper = list_to_wrapper(arg)
         return ListType(type=wrapper.type)
+    elif isinstance(arg, MapWrapper):
+        return MapType(key_type=arg.key_type, value_type=arg.value_type)
     else:
         raise ValueError(f"Cannot parse arg datatype for arg of raw type {type(arg)}")

trilogy/core/optimization.py

@@ -1,10 +1,10 @@
 from trilogy.core.models import (
     CTE,
     SelectStatement,
-    PersistStatement,
     MultiSelectStatement,
+    Conditional,
 )
-from trilogy.core.enums import PurposeLineage
+from trilogy.core.enums import PurposeLineage, BooleanOperator
 from trilogy.constants import logger, CONFIG
 from trilogy.core.optimizations import (
     OptimizationRule,
@@ -42,34 +42,45 @@ def gen_inverse_map(input: list[CTE]) -> dict[str, list[CTE]]:
     return inverse_map
 
 
-def is_direct_return_eligible(
-    cte: CTE, select: SelectStatement | PersistStatement | MultiSelectStatement
-) -> bool:
-    if isinstance(select, (PersistStatement, MultiSelectStatement)):
-        return False
+def is_direct_return_eligible(cte: CTE) -> CTE | None:
+    # if isinstance(select, (PersistStatement, MultiSelectStatement)):
+    #     return False
+    if len(cte.parent_ctes) != 1:
+        return None
+    direct_parent = cte.parent_ctes[0]
+
+    output_addresses = set([x.address for x in cte.output_columns])
+    parent_output_addresses = set([x.address for x in direct_parent.output_columns])
+    if not output_addresses.issubset(parent_output_addresses):
+        return None
+    if not direct_parent.grain == cte.grain:
+        return None
     derived_concepts = [
         c
         for c in cte.source.output_concepts + cte.source.hidden_concepts
         if c not in cte.source.input_concepts
     ]
-    eligible = True
     conditions = (
-        set(x.address for x in select.where_clause.concept_arguments)
-        if select.where_clause
+        set(x.address for x in direct_parent.condition.concept_arguments)
+        if direct_parent.condition
         else set()
     )
     for x in derived_concepts:
         if x.derivation == PurposeLineage.WINDOW:
-            return False
+            return None
         if x.derivation == PurposeLineage.UNNEST:
-            return False
+            return None
         if x.derivation == PurposeLineage.AGGREGATE:
             if x.address in conditions:
-                return False
+                return None
+    # handling top level nodes that require unpacking
+    for x in cte.output_columns:
+        if x.derivation == PurposeLineage.UNNEST:
+            return None
     logger.info(
-        f"[Optimization][EarlyReturn] Upleveling output select to final CTE with derived_concepts {[x.address for x in derived_concepts]}"
+        f"[Optimization][EarlyReturn] Removing redundant output CTE with derived_concepts {[x.address for x in derived_concepts]}"
     )
-    return eligible
+    return direct_parent
 
 
 def sort_select_output(cte: CTE, query: SelectStatement | MultiSelectStatement):
@@ -90,23 +101,27 @@ def optimize_ctes(
     input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
 ) -> list[CTE]:
 
-    if CONFIG.optimizations.direct_return and is_direct_return_eligible(
-        root_cte, select
+    direct_parent: CTE | None = root_cte
+    while CONFIG.optimizations.direct_return and (
+        direct_parent := is_direct_return_eligible(root_cte)
     ):
-        root_cte.order_by = select.order_by
-        root_cte.limit = select.limit
-        # if select.where_clause:
-
-        #     if root_cte.condition:
-        #         root_cte.condition = Conditional(
-        #             left=root_cte.condition,
-        #             operator=BooleanOperator.AND,
-        #             right=select.where_clause.conditional,
-        #         )
-        #     else:
-        #         root_cte.condition = select.where_clause.conditional
-        root_cte.requires_nesting = False
-        sort_select_output(root_cte, select)
+        direct_parent.order_by = root_cte.order_by
+        direct_parent.limit = root_cte.limit
+        direct_parent.hidden_concepts = (
+            root_cte.hidden_concepts + direct_parent.hidden_concepts
+        )
+        if root_cte.condition:
+            if direct_parent.condition:
+                direct_parent.condition = Conditional(
+                    left=direct_parent.condition,
+                    operator=BooleanOperator.AND,
+                    right=root_cte.condition,
+                )
+            else:
+                direct_parent.condition = root_cte.condition
+        root_cte = direct_parent
+
+    sort_select_output(root_cte, select)
 
     REGISTERED_RULES: list["OptimizationRule"] = []
     if CONFIG.optimizations.constant_inlining:

trilogy/core/optimizations/predicate_pushdown.py

@@ -3,18 +3,43 @@ from trilogy.core.models import (
     Conditional,
     BooleanOperator,
     Datasource,
+    SubselectComparison,
+    Comparison,
+    Parenthetical,
+    Function,
+    FilterItem,
+    MagicConstants,
+    Concept,
+    WindowItem,
+    AggregateWrapper,
+    DataType,
 )
 from trilogy.core.optimizations.base_optimization import OptimizationRule
+from trilogy.core.enums import FunctionClass
 
 
-def decompose_condition(conditional: Conditional):
-    chunks = []
+def decompose_condition(
+    conditional: Conditional,
+) -> list[SubselectComparison | Comparison | Conditional | Parenthetical]:
+    chunks: list[SubselectComparison | Comparison | Conditional | Parenthetical] = []
     if conditional.operator == BooleanOperator.AND:
-        for val in [conditional.left, conditional.right]:
-            if isinstance(val, Conditional):
-                chunks.extend(decompose_condition(val))
-            else:
-                chunks.append(val)
+        if not (
+            isinstance(
+                conditional.left,
+                (SubselectComparison, Comparison, Conditional, Parenthetical),
+            )
+            and isinstance(
+                conditional.right,
+                (SubselectComparison, Comparison, Conditional, Parenthetical),
+            )
+        ):
+            chunks.append(conditional)
+        else:
+            for val in [conditional.left, conditional.right]:
+                if isinstance(val, Conditional):
+                    chunks.extend(decompose_condition(val))
+                else:
+                    chunks.append(val)
     else:
         chunks.append(conditional)
     return chunks
@@ -31,6 +56,40 @@ def is_child_of(a, comparison):
     return base
 
 
+def is_basic(
+    element: (
+        int
+        | str
+        | float
+        | list
+        | WindowItem
+        | FilterItem
+        | Concept
+        | Comparison
+        | Conditional
+        | Parenthetical
+        | Function
+        | AggregateWrapper
+        | MagicConstants
+        | DataType
+    ),
+) -> bool:
+    if isinstance(element, Parenthetical):
+        return is_basic(element.content)
+    elif isinstance(element, SubselectComparison):
+        return True
+    elif isinstance(element, Comparison):
+        return is_basic(element.left) and is_basic(element.right)
+    elif isinstance(element, Function):
+        if element.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
+            return False
+    elif isinstance(element, AggregateWrapper):
+        return is_basic(element.function)
+    elif isinstance(element, Conditional):
+        return is_basic(element.left) and is_basic(element.right)
+    return True
+
+
 class PredicatePushdown(OptimizationRule):
 
     def __init__(self, *args, **kwargs) -> None:
@@ -128,7 +187,14 @@ class PredicatePushdown(OptimizationRule):
         )
         optimized = False
         for candidate in candidates:
-            self.debug(f"Checking candidate {candidate}")
+            if not is_basic(candidate):
+                self.debug(
+                    f"Skipping {candidate} as not a basic [no aggregate, etc] condition"
+                )
+                continue
+            self.log(
+                f"Checking candidate {candidate}, {type(candidate)}, {is_basic(candidate)}"
+            )
             for parent_cte in cte.parent_ctes:
                 local_pushdown = self._check_parent(
                     parent_cte=parent_cte, candidate=candidate, inverse_map=inverse_map

trilogy/core/processing/concept_strategies_v3.py

@@ -336,8 +336,11 @@ def validate_concept(
     virtual_addresses: set[str],
     found_map: dict[str, set[Concept]],
     accept_partial: bool,
+    seen: set[str],
 ):
+
     found_map[str(node)].add(concept)
+    seen.add(concept.address)
     if concept not in node.partial_concepts:
 
         found_addresses.add(concept.address)
@@ -357,6 +360,8 @@ def validate_concept(
     for _, v in concept.pseudonyms.items():
         if v.address == concept.address:
             return
+        if v.address in seen:
+            return
         validate_concept(
             v,
             node,
@@ -366,6 +371,7 @@ def validate_concept(
             virtual_addresses,
             found_map,
             accept_partial,
+            seen=seen,
         )
 
 
@@ -379,8 +385,10 @@ def validate_stack(
     non_partial_addresses: set[str] = set()
     partial_addresses: set[str] = set()
     virtual_addresses: set[str] = set()
+    seen: set[str] = set()
     for node in stack:
         resolved = node.resolve()
+
         for concept in resolved.output_concepts:
             validate_concept(
                 concept,
@@ -391,6 +399,7 @@ def validate_stack(
                 virtual_addresses,
                 found_map,
                 accept_partial,
+                seen,
             )
         for concept in node.virtual_output_concepts:
             if concept.address in non_partial_addresses:

trilogy/core/processing/node_generators/basic_node.py

@@ -44,7 +44,7 @@ def gen_basic_node(
                         list(optional_set) + [concept],
                     )
                 )
-
+    # check for the concept by itself
     for attempt, basic_output in reversed(attempts):
         partials = []
         attempt = unique(attempt, "address")

trilogy/core/processing/node_generators/node_merge_node.py

@@ -4,7 +4,6 @@ from trilogy.core.models import Concept, Environment, Conditional
 from trilogy.core.processing.nodes import MergeNode, History, StrategyNode
 import networkx as nx
 from trilogy.core.graph_models import concept_to_node
-from trilogy.core.processing.utility import PathInfo
 from trilogy.constants import logger
 from trilogy.utility import unique
 from trilogy.core.exceptions import AmbiguousRelationshipResolutionException
@@ -63,7 +62,9 @@ def extract_ds_components(g: nx.DiGraph, nodelist: list[str]) -> list[list[str]]
                     if not str(x).startswith("ds~")
                 ]
             )
-
+    # if we had no ego graphs, return all concepts
+    if not graphs:
+        return [[extract_address(node) for node in nodelist]]
     graphs = filter_unique_graphs(graphs)
     for node in nodelist:
         parsed = extract_address(node)
@@ -82,6 +83,7 @@ def determine_induced_minimal_nodes(
     H: nx.Graph = nx.to_undirected(G).copy()
     nodes_to_remove = []
     concepts = nx.get_node_attributes(G, "concept")
+
     for node in G.nodes:
         if concepts.get(node):
             lookup = concepts[node]
@@ -107,9 +109,11 @@ def determine_induced_minimal_nodes(
         paths = nx.multi_source_dijkstra_path(H, nodelist)
     except nx.exception.NodeNotFound:
         return None
+
     H.remove_nodes_from(list(x for x in H.nodes if x not in paths))
     sG: nx.Graph = ax.steinertree.steiner_tree(H, nodelist).copy()
     final: nx.DiGraph = nx.subgraph(G, sG.nodes).copy()
+
     for edge in G.edges:
         if edge[1] in final.nodes and edge[0].startswith("ds~"):
             ds_name = extract_address(edge[0])
@@ -125,6 +129,7 @@ def determine_induced_minimal_nodes(
         [final.in_degree(node) > 0 for node in final.nodes if node.startswith("c~")]
     ):
         return None
+
     if not all([node in final.nodes for node in nodelist]):
         return None
     return final
@@ -308,111 +313,44 @@ def gen_merge_node(
     history: History | None = None,
     conditions: Conditional | None = None,
 ) -> Optional[MergeNode]:
-    join_candidates: List[PathInfo] = []
-
-    # inject new concepts into search, and identify if two dses can reach there
-    if not join_candidates:
-        for filter_downstream in [True, False]:
-            weak_resolve = resolve_weak_components(
-                all_concepts,
-                environment,
-                g,
-                filter_downstream=filter_downstream,
-                accept_partial=accept_partial,
-            )
-            if weak_resolve:
-                log_graph = [[y.address for y in x] for x in weak_resolve]
-                logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} Was able to resolve graph through weak component resolution - final graph {log_graph}"
-                )
-                return subgraphs_to_merge_node(
-                    weak_resolve,
-                    depth=depth,
-                    all_concepts=all_concepts,
-                    environment=environment,
-                    g=g,
-                    source_concepts=source_concepts,
-                    history=history,
-                    conditions=conditions,
-                )
-    if not join_candidates:
-        return None
-    join_additions: list[set[str]] = []
-    for candidate in join_candidates:
-        join_additions.append(candidate.reduced_concepts)
-
-    common: set[str] = set()
-    final_candidates: list[set[str]] = []
-    # find all values that show up in every join_additions
-    for ja in join_additions:
-        if not common:
-            common = ja
-        else:
-            common = common.intersection(ja)
-        if all(ja.issubset(y) for y in join_additions):
-            final_candidates.append(ja)
 
-    if not final_candidates:
-        filtered_paths = [x.difference(common) for x in join_additions]
-        raise AmbiguousRelationshipResolutionException(
-            f"Ambiguous concept join resolution fetching {[x.address for x in all_concepts]} - unique values in possible paths = {filtered_paths}. Include an additional concept to disambiguate",
-            join_additions,
-        )
-    if not join_candidates:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} No additional join candidates could be found"
+    for filter_downstream in [True, False]:
+        weak_resolve = resolve_weak_components(
+            all_concepts,
+            environment,
+            g,
+            filter_downstream=filter_downstream,
+            accept_partial=accept_partial,
         )
-        return None
-    shortest: PathInfo = sorted(
-        [x for x in join_candidates if x.reduced_concepts in final_candidates],
-        key=lambda x: len(x.reduced_concepts),
-    )[0]
-    logger.info(f"{padding(depth)}{LOGGER_PREFIX} final path is {shortest.paths}")
-
-    return subgraphs_to_merge_node(
-        shortest.concept_subgraphs,
-        depth=depth,
-        all_concepts=all_concepts,
-        environment=environment,
-        g=g,
-        source_concepts=source_concepts,
-        history=history,
-        conditions=conditions,
-    )
-    # parents = []
-    # for graph in shortest.concept_subgraphs:
-    #     logger.info(
-    #         f"{padding(depth)}{LOGGER_PREFIX} fetching subgraph {[c.address for c in graph]}"
-    #     )
-    #     parent = source_concepts(
-    #         mandatory_list=graph,
-    #         environment=environment,
-    #         g=g,
-    #         depth=depth + 1,
-    #         history=history,
-    #     )
-    #     if not parent:
-    #         logger.info(
-    #             f"{padding(depth)}{LOGGER_PREFIX} Unable to instantiate target subgraph"
-    #         )
-    #         return None
-    #     logger.info(
-    #         f"{padding(depth)}{LOGGER_PREFIX} finished subgraph fetch for {[c.address for c in graph]}, have parent {type(parent)}"
-    #     )
-    #     parents.append(parent)
-
-    # return MergeNode(
-    #     input_concepts=[
-    #         environment.concepts[x]
-    #         for x in shortest.reduced_concepts
-    #         if environment.concepts[x].derivation != PurposeLineage.MERGE
-    #     ],
-    #     output_concepts=[
-    #         x for x in all_concepts if x.derivation != PurposeLineage.MERGE
-    #     ],
-    #     environment=environment,
-    #     g=g,
-    #     parents=parents,
-    #     depth=depth,
-    #     conditions=conditions,
-    # )
+        if weak_resolve:
+            log_graph = [[y.address for y in x] for x in weak_resolve]
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} Was able to resolve graph through weak component resolution - final graph {log_graph}"
+            )
+            return subgraphs_to_merge_node(
+                weak_resolve,
+                depth=depth,
+                all_concepts=all_concepts,
+                environment=environment,
+                g=g,
+                source_concepts=source_concepts,
+                history=history,
+                conditions=conditions,
+            )
+    # one concept handling may need to be kicked to alias
+    if len(all_concepts) == 1:
+        concept = all_concepts[0]
+        for k, v in concept.pseudonyms.items():
+            test = subgraphs_to_merge_node(
+                [[concept, v]],
+                g=g,
+                all_concepts=[concept],
+                environment=environment,
+                depth=depth,
+                source_concepts=source_concepts,
+                history=history,
+                conditions=conditions,
+            )
+            if test:
+                return test
+    return None

trilogy/core/processing/utility.py

@@ -124,15 +124,20 @@ def resolve_join_order(joins: List[BaseJoin]) -> List[BaseJoin]:
     return final_joins
 
 
-def add_node_join_concept(graph, concept, datasource, concepts):
-    # we don't need to join on a concept if all of the keys exist in the grain
-    # if concept.keys and all([x in grain for x in concept.keys]):
-    #     continue
+def add_node_join_concept(
+    graph: nx.DiGraph,
+    concept: Concept,
+    datasource: Datasource | QueryDatasource,
+    concepts: List[Concept],
+):
+
     concepts.append(concept)
 
     graph.add_node(concept.address, type=NodeType.CONCEPT)
     graph.add_edge(datasource.identifier, concept.address)
-    for k, v in concept.pseudonyms.items():
+    for _, v in concept.pseudonyms.items():
+        if v in concepts:
+            continue
         if v.address != concept.address:
             add_node_join_concept(graph, v, datasource, concepts)
 
@@ -149,13 +154,6 @@ def get_node_joins(
         graph.add_node(datasource.identifier, type=NodeType.NODE)
         for concept in datasource.output_concepts:
             add_node_join_concept(graph, concept, datasource, concepts)
-            # we don't need to join on a concept if all of the keys exist in the grain
-            # if concept.keys and all([x in grain for x in concept.keys]):
-            #     continue
-            # concepts.append(concept)
-
-            # graph.add_node(concept.address, type=NodeType.CONCEPT)
-            # graph.add_edge(datasource.identifier, concept.address)
 
     # add edges for every constant to every datasource
     for datasource in datasources:
@@ -195,26 +193,6 @@ def get_node_joins(
         ),
     )
 
-    node_map = {
-        x[0:20]: len(
-            [
-                partial
-                for partial in identifier_map[x].partial_concepts
-                if partial in grain
-            ]
-            + [
-                output
-                for output in identifier_map[x].output_concepts
-                if output.address in grain_pseudonyms
-            ]
-        )
-        for x in node_list
-    }
-    print("NODE MAP")
-    print(node_map)
-    print([x.address for x in grain])
-    print(grain_pseudonyms)
-
     for left in node_list:
         # the constant dataset is a special case
         # and can never be on the left of a join

trilogy/core/query_processor.py

@@ -467,9 +467,11 @@ def process_query(
     for cte in raw_ctes:
         cte.parent_ctes = [seen[x.name] for x in cte.parent_ctes]
     deduped_ctes: List[CTE] = list(seen.values())
+    root_cte.order_by = statement.order_by
+    root_cte.limit = statement.limit
+    root_cte.hidden_concepts = [x for x in statement.hidden_components]
 
     final_ctes = optimize_ctes(deduped_ctes, root_cte, statement)
-
     return ProcessedQuery(
         order_by=statement.order_by,
         grain=statement.grain,

trilogy/dialect/base.py

@@ -8,7 +8,6 @@ from trilogy.core.enums import (
     FunctionType,
     WindowType,
     DatePart,
-    PurposeLineage,
     ComparisonOperator,
 )
 from trilogy.core.models import (
@@ -36,6 +35,7 @@ from trilogy.core.models import (
     Environment,
     RawColumnExpr,
     ListWrapper,
+    MapWrapper,
     ShowStatement,
     RowsetItem,
     MultiSelectStatement,
@@ -45,6 +45,7 @@ from trilogy.core.models import (
     RawSQLStatement,
     ProcessedRawSQLStatement,
     NumericType,
+    MapType,
     MergeStatementV2,
 )
 from trilogy.core.query_processor import process_query, process_persist
@@ -97,6 +98,7 @@ DATATYPE_MAP = {
     DataType.FLOAT: "float",
     DataType.BOOL: "bool",
     DataType.NUMERIC: "numeric",
+    DataType.MAP: "map",
 }
 
 
@@ -116,6 +118,7 @@ FUNCTION_MAP = {
     FunctionType.IS_NULL: lambda x: f"isnull({x[0]})",
     # complex
     FunctionType.INDEX_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
+    FunctionType.MAP_ACCESS: lambda x: f"{x[0]}[{x[1]}][1]",
     FunctionType.UNNEST: lambda x: f"unnest({x[0]})",
     # math
     FunctionType.ADD: lambda x: f"{x[0]} + {x[1]}",
@@ -189,14 +192,13 @@ TOP {{ limit }}{% endif %}
 \t{{ select }}{% if not loop.last %},{% endif %}{% endfor %}
 {% if base %}FROM
 \t{{ base }}{% endif %}{% if joins %}{% for join in joins %}
-\t{{ join }}{% endfor %}{% endif %}
-{% if where %}WHERE
-\t{{ where }}
-{% endif %}{%- if group_by %}GROUP BY {% for group in group_by %}
-\t{{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
-{%- if order_by %}
-ORDER BY {% for order in order_by %}
-    {{ order }}{% if not loop.last %},{% endif %}{% endfor %}
+\t{{ join }}{% endfor %}{% endif %}{% if where %}
+WHERE
+\t{{ where }}{% endif %}{%- if group_by %}
+GROUP BY {% for group in group_by %}
+\t{{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{%- if order_by %}
+ORDER BY{% for order in order_by %}
+\t{{ order }}{% if not loop.last %},{% endif %}{% endfor %}
 {% endif %}{% endif %}
 """
 )
@@ -214,7 +216,13 @@ def safe_get_cte_value(coalesce, cte: CTE, c: Concept, quote_char: str):
     raw = cte.source_map.get(address, None)
 
     if not raw:
-        return INVALID_REFERENCE_STRING("Missing source reference")
+        for k, v in c.pseudonyms.items():
+            if cte.source_map.get(k):
+                c = v
+                raw = cte.source_map[k]
+                break
+        if not raw:
+            return INVALID_REFERENCE_STRING("Missing source reference")
     if isinstance(raw, str):
         rendered = cte.get_alias(c, raw)
         return f"{raw}.{safe_quote(rendered, quote_char)}"
@@ -291,6 +299,7 @@ class BaseDialect:
                     self.render_expr(v, cte)  # , alias=False)
                     for v in c.lineage.arguments
                 ]
+
                 if cte.group_to_grain:
                     rval = f"{self.FUNCTION_MAP[c.lineage.operator](args)}"
                 else:
@@ -335,11 +344,11 @@ class BaseDialect:
             Parenthetical,
             AggregateWrapper,
             MagicConstants,
+            MapWrapper[Any, Any],
+            MapType,
             NumericType,
             ListType,
-            ListWrapper[int],
-            ListWrapper[str],
-            ListWrapper[float],
+            ListWrapper[Any],
             DatePart,
             CaseWhen,
             CaseElse,
@@ -435,6 +444,8 @@ class BaseDialect:
             return str(e)
         elif isinstance(e, ListWrapper):
             return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}]"
+        elif isinstance(e, MapWrapper):
+            return f"MAP {{{','.join([f'{self.render_expr(k, cte=cte, cte_map=cte_map)}:{self.render_expr(v, cte=cte, cte_map=cte_map)}' for k, v in e.items()])}}}"
         elif isinstance(e, list):
             return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}]"
         elif isinstance(e, DataType):
@@ -643,82 +654,20 @@ class BaseDialect:
                 f" {selected}"
             )
 
-        # where assignment
-        output_where = False
-        if query.where_clause:
-            # found = False
-            filter = set(
-                [
-                    str(x.address)
-                    for x in query.where_clause.row_arguments
-                    if not x.derivation == PurposeLineage.CONSTANT
-                ]
-            )
-            query_output = set([str(z.address) for z in query.output_columns])
-            # if it wasn't an output
-            # we would have forced it up earlier and we don't need to render at this point
-            if filter.issubset(query_output):
-                output_where = True
-            for ex_set in query.where_clause.existence_arguments:
-                for c in ex_set:
-                    if c.address not in cte_output_map:
-                        cts = [
-                            ct
-                            for ct in query.ctes
-                            if ct.name in query.base.existence_source_map[c.address]
-                        ]
-                        if not cts:
-                            raise ValueError(query.base.existence_source_map[c.address])
-                        cte_output_map[c.address] = cts[0]
-
         compiled_ctes = self.generate_ctes(query)
 
         # restort selections by the order they were written in
         sorted_select: List[str] = []
         for output_c in output_addresses:
             sorted_select.append(select_columns[output_c])
-        if not query.base.requires_nesting:
-            final = self.SQL_TEMPLATE.render(
-                output=(
-                    query.output_to
-                    if isinstance(query, ProcessedQueryPersist)
-                    else None
-                ),
-                full_select=compiled_ctes[-1].statement,
-                ctes=compiled_ctes[:-1],
-            )
-        else:
-            final = self.SQL_TEMPLATE.render(
-                output=(
-                    query.output_to
-                    if isinstance(query, ProcessedQueryPersist)
-                    else None
-                ),
-                select_columns=sorted_select,
-                base=query.base.name,
-                joins=[
-                    render_join(join, self.QUOTE_CHARACTER, None)
-                    for join in query.joins
-                ],
-                ctes=compiled_ctes,
-                limit=query.limit,
-                # move up to CTEs
-                where=(
-                    self.render_expr(
-                        query.where_clause.conditional, cte_map=cte_output_map
-                    )
-                    if query.where_clause and output_where
-                    else None
-                ),
-                order_by=(
-                    [
-                        self.render_order_item(i, query.base, final=True)
-                        for i in query.order_by.items
-                    ]
-                    if query.order_by
-                    else None
-                ),
-            )
+
+        final = self.SQL_TEMPLATE.render(
+            output=(
+                query.output_to if isinstance(query, ProcessedQueryPersist) else None
+            ),
+            full_select=compiled_ctes[-1].statement,
+            ctes=compiled_ctes[:-1],
+        )
 
         if CONFIG.strict_mode and INVALID_REFERENCE_STRING(1) in final:
             raise ValueError(

trilogy/dialect/presto.py

@@ -15,6 +15,7 @@ FUNCTION_MAP = {
     FunctionType.LENGTH: lambda x: f"length({x[0]})",
     FunctionType.AVG: lambda x: f"avg({x[0]})",
     FunctionType.INDEX_ACCESS: lambda x: f"element_at({x[0]},{x[1]})",
+    FunctionType.MAP_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
     FunctionType.LIKE: lambda x: (
         f" CASE WHEN {x[0]} like {x[1]} THEN True ELSE False END"
     ),

trilogy/parsing/common.py

@@ -7,6 +7,7 @@ from trilogy.core.models import (
     Metadata,
     FilterItem,
     ListWrapper,
+    MapWrapper,
     WindowItem,
 )
 from typing import List, Tuple
@@ -41,7 +42,7 @@ def concept_list_to_keys(concepts: Tuple[Concept, ...]) -> Tuple[Concept, ...]:
 
 
 def constant_to_concept(
-    parent: ListWrapper | list | int | float | str,
+    parent: ListWrapper | MapWrapper | list | int | float | str,
     name: str,
     namespace: str,
     purpose: Purpose | None = None,
@@ -53,6 +54,7 @@ def constant_to_concept(
         output_purpose=Purpose.CONSTANT,
         arguments=[parent],
     )
+    assert const_function.arguments[0] == parent, const_function.arguments[0]
     fmetadata = metadata or Metadata()
     return Concept(
         name=name,
@@ -186,6 +188,7 @@ def arbitrary_to_concept(
         | FilterItem
         | Function
         | ListWrapper
+        | MapWrapper
         | int
         | float
         | str

trilogy/parsing/parse_engine.py

@@ -43,6 +43,7 @@ from trilogy.core.functions import (
     Min,
     Split,
     IndexAccess,
+    MapAccess,
     AttrAccess,
     Abs,
     Unnest,
@@ -94,6 +95,7 @@ from trilogy.core.models import (
     RawColumnExpr,
     arg_to_datatype,
     ListWrapper,
+    MapWrapper,
     MapType,
     ShowStatement,
     DataType,
@@ -104,6 +106,7 @@ from trilogy.core.models import (
     RowsetDerivationStatement,
     LooseConceptList,
     list_to_wrapper,
+    dict_to_map_wrapper,
     NumericType,
 )
 from trilogy.parsing.exceptions import ParseError
@@ -117,7 +120,7 @@ from trilogy.parsing.common import (
     arbitrary_to_concept,
 )
 
-CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper)
+CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper, MapWrapper)
 
 with open(join(dirname(__file__), "trilogy.lark"), "r") as f:
     PARSER = Lark(
@@ -253,7 +256,7 @@ class ParseToObjects(Transformer):
                 self.environment.add_concept(concept, meta=meta)
                 final.append(concept)
             elif isinstance(
-                arg, (FilterItem, WindowItem, AggregateWrapper, ListWrapper)
+                arg, (FilterItem, WindowItem, AggregateWrapper, ListWrapper, MapWrapper)
             ):
                 id_hash = string_to_hash(str(arg))
                 concept = arbitrary_to_concept(
@@ -330,7 +333,12 @@ class ParseToObjects(Transformer):
     def numeric_type(self, args) -> NumericType:
         return NumericType(precision=args[0], scale=args[1])
 
-    def data_type(self, args) -> DataType | ListType | StructType | NumericType:
+    def map_type(self, args) -> MapType:
+        return MapType(key_type=args[0], value_type=args[1])
+
+    def data_type(
+        self, args
+    ) -> DataType | ListType | StructType | MapType | NumericType:
         resolved = args[0]
         if isinstance(resolved, StructType):
             return resolved
@@ -338,6 +346,8 @@ class ParseToObjects(Transformer):
             return resolved
         elif isinstance(resolved, NumericType):
             return resolved
+        elif isinstance(resolved, MapType):
+            return resolved
         return DataType(args[0].lower())
 
     def array_comparison(self, args) -> ComparisonOperator:
@@ -490,7 +500,6 @@ class ParseToObjects(Transformer):
         # we need to strip off every parenthetical to see what is being assigned.
         while isinstance(source_value, Parenthetical):
             source_value = source_value.content
-
         if isinstance(
             source_value, (FilterItem, WindowItem, AggregateWrapper, Function)
         ):
@@ -549,7 +558,7 @@ class ParseToObjects(Transformer):
         else:
             metadata = None
         name = args[1]
-        constant: Union[str, float, int, bool] = args[2]
+        constant: Union[str, float, int, bool, MapWrapper, ListWrapper] = args[2]
         lookup, namespace, name, parent = parse_concept_reference(
             name, self.environment
         )
@@ -1007,6 +1016,11 @@ class ParseToObjects(Transformer):
     def array_lit(self, args):
         return list_to_wrapper(args)
 
+    def map_lit(self, args):
+        parsed = dict(zip(args[::2], args[1::2]))
+        wrapped = dict_to_map_wrapper(parsed)
+        return wrapped
+
     def literal(self, args):
         return args[0]
 
@@ -1144,6 +1158,8 @@ class ParseToObjects(Transformer):
     @v_args(meta=True)
     def index_access(self, meta, args):
         args = self.process_function_args(args, meta=meta)
+        if args[0].datatype == DataType.MAP or isinstance(args[0].datatype, MapType):
+            return MapAccess(args)
         return IndexAccess(args)
 
     @v_args(meta=True)

trilogy/parsing/trilogy.lark

@@ -274,12 +274,14 @@
     float_lit: /[0-9]*\.[0-9]+/
 
     array_lit: "[" (literal ",")* literal ","? "]"()
+
+    map_lit: "{" (literal ":" literal ",")* literal ":" literal ","? "}"
     
     !bool_lit: "True"i | "False"i
 
     !null_lit.1: "null"i
     
-    literal: null_lit | _string_lit | int_lit | float_lit | bool_lit | array_lit
+    literal: null_lit | _string_lit | int_lit | float_lit | bool_lit | array_lit | map_lit
 
     MODIFIER: "Optional"i | "Partial"i | "Nullable"i
 
@@ -291,8 +293,9 @@
 
     numeric_type: "numeric"i "(" int_lit "," int_lit ")"
 
+    map_type: "map"i "<" data_type "," data_type ">"
 
-    !data_type: "string"i | "number"i | "numeric"i | "map"i | "list"i | "array"i | "any"i | "int"i | "bigint"i | "date"i | "datetime"i | "timestamp"i | "float"i | "bool"i | numeric_type | struct_type | list_type
+    !data_type: "string"i | "number"i | "numeric"i | "map"i | "list"i | "array"i | "any"i | "int"i | "bigint"i | "date"i | "datetime"i | "timestamp"i | "float"i | "bool"i | numeric_type | map_type | struct_type | list_type
     
     PURPOSE:  "key"i | "metric"i  | CONST
     PROPERTY: "property"i