pytrilogy 0.0.2.2__py3-none-any.whl → 0.0.2.4__py3-none-any.whl

{pytrilogy-0.0.2.2.dist-info → pytrilogy-0.0.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.2
+Version: 0.0.2.4
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -247,7 +247,7 @@ N/A, only supports default auth. In python you can pass in a custom client.
 
 
 > [!TIP]
-> The CLI can also be used for formatting. PreQL has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
+> The CLI can also be used for formatting. Trilogy has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
 
 
 ## More Examples
@@ -284,7 +284,7 @@ but all are worth checking out. Please open PRs/comment for anything missed!
 
 #### CONCEPT
 
-Types: `string | int | float | bool | date | datetime | time | timestamp | interval`;
+Types: `string | int | float | bool | date | datetime | time | numeric(scale, precision) | timestamp | interval`;
 
 Key:
 `key <name> <type>;`

{pytrilogy-0.0.2.2.dist-info → pytrilogy-0.0.2.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-trilogy/__init__.py,sha256=mSeU_j02wb5aKd5vDcHQJdZaoG1AB96bCjUn2tIVpwk,290
+trilogy/__init__.py,sha256=sspl5Npt0npzVRC6wCwiYXa0nK0E7a_biwiL2T4kbjM,290
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=KIvi-cgU4R9urNgDdGiCsRkCrzjAfM4xGHhZb1SVy2w,881
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
@@ -8,34 +8,34 @@ trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=DWKPuShM_DbyAenjNFOAxcIH2l0QC9WgIwqZKPQqk_o,5850
-trilogy/core/env_processor.py,sha256=Wpy-iiduBwHntTsQTYWBO1O0i3Ij9_VoL3d4IWDEoj4,2126
+trilogy/core/enums.py,sha256=ozgw8G0s4CdbL6IUsX5nNT_ZGd4mMkoo8_EGLZXeQYY,5902
+trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
 trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
 trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
-trilogy/core/functions.py,sha256=xISGHMiUR9hFyoZe2l3VLTMO6UEtyGHQcMuovzlVRqw,9492
+trilogy/core/functions.py,sha256=ARJAyBjeS415-54k3G_bx807rkPZonEulMaLRxSP7vU,10371
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=4iwlTfHzzmbDhm9vt7zgExMqY1xKyQL-fVG5Nrm-YTU,134722
-trilogy/core/optimization.py,sha256=iX8BCh2BIEPJ6cYlcn7piDKFfig47azphhyqkwkYJM4,4291
-trilogy/core/query_processor.py,sha256=PROehLEOMOuKy1EAUeIgdfGq1LrJsX0N7Z2qDED6S30,17580
+trilogy/core/models.py,sha256=KVPUfupZVhQxJZCViMP7PELzvNR7WOwkjmqqgEM4oeM,137120
+trilogy/core/optimization.py,sha256=A8S9C9H5RcQcFSQLYtEEBnm-r1CW_e9GEWlLK7q3MqA,4930
+trilogy/core/query_processor.py,sha256=6JiX6YqsTgnsEkIxcmWFxJuM6kRJbXbPGRCQgi8g-B8,17941
 trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
 trilogy/core/optimizations/inline_datasource.py,sha256=KiwZ4fnRnSOVK8zzdwMAkafszo3fNn5LY8xgb0X1CbI,3194
-trilogy/core/optimizations/predicate_pushdown.py,sha256=mduJn4zzKmLVNUja60ZlY6LlNp389OwjEbq2Rj8K6co,5389
+trilogy/core/optimizations/predicate_pushdown.py,sha256=iVZV_BVaL3I6wlELPOuJAiuF0heOM7bCUnNqyCxxRVw,6370
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=hmbck3a23N61HfMwC2z3lxA6Yms9TP7Lm3Aw7GiDcMA,23765
+trilogy/core/processing/concept_strategies_v3.py,sha256=xCPfg1ySHP6C-u777kqnV8eqIK1X68hqwvRcw61q5X0,23968
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=hgfPfz5FlghN8edIVGTQ21eDUof5EMvnl3Vu64hjfqY,13289
+trilogy/core/processing/utility.py,sha256=rhnkn3KgB1MW17mZYSEZ8XFEbC__B7PPmZgpDql4Myg,13614
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
-trilogy/core/processing/node_generators/basic_node.py,sha256=uD6noJOgrtmhjb-1aa-vD8sQ97s9Tya6wbso7V_AdyE,2819
-trilogy/core/processing/node_generators/common.py,sha256=liZDth7mvhkF_sUFXK7JitJsiaKD132w3ySLbF7l-nE,8956
-trilogy/core/processing/node_generators/filter_node.py,sha256=5B7UCK84A9lGgJ7EjCiC7YGWWM1xPyQGHkBwZgYypFM,4585
+trilogy/core/processing/node_generators/basic_node.py,sha256=4242PNGTCm2tklqMIkqVu5Iv4m_IeTnOYXxDveuCDZM,2856
+trilogy/core/processing/node_generators/common.py,sha256=uLrqbEsuYREswiTfuTQuxldO9SsS6ylZEQT_q9iB2uY,8948
+trilogy/core/processing/node_generators/filter_node.py,sha256=kVsK9O9QWUgknui3ESHJJCnFY9dVMCHlGpHXn1QtXkU,4824
 trilogy/core/processing/node_generators/group_node.py,sha256=xWI1xNIXEOj6jlRGD9hcv2_vVNvY6lpzJl6pQ8HuFBE,2988
 trilogy/core/processing/node_generators/group_to_node.py,sha256=BzPdYwzoo8gRMH7BDffTTXq4z-mjfCEzvfB5I-P0_nw,2941
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=YivTVrTEUJuQbQAHFVB2lUD4mJYQ-kTVVpwY4oT9A-Y,14664
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=JDQEIs43BJCgz0yofztcjd7J20Ybk4W40IJG_6blQJI,12168
 trilogy/core/processing/node_generators/rowset_node.py,sha256=2ROd2c1_o4h5sbnDdaiL1o9UiiwyYw-B6OKyfHYNe0A,6188
 trilogy/core/processing/node_generators/select_node.py,sha256=XGcz74XsWv5POWhV2gGC_jbi7T9g7sY-VtP3sjlppFc,19057
 trilogy/core/processing/node_generators/unnest_node.py,sha256=6CH66eGwpadNX7TzUhWZ8aqIisOtQeHINbLV6X3QBUk,1779
@@ -43,40 +43,40 @@ trilogy/core/processing/node_generators/window_node.py,sha256=9nXUXUgQrNczU1gaOq
 trilogy/core/processing/nodes/__init__.py,sha256=-8ONG7Nqa6sNLpst5wNNyMigyaHRfUKGdE3xce8E3mM,3998
 trilogy/core/processing/nodes/base_node.py,sha256=q1G4FO3Q4OuboWL8EaJgJ4jcnnqM9QsothMIcF2ot1g,11161
 trilogy/core/processing/nodes/filter_node.py,sha256=ImUHYj5G5eHbOtki-ZMR31Op38wKOjR1xu_ETiE_D6I,1932
-trilogy/core/processing/nodes/group_node.py,sha256=5dqGlonXG5Ou7HRSsTStbM1KyxeDSZROz9qKTjOUMEc,5166
+trilogy/core/processing/nodes/group_node.py,sha256=j3vtFQIn_t4EfCke8dGULlwVkG2KKPYfgGwo2rvgcww,6322
 trilogy/core/processing/nodes/merge_node.py,sha256=W7ryL9rceylCQGNgcuGY-dSiNoIz8qrbpJXKzg_Niwo,14056
 trilogy/core/processing/nodes/select_node_v2.py,sha256=COu-WPuyabGAc3HTkJB-_7eNzR4OgxIL4YDuZIRrST0,7179
 trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=EZCP3xU8-B4xpU1pFFew4CGtkNWignbyPll6OTENFLg,29832
-trilogy/dialect/bigquery.py,sha256=BAN2o0SOuLvzLYLV7Sc11woapS5L7Cc36aMBeaphW5k,2905
+trilogy/dialect/base.py,sha256=aE8iJJhkGtZE04mYm_wQx3ebQKYbcRsKISUd9mbrZ2A,28782
+trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
 trilogy/dialect/common.py,sha256=5jdOHWIj3Xv8F8y5mnyWHLjxD_we2fncM-ZnNetJP7U,2781
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
-trilogy/dialect/duckdb.py,sha256=JQPnR5F39iDQXfTccKhbq7xWYGzYc0dPpn5vo0Qd_Vk,3076
+trilogy/dialect/duckdb.py,sha256=JEdhctDxZr-FEodot5V4De6JnrQDJT02zbjX48j6ay8,3125
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
-trilogy/dialect/postgres.py,sha256=r47xbCA7nfEYENofiVfLZ-SnReNfDmUmW4OSHVkkP4E,3206
-trilogy/dialect/presto.py,sha256=ouSVNbs7e3eEc20emLfxKyRjdINSibPGUFL4X9OwboA,3163
-trilogy/dialect/snowflake.py,sha256=N3HknYgN-fjD7BLX1Ucj-ss_ku2Ox8DgLsF3BIHutHo,2941
-trilogy/dialect/sql_server.py,sha256=HX68vNTrcDaTnOxe6Zbx_PBgrO42e2VuThxO6CYQ2cY,3026
+trilogy/dialect/postgres.py,sha256=ev1RJZsC8BB3vJSxJ4q-TTYqZ4Hk1NXUtuRkLrQEBX0,3254
+trilogy/dialect/presto.py,sha256=InvHDXsGLDG3E3ffl7hGZ-9y3vH8RO6UhNE0OSra01g,3269
+trilogy/dialect/snowflake.py,sha256=_Bf4XO7-nImMv9XCSsTfVM3g2f_KHdO17VTa9J-HgSM,2989
+trilogy/dialect/sql_server.py,sha256=OtXbm1v6NIGyXeC5i18ojUvLeqescc_Pbv3EASUBB94,3074
 trilogy/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/hooks/base_hook.py,sha256=Xkb-A2qCHozYjum0A36zOy5PwTVwrP3NLDF0U2GpgHo,1100
 trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2467
 trilogy/hooks/query_debugger.py,sha256=NDChfkPmmW-KINa4TaQmDe_adGiwsKFdGLDSYpbodeU,4282
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=8MDRPccWymtVkHVQMzESjL5tOH79flWU5jo6Ys-C5UQ,5963
+trilogy/parsing/common.py,sha256=aYmLtsEk8VJj5yYASzUcTf0i0lXVsZBbZJwphwokxK4,6092
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=rjTDs8AhJ6NjPCugfZCTdu7d_bUOSC9M8MAl9KmURNw,57763
+trilogy/parsing/parse_engine.py,sha256=_Cpq7bq6QyuDcnmYX_fDLGNFNSQPj173cqYsh3V6jH8,60969
 trilogy/parsing/render.py,sha256=Gy_6wVYPwYLf35Iota08sbqveuWILtUhI8MYStcvtJM,12174
-trilogy/parsing/trilogy.lark,sha256=Zn5UHqiNoVKa76r1af9KND0TNdsLIeHpyG3hJjUOOTQ,11000
+trilogy/parsing/trilogy.lark,sha256=uQlyb4RQ6Tpp8lRlXLK_Qb8sD1qzYVYErRI97v22Zak,11317
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.2.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.2.dist-info/METADATA,sha256=rjwj8b_CWxQc82n5HJfPTBYlhtVKNUwIhAkWLusfA7g,7876
-pytrilogy-0.0.2.2.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-pytrilogy-0.0.2.2.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.2.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.2.dist-info/RECORD,,
+pytrilogy-0.0.2.4.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.4.dist-info/METADATA,sha256=R3D4ULv2Y2yJRO0sToJ5FlPPiqkU8H02X035M-vaG9s,7906
+pytrilogy-0.0.2.4.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+pytrilogy-0.0.2.4.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.4.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.4.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.2"
+__version__ = "0.0.2.4"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/enums.py

@@ -119,7 +119,9 @@ class FunctionType(Enum):
 
     # COMPLEX
     INDEX_ACCESS = "index_access"
+    MAP_ACCESS = "map_access"
     ATTR_ACCESS = "attr_access"
+    STRUCT = "struct"
 
     # TEXT AND MAYBE MORE
     SPLIT = "split"

trilogy/core/env_processor.py

@@ -17,6 +17,11 @@ def add_concept(concept: Concept, g: ReferenceGraph):
     for _, pseudonym in concept.pseudonyms.items():
         pseudonym = pseudonym.with_default_grain()
         pseudonym_node = concept_to_node(pseudonym)
+        if (pseudonym_node, node_name) in g.edges and (
+            node_name,
+            pseudonym_node,
+        ) in g.edges:
+            continue
         if pseudonym_node.split("@")[0] == node_name.split("@")[0]:
             continue
         g.add_edge(pseudonym_node, node_name, pseudonym=True)

trilogy/core/functions.py

@@ -202,15 +202,21 @@ def Split(args: list[Concept]) -> Function:
     )
 
 
+def get_index_output_type(
+    arg: Concept,
+) -> DataType | StructType | MapType | ListType | NumericType:
+    if isinstance(arg.datatype, ListType):
+        return arg.datatype.value_data_type
+    elif isinstance(arg.datatype, MapType):
+        return arg.datatype.value_data_type
+    return arg.datatype
+
+
 def IndexAccess(args: list[Concept]):
     return Function(
         operator=FunctionType.INDEX_ACCESS,
         arguments=args,
-        output_datatype=(
-            args[0].datatype.value_data_type
-            if isinstance(args[0].datatype, ListType)
-            else args[0].datatype
-        ),
+        output_datatype=get_index_output_type(args[0]),
         output_purpose=Purpose.PROPERTY,
         valid_inputs=[
             {
@@ -226,11 +232,38 @@ def IndexAccess(args: list[Concept]):
     )
 
 
+def MapAccess(args: list[Concept]):
+    return Function(
+        operator=FunctionType.MAP_ACCESS,
+        arguments=args,
+        output_datatype=get_index_output_type(args[0]),
+        output_purpose=Purpose.PROPERTY,
+        valid_inputs=[
+            {
+                DataType.MAP,
+            },
+            {
+                DataType.INTEGER,
+                DataType.STRING,
+            },
+        ],
+        arg_count=2,
+    )
+
+
+def get_attr_datatype(
+    arg: Concept, lookup
+) -> DataType | ListType | StructType | MapType | NumericType:
+    if isinstance(arg.datatype, StructType):
+        return arg_to_datatype(arg.datatype.fields_map[lookup])
+    return arg.datatype
+
+
 def AttrAccess(args: list[Concept]):
     return Function(
         operator=FunctionType.ATTR_ACCESS,
         arguments=args,
-        output_datatype=args[0].field_map[args[1]].datatype,  # type: ignore
+        output_datatype=get_attr_datatype(args[0], args[1]),  # type: ignore
         output_purpose=Purpose.PROPERTY,
         valid_inputs=[
             {DataType.STRUCT},

trilogy/core/models.py

@@ -67,7 +67,7 @@ from trilogy.core.enums import (
 )
 from trilogy.core.exceptions import UndefinedConceptException, InvalidSyntaxException
 from trilogy.utility import unique
-from collections import UserList
+from collections import UserList, UserDict
 from functools import cached_property
 from abc import ABC
 
@@ -267,7 +267,7 @@ class ListType(BaseModel):
 
 class MapType(BaseModel):
     key_type: DataType
-    content_type: ALL_TYPES
+    value_type: ALL_TYPES
 
     @property
     def data_type(self):
@@ -277,10 +277,26 @@ class MapType(BaseModel):
     def value(self):
         return self.data_type.value
 
+    @property
+    def value_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
+        if isinstance(self.value_type, Concept):
+            return self.value_type.datatype
+        return self.value_type
+
+    @property
+    def key_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
+        if isinstance(self.key_type, Concept):
+            return self.key_type.datatype
+        return self.key_type
+
 
 class StructType(BaseModel):
     fields: List[ALL_TYPES]
-    fields_map: Dict[str, Concept] = Field(default_factory=dict)
+    fields_map: Dict[str, Concept | int | float | str] = Field(default_factory=dict)
 
     @property
     def data_type(self):
@@ -314,6 +330,34 @@ class ListWrapper(Generic[VT], UserList):
         return cls(v, type=arg_to_datatype(v[0]))
 
 
+class MapWrapper(Generic[KT, VT], UserDict):
+    """Used to distinguish parsed map objects from other dicts"""
+
+    def __init__(self, *args, key_type: DataType, value_type: DataType, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.key_type = key_type
+        self.value_type = value_type
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: Callable[[Any], core_schema.CoreSchema]
+    ) -> core_schema.CoreSchema:
+        args = get_args(source_type)
+        if args:
+            schema = handler(Dict[args])  # type: ignore
+        else:
+            schema = handler(Dict)
+        return core_schema.no_info_after_validator_function(cls.validate, schema)
+
+    @classmethod
+    def validate(cls, v):
+        return cls(
+            v,
+            key_type=arg_to_datatype(list(v.keys()).pop()),
+            value_type=arg_to_datatype(list(v.values()).pop()),
+        )
+
+
 class Metadata(BaseModel):
     """Metadata container object.
     TODO: support arbitrary tags"""
@@ -375,6 +419,23 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     def __hash__(self):
         return hash(str(self))
 
+    @property
+    def is_aggregate(self):
+        if (
+            self.lineage
+            and isinstance(self.lineage, Function)
+            and self.lineage.operator in FunctionClass.AGGREGATE_FUNCTIONS.value
+        ):
+            return True
+        if (
+            self.lineage
+            and isinstance(self.lineage, AggregateWrapper)
+            and self.lineage.function.operator
+            in FunctionClass.AGGREGATE_FUNCTIONS.value
+        ):
+            return True
+        return False
+
     def with_merge(self, source: Concept, target: Concept, modifiers: List[Modifier]):
         if self.address == source.address:
             new = target.with_grain(self.grain.with_merge(source, target, modifiers))
@@ -949,8 +1010,8 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
     output_purpose: Purpose
     valid_inputs: Optional[
         Union[
-            Set[DataType | ListType | StructType | NumericType],
-            List[Set[DataType | ListType | StructType] | NumericType],
+            Set[DataType | ListType | StructType | MapType | NumericType],
+            List[Set[DataType | ListType | StructType | MapType | NumericType]],
         ]
     ] = None
     arguments: Sequence[
@@ -961,17 +1022,17 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
             int,
             float,
             str,
+            MapWrapper[Any, Any],
             DataType,
             ListType,
+            MapType,
             NumericType,
             DatePart,
             "Parenthetical",
             CaseWhen,
             "CaseElse",
             list,
-            ListWrapper[int],
-            ListWrapper[str],
-            ListWrapper[float],
+            ListWrapper[Any],
         ]
     ]
 
@@ -2342,7 +2403,6 @@ class CTE(BaseModel):
     hidden_concepts: List[Concept] = Field(default_factory=list)
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
-    requires_nesting: bool = True
     base_name_override: Optional[str] = None
     base_alias_override: Optional[str] = None
 
@@ -3141,6 +3201,7 @@ class Environment(BaseModel):
                 v.pseudonyms[source.address] = source
             if v.address == source.address:
                 replacements[k] = target
+                v.pseudonyms[target.address] = target
         self.concepts.update(replacements)
 
         for k, ds in self.datasources.items():
@@ -4115,6 +4176,15 @@ def list_to_wrapper(args):
     return ListWrapper(args, type=types[0])
 
 
+def dict_to_map_wrapper(arg):
+    key_types = [arg_to_datatype(arg) for arg in arg.keys()]
+
+    value_types = [arg_to_datatype(arg) for arg in arg.values()]
+    assert len(set(key_types)) == 1
+    assert len(set(key_types)) == 1
+    return MapWrapper(arg, key_type=key_types[0], value_type=value_types[0])
+
+
 def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | NumericType:
     if isinstance(arg, Function):
         return arg.output_datatype
@@ -4143,5 +4213,7 @@ def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | Numeric
     elif isinstance(arg, list):
         wrapper = list_to_wrapper(arg)
         return ListType(type=wrapper.type)
+    elif isinstance(arg, MapWrapper):
+        return MapType(key_type=arg.key_type, value_type=arg.value_type)
     else:
         raise ValueError(f"Cannot parse arg datatype for arg of raw type {type(arg)}")

trilogy/core/optimization.py

@@ -1,10 +1,10 @@
 from trilogy.core.models import (
     CTE,
     SelectStatement,
-    PersistStatement,
     MultiSelectStatement,
+    Conditional,
 )
-from trilogy.core.enums import PurposeLineage
+from trilogy.core.enums import PurposeLineage, BooleanOperator
 from trilogy.constants import logger, CONFIG
 from trilogy.core.optimizations import (
     OptimizationRule,
@@ -42,34 +42,45 @@ def gen_inverse_map(input: list[CTE]) -> dict[str, list[CTE]]:
     return inverse_map
 
 
-def is_direct_return_eligible(
-    cte: CTE, select: SelectStatement | PersistStatement | MultiSelectStatement
-) -> bool:
-    if isinstance(select, (PersistStatement, MultiSelectStatement)):
-        return False
+def is_direct_return_eligible(cte: CTE) -> CTE | None:
+    # if isinstance(select, (PersistStatement, MultiSelectStatement)):
+    #     return False
+    if len(cte.parent_ctes) != 1:
+        return None
+    direct_parent = cte.parent_ctes[0]
+
+    output_addresses = set([x.address for x in cte.output_columns])
+    parent_output_addresses = set([x.address for x in direct_parent.output_columns])
+    if not output_addresses.issubset(parent_output_addresses):
+        return None
+    if not direct_parent.grain == cte.grain:
+        return None
     derived_concepts = [
         c
         for c in cte.source.output_concepts + cte.source.hidden_concepts
         if c not in cte.source.input_concepts
     ]
-    eligible = True
     conditions = (
-        set(x.address for x in select.where_clause.concept_arguments)
-        if select.where_clause
+        set(x.address for x in direct_parent.condition.concept_arguments)
+        if direct_parent.condition
         else set()
     )
     for x in derived_concepts:
         if x.derivation == PurposeLineage.WINDOW:
-            return False
+            return None
         if x.derivation == PurposeLineage.UNNEST:
-            return False
+            return None
         if x.derivation == PurposeLineage.AGGREGATE:
             if x.address in conditions:
-                return False
+                return None
+    # handling top level nodes that require unpacking
+    for x in cte.output_columns:
+        if x.derivation == PurposeLineage.UNNEST:
+            return None
     logger.info(
-        f"[Optimization][EarlyReturn] Upleveling output select to final CTE with derived_concepts {[x.address for x in derived_concepts]}"
+        f"[Optimization][EarlyReturn] Removing redundant output CTE with derived_concepts {[x.address for x in derived_concepts]}"
     )
-    return eligible
+    return direct_parent
 
 
 def sort_select_output(cte: CTE, query: SelectStatement | MultiSelectStatement):
@@ -90,23 +101,27 @@ def optimize_ctes(
     input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
 ) -> list[CTE]:
 
-    if CONFIG.optimizations.direct_return and is_direct_return_eligible(
-        root_cte, select
+    direct_parent: CTE | None = root_cte
+    while CONFIG.optimizations.direct_return and (
+        direct_parent := is_direct_return_eligible(root_cte)
     ):
-        root_cte.order_by = select.order_by
-        root_cte.limit = select.limit
-        # if select.where_clause:
-
-        #     if root_cte.condition:
-        #         root_cte.condition = Conditional(
-        #             left=root_cte.condition,
-        #             operator=BooleanOperator.AND,
-        #             right=select.where_clause.conditional,
-        #         )
-        #     else:
-        #         root_cte.condition = select.where_clause.conditional
-        root_cte.requires_nesting = False
-        sort_select_output(root_cte, select)
+        direct_parent.order_by = root_cte.order_by
+        direct_parent.limit = root_cte.limit
+        direct_parent.hidden_concepts = (
+            root_cte.hidden_concepts + direct_parent.hidden_concepts
+        )
+        if root_cte.condition:
+            if direct_parent.condition:
+                direct_parent.condition = Conditional(
+                    left=direct_parent.condition,
+                    operator=BooleanOperator.AND,
+                    right=root_cte.condition,
+                )
+            else:
+                direct_parent.condition = root_cte.condition
+        root_cte = direct_parent
+
+    sort_select_output(root_cte, select)
 
     REGISTERED_RULES: list["OptimizationRule"] = []
     if CONFIG.optimizations.constant_inlining:

trilogy/core/optimizations/predicate_pushdown.py

@@ -3,18 +3,36 @@ from trilogy.core.models import (
     Conditional,
     BooleanOperator,
     Datasource,
+    SubselectComparison,
+    Comparison,
+    Parenthetical,
 )
 from trilogy.core.optimizations.base_optimization import OptimizationRule
+from trilogy.core.processing.utility import is_scalar_condition
 
 
-def decompose_condition(conditional: Conditional):
-    chunks = []
+def decompose_condition(
+    conditional: Conditional,
+) -> list[SubselectComparison | Comparison | Conditional | Parenthetical]:
+    chunks: list[SubselectComparison | Comparison | Conditional | Parenthetical] = []
     if conditional.operator == BooleanOperator.AND:
-        for val in [conditional.left, conditional.right]:
-            if isinstance(val, Conditional):
-                chunks.extend(decompose_condition(val))
-            else:
-                chunks.append(val)
+        if not (
+            isinstance(
+                conditional.left,
+                (SubselectComparison, Comparison, Conditional, Parenthetical),
+            )
+            and isinstance(
+                conditional.right,
+                (SubselectComparison, Comparison, Conditional, Parenthetical),
+            )
+        ):
+            chunks.append(conditional)
+        else:
+            for val in [conditional.left, conditional.right]:
+                if isinstance(val, Conditional):
+                    chunks.extend(decompose_condition(val))
+                else:
+                    chunks.append(val)
     else:
         chunks.append(conditional)
     return chunks
@@ -128,7 +146,14 @@ class PredicatePushdown(OptimizationRule):
         )
         optimized = False
         for candidate in candidates:
-            self.debug(f"Checking candidate {candidate}")
+            if not is_scalar_condition(candidate):
+                self.debug(
+                    f"Skipping {candidate} as not a basic [no aggregate, etc] condition"
+                )
+                continue
+            self.log(
+                f"Checking candidate {candidate}, {type(candidate)}, scalar: {is_scalar_condition(candidate)}"
+            )
             for parent_cte in cte.parent_ctes:
                 local_pushdown = self._check_parent(
                     parent_cte=parent_cte, candidate=candidate, inverse_map=inverse_map

trilogy/core/processing/concept_strategies_v3.py

@@ -336,8 +336,11 @@ def validate_concept(
     virtual_addresses: set[str],
     found_map: dict[str, set[Concept]],
     accept_partial: bool,
+    seen: set[str],
 ):
+
     found_map[str(node)].add(concept)
+    seen.add(concept.address)
     if concept not in node.partial_concepts:
 
         found_addresses.add(concept.address)
@@ -357,6 +360,8 @@ def validate_concept(
     for _, v in concept.pseudonyms.items():
         if v.address == concept.address:
             return
+        if v.address in seen:
+            return
         validate_concept(
             v,
             node,
@@ -366,6 +371,7 @@ def validate_concept(
             virtual_addresses,
             found_map,
             accept_partial,
+            seen=seen,
         )
 
 
@@ -379,8 +385,10 @@ def validate_stack(
     non_partial_addresses: set[str] = set()
     partial_addresses: set[str] = set()
     virtual_addresses: set[str] = set()
+    seen: set[str] = set()
     for node in stack:
         resolved = node.resolve()
+
         for concept in resolved.output_concepts:
             validate_concept(
                 concept,
@@ -391,6 +399,7 @@ def validate_stack(
                 virtual_addresses,
                 found_map,
                 accept_partial,
+                seen,
             )
         for concept in node.virtual_output_concepts:
             if concept.address in non_partial_addresses:
@@ -480,6 +489,7 @@ def _search_concepts(
 
     found: set[str] = set()
     skip: set[str] = set()
+    virtual: set[str] = set()
     stack: List[StrategyNode] = []
     complete = ValidationResult.INCOMPLETE
 

trilogy/core/processing/node_generators/basic_node.py

@@ -44,7 +44,7 @@ def gen_basic_node(
                         list(optional_set) + [concept],
                     )
                 )
-
+    # check for the concept by itself
     for attempt, basic_output in reversed(attempts):
         partials = []
         attempt = unique(attempt, "address")