pytrilogy 0.0.2.1__py3-none-any.whl → 0.0.2.3__py3-none-any.whl

{pytrilogy-0.0.2.1.dist-info → pytrilogy-0.0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.1
+Version: 0.0.2.3
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -247,7 +247,7 @@ N/A, only supports default auth. In python you can pass in a custom client.
 
 
 > [!TIP]
-> The CLI can also be used for formatting. PreQL has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
+> The CLI can also be used for formatting. Trilogy has a default formatting style that should always be adhered to. `trilogy fmt <path to trilogy file>`
 
 
 ## More Examples
@@ -284,7 +284,7 @@ but all are worth checking out. Please open PRs/comment for anything missed!
 
 #### CONCEPT
 
-Types: `string | int | float | bool | date | datetime | time | timestamp | interval`;
+Types: `string | int | float | bool | date | datetime | time | numeric(scale, precision) | timestamp | interval`;
 
 Key:
 `key <name> <type>;`

{pytrilogy-0.0.2.1.dist-info → pytrilogy-0.0.2.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-trilogy/__init__.py,sha256=f4K50J-SBruXMjFtcHLn_pB9eN7oCkbwqmtEjcxjSoM,290
+trilogy/__init__.py,sha256=P6eUqLnB3qFOCWULAL337Zc9648DuD0E0pGbTqzHChw,290
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=KIvi-cgU4R9urNgDdGiCsRkCrzjAfM4xGHhZb1SVy2w,881
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
@@ -8,36 +8,36 @@ trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=DWKPuShM_DbyAenjNFOAxcIH2l0QC9WgIwqZKPQqk_o,5850
-trilogy/core/env_processor.py,sha256=Wpy-iiduBwHntTsQTYWBO1O0i3Ij9_VoL3d4IWDEoj4,2126
+trilogy/core/enums.py,sha256=zf5VvxkXB9wz-cIMwGB4d00KXOTYfBhkbIp6Ff45YVs,5880
+trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
 trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
 trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
-trilogy/core/functions.py,sha256=xISGHMiUR9hFyoZe2l3VLTMO6UEtyGHQcMuovzlVRqw,9492
+trilogy/core/functions.py,sha256=MSNe-OLAEvj72qn5mif3ce9ncRh3ZATpX3V300yAm98,10125
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=432UOrJNzdY0_2vCbeBrNBDQOG_v-WaWqYSniM1CJpE,134387
-trilogy/core/optimization.py,sha256=iX8BCh2BIEPJ6cYlcn7piDKFfig47azphhyqkwkYJM4,4291
-trilogy/core/query_processor.py,sha256=PROehLEOMOuKy1EAUeIgdfGq1LrJsX0N7Z2qDED6S30,17580
+trilogy/core/models.py,sha256=eZKTOD4mXGCH78YE0xNlHex15t1Ur4d6CGAZ1QNcBCQ,136597
+trilogy/core/optimization.py,sha256=A8S9C9H5RcQcFSQLYtEEBnm-r1CW_e9GEWlLK7q3MqA,4930
+trilogy/core/query_processor.py,sha256=BrP4x96Nbzfq6MfxYzxnOL6zYCLIoHLEii65MBT-Daw,17731
 trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
 trilogy/core/optimizations/inline_datasource.py,sha256=KiwZ4fnRnSOVK8zzdwMAkafszo3fNn5LY8xgb0X1CbI,3194
-trilogy/core/optimizations/predicate_pushdown.py,sha256=mduJn4zzKmLVNUja60ZlY6LlNp389OwjEbq2Rj8K6co,5389
+trilogy/core/optimizations/predicate_pushdown.py,sha256=HWHHG0hejBoIv985hVlsh3INPTrx8EGwCRWeeiPEHsc,7374
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=hmbck3a23N61HfMwC2z3lxA6Yms9TP7Lm3Aw7GiDcMA,23765
+trilogy/core/processing/concept_strategies_v3.py,sha256=Ssekd9sA7W-JgAif-Bp7ID1pZujaKJzlcP_KuoH2h4I,23938
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=hgfPfz5FlghN8edIVGTQ21eDUof5EMvnl3Vu64hjfqY,13289
+trilogy/core/processing/utility.py,sha256=L92nMZQqVSb_aOtYGi2rkvDD10e8WWg5-oQ7ApJBlI4,12374
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
-trilogy/core/processing/node_generators/basic_node.py,sha256=uD6noJOgrtmhjb-1aa-vD8sQ97s9Tya6wbso7V_AdyE,2819
+trilogy/core/processing/node_generators/basic_node.py,sha256=4242PNGTCm2tklqMIkqVu5Iv4m_IeTnOYXxDveuCDZM,2856
 trilogy/core/processing/node_generators/common.py,sha256=liZDth7mvhkF_sUFXK7JitJsiaKD132w3ySLbF7l-nE,8956
 trilogy/core/processing/node_generators/filter_node.py,sha256=5B7UCK84A9lGgJ7EjCiC7YGWWM1xPyQGHkBwZgYypFM,4585
 trilogy/core/processing/node_generators/group_node.py,sha256=xWI1xNIXEOj6jlRGD9hcv2_vVNvY6lpzJl6pQ8HuFBE,2988
 trilogy/core/processing/node_generators/group_to_node.py,sha256=BzPdYwzoo8gRMH7BDffTTXq4z-mjfCEzvfB5I-P0_nw,2941
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=YivTVrTEUJuQbQAHFVB2lUD4mJYQ-kTVVpwY4oT9A-Y,14664
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=JDQEIs43BJCgz0yofztcjd7J20Ybk4W40IJG_6blQJI,12168
 trilogy/core/processing/node_generators/rowset_node.py,sha256=2ROd2c1_o4h5sbnDdaiL1o9UiiwyYw-B6OKyfHYNe0A,6188
-trilogy/core/processing/node_generators/select_node.py,sha256=uk02lRvSJB1gWYRcSNfRZ8aYk3fME_tgg0I6EZyuxkI,19052
+trilogy/core/processing/node_generators/select_node.py,sha256=XGcz74XsWv5POWhV2gGC_jbi7T9g7sY-VtP3sjlppFc,19057
 trilogy/core/processing/node_generators/unnest_node.py,sha256=6CH66eGwpadNX7TzUhWZ8aqIisOtQeHINbLV6X3QBUk,1779
 trilogy/core/processing/node_generators/window_node.py,sha256=9nXUXUgQrNczU1gaOqhOZPNzCUxw-lkxt0R7HORI6ss,2582
 trilogy/core/processing/nodes/__init__.py,sha256=-8ONG7Nqa6sNLpst5wNNyMigyaHRfUKGdE3xce8E3mM,3998
@@ -49,14 +49,14 @@ trilogy/core/processing/nodes/select_node_v2.py,sha256=COu-WPuyabGAc3HTkJB-_7eNz
 trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=EZCP3xU8-B4xpU1pFFew4CGtkNWignbyPll6OTENFLg,29832
+trilogy/dialect/base.py,sha256=bjoDwVAARpCVLjJvnfNpr1JtvoamkAAYZxw-G5QRV_A,27845
 trilogy/dialect/bigquery.py,sha256=BAN2o0SOuLvzLYLV7Sc11woapS5L7Cc36aMBeaphW5k,2905
 trilogy/dialect/common.py,sha256=5jdOHWIj3Xv8F8y5mnyWHLjxD_we2fncM-ZnNetJP7U,2781
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
 trilogy/dialect/duckdb.py,sha256=JQPnR5F39iDQXfTccKhbq7xWYGzYc0dPpn5vo0Qd_Vk,3076
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
 trilogy/dialect/postgres.py,sha256=r47xbCA7nfEYENofiVfLZ-SnReNfDmUmW4OSHVkkP4E,3206
-trilogy/dialect/presto.py,sha256=ouSVNbs7e3eEc20emLfxKyRjdINSibPGUFL4X9OwboA,3163
+trilogy/dialect/presto.py,sha256=evb7Tq77l4iBrZnBeSPfEdmdObgrca-p5H72CIrEips,3221
 trilogy/dialect/snowflake.py,sha256=N3HknYgN-fjD7BLX1Ucj-ss_ku2Ox8DgLsF3BIHutHo,2941
 trilogy/dialect/sql_server.py,sha256=HX68vNTrcDaTnOxe6Zbx_PBgrO42e2VuThxO6CYQ2cY,3026
 trilogy/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -65,18 +65,18 @@ trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2
 trilogy/hooks/query_debugger.py,sha256=NDChfkPmmW-KINa4TaQmDe_adGiwsKFdGLDSYpbodeU,4282
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=8MDRPccWymtVkHVQMzESjL5tOH79flWU5jo6Ys-C5UQ,5963
+trilogy/parsing/common.py,sha256=UZdG51pHNMGokvHjUxv7Zt5Xrh2snxfZuVOZVyg9yZ8,6091
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=hPvPZuO_ZBjoKRMh-bKzhnt6oo8R5l4gL9bJqZoMUfs,57619
-trilogy/parsing/render.py,sha256=ym8DcgU-frNxd16ThWYGJFyChF-d0yd2gp9tEA6RFJU,12059
-trilogy/parsing/trilogy.lark,sha256=4VXqLWOdxJsfZoJIp9Ug1G3p_LrXRDm5C2yAxaRgq98,10993
+trilogy/parsing/parse_engine.py,sha256=-dXf-W6cOj-l2hDvgb7TWJKLZZEHAj0kwaw-DT0YKNU,58336
+trilogy/parsing/render.py,sha256=Gy_6wVYPwYLf35Iota08sbqveuWILtUhI8MYStcvtJM,12174
+trilogy/parsing/trilogy.lark,sha256=DiJe_XQZ9lXVUW5AF9eZl531lPAw7mm9MMeHCGkw7kc,11148
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.1.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.1.dist-info/METADATA,sha256=YN0xCd-8tn38c-px20-D1WgFDUjn-P6J6hNtsyhUFmE,7876
-pytrilogy-0.0.2.1.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
-pytrilogy-0.0.2.1.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.1.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.1.dist-info/RECORD,,
+pytrilogy-0.0.2.3.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.3.dist-info/METADATA,sha256=9Q2murSlwgsOWkr6A3FkEWWtlPKoI-VTmkONLbcnNPU,7906
+pytrilogy-0.0.2.3.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+pytrilogy-0.0.2.3.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.3.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.3.dist-info/RECORD,,

{pytrilogy-0.0.2.1.dist-info → pytrilogy-0.0.2.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (73.0.1)
+Generator: setuptools (74.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.1"
+__version__ = "0.0.2.3"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/enums.py

@@ -119,6 +119,7 @@ class FunctionType(Enum):
 
     # COMPLEX
     INDEX_ACCESS = "index_access"
+    MAP_ACCESS = "map_access"
     ATTR_ACCESS = "attr_access"
 
     # TEXT AND MAYBE MORE

trilogy/core/env_processor.py

@@ -17,6 +17,11 @@ def add_concept(concept: Concept, g: ReferenceGraph):
     for _, pseudonym in concept.pseudonyms.items():
         pseudonym = pseudonym.with_default_grain()
         pseudonym_node = concept_to_node(pseudonym)
+        if (pseudonym_node, node_name) in g.edges and (
+            node_name,
+            pseudonym_node,
+        ) in g.edges:
+            continue
         if pseudonym_node.split("@")[0] == node_name.split("@")[0]:
             continue
         g.add_edge(pseudonym_node, node_name, pseudonym=True)

trilogy/core/functions.py

@@ -202,15 +202,21 @@ def Split(args: list[Concept]) -> Function:
     )
 
 
+def get_index_output_type(
+    arg: Concept,
+) -> DataType | StructType | MapType | ListType | NumericType:
+    if isinstance(arg.datatype, ListType):
+        return arg.datatype.value_data_type
+    elif isinstance(arg.datatype, MapType):
+        return arg.datatype.value_data_type
+    return arg.datatype
+
+
 def IndexAccess(args: list[Concept]):
     return Function(
         operator=FunctionType.INDEX_ACCESS,
         arguments=args,
-        output_datatype=(
-            args[0].datatype.value_data_type
-            if isinstance(args[0].datatype, ListType)
-            else args[0].datatype
-        ),
+        output_datatype=get_index_output_type(args[0]),
         output_purpose=Purpose.PROPERTY,
         valid_inputs=[
             {
@@ -226,6 +232,25 @@ def IndexAccess(args: list[Concept]):
     )
 
 
+def MapAccess(args: list[Concept]):
+    return Function(
+        operator=FunctionType.MAP_ACCESS,
+        arguments=args,
+        output_datatype=get_index_output_type(args[0]),
+        output_purpose=Purpose.PROPERTY,
+        valid_inputs=[
+            {
+                DataType.MAP,
+            },
+            {
+                DataType.INTEGER,
+                DataType.STRING,
+            },
+        ],
+        arg_count=2,
+    )
+
+
 def AttrAccess(args: list[Concept]):
     return Function(
         operator=FunctionType.ATTR_ACCESS,

trilogy/core/models.py

@@ -67,7 +67,7 @@ from trilogy.core.enums import (
 )
 from trilogy.core.exceptions import UndefinedConceptException, InvalidSyntaxException
 from trilogy.utility import unique
-from collections import UserList
+from collections import UserList, UserDict
 from functools import cached_property
 from abc import ABC
 
@@ -267,7 +267,7 @@ class ListType(BaseModel):
 
 class MapType(BaseModel):
     key_type: DataType
-    content_type: ALL_TYPES
+    value_type: ALL_TYPES
 
     @property
     def data_type(self):
@@ -277,6 +277,22 @@ class MapType(BaseModel):
     def value(self):
         return self.data_type.value
 
+    @property
+    def value_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
+        if isinstance(self.value_type, Concept):
+            return self.value_type.datatype
+        return self.value_type
+
+    @property
+    def key_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
+        if isinstance(self.key_type, Concept):
+            return self.key_type.datatype
+        return self.key_type
+
 
 class StructType(BaseModel):
     fields: List[ALL_TYPES]
@@ -314,6 +330,34 @@ class ListWrapper(Generic[VT], UserList):
         return cls(v, type=arg_to_datatype(v[0]))
 
 
+class MapWrapper(Generic[KT, VT], UserDict):
+    """Used to distinguish parsed map objects from other dicts"""
+
+    def __init__(self, *args, key_type: DataType, value_type: DataType, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.key_type = key_type
+        self.value_type = value_type
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: Callable[[Any], core_schema.CoreSchema]
+    ) -> core_schema.CoreSchema:
+        args = get_args(source_type)
+        if args:
+            schema = handler(Dict[args])  # type: ignore
+        else:
+            schema = handler(Dict)
+        return core_schema.no_info_after_validator_function(cls.validate, schema)
+
+    @classmethod
+    def validate(cls, v):
+        return cls(
+            v,
+            key_type=arg_to_datatype(list(v.keys()).pop()),
+            value_type=arg_to_datatype(list(v.values()).pop()),
+        )
+
+
 class Metadata(BaseModel):
     """Metadata container object.
     TODO: support arbitrary tags"""
@@ -949,8 +993,8 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
     output_purpose: Purpose
     valid_inputs: Optional[
         Union[
-            Set[DataType | ListType | StructType | NumericType],
-            List[Set[DataType | ListType | StructType] | NumericType],
+            Set[DataType | ListType | StructType | MapType | NumericType],
+            List[Set[DataType | ListType | StructType | MapType | NumericType]],
         ]
     ] = None
     arguments: Sequence[
@@ -961,17 +1005,17 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
             int,
             float,
             str,
+            MapWrapper[Any, Any],
             DataType,
             ListType,
+            MapType,
             NumericType,
             DatePart,
             "Parenthetical",
             CaseWhen,
             "CaseElse",
             list,
-            ListWrapper[int],
-            ListWrapper[str],
-            ListWrapper[float],
+            ListWrapper[Any],
         ]
     ]
 
@@ -1822,6 +1866,7 @@ class Datasource(Namespaced, BaseModel):
     metadata: DatasourceMetadata = Field(
         default_factory=lambda: DatasourceMetadata(freshness_concept=None)
     )
+    where: Optional[WhereClause] = None
 
     def merge_concept(
         self, source: Concept, target: Concept, modifiers: List[Modifier]
@@ -1837,6 +1882,9 @@ class Datasource(Namespaced, BaseModel):
             for c in self.columns
         ] + original
         self.grain = self.grain.with_merge(source, target, modifiers)
+        self.where = (
+            self.where.with_merge(source, target, modifiers) if self.where else None
+        )
         del self.output_lcl
 
     @property
@@ -1929,6 +1977,7 @@ class Datasource(Namespaced, BaseModel):
             grain=self.grain.with_namespace(namespace),
             address=self.address,
             columns=[c.with_namespace(namespace) for c in self.columns],
+            where=self.where.with_namespace(namespace) if self.where else None,
         )
 
     @cached_property
@@ -2337,7 +2386,6 @@ class CTE(BaseModel):
     hidden_concepts: List[Concept] = Field(default_factory=list)
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
-    requires_nesting: bool = True
     base_name_override: Optional[str] = None
     base_alias_override: Optional[str] = None
 
@@ -2806,6 +2854,9 @@ class EnvironmentDatasourceDict(dict):
                 return self.__getitem__(key.split(".")[1])
             raise
 
+    def values(self) -> ValuesView[Datasource]:  # type: ignore
+        return super().values()
+
 
 class EnvironmentConceptDict(dict):
     def __init__(self, *args, **kwargs) -> None:
@@ -3133,6 +3184,7 @@ class Environment(BaseModel):
                 v.pseudonyms[source.address] = source
             if v.address == source.address:
                 replacements[k] = target
+                v.pseudonyms[target.address] = target
         self.concepts.update(replacements)
 
         for k, ds in self.datasources.items():
@@ -4107,6 +4159,15 @@ def list_to_wrapper(args):
     return ListWrapper(args, type=types[0])
 
 
+def dict_to_map_wrapper(arg):
+    key_types = [arg_to_datatype(arg) for arg in arg.keys()]
+
+    value_types = [arg_to_datatype(arg) for arg in arg.values()]
+    assert len(set(key_types)) == 1
+    assert len(set(key_types)) == 1
+    return MapWrapper(arg, key_type=key_types[0], value_type=value_types[0])
+
+
 def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | NumericType:
     if isinstance(arg, Function):
         return arg.output_datatype
@@ -4135,5 +4196,7 @@ def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | Numeric
     elif isinstance(arg, list):
         wrapper = list_to_wrapper(arg)
         return ListType(type=wrapper.type)
+    elif isinstance(arg, MapWrapper):
+        return MapType(key_type=arg.key_type, value_type=arg.value_type)
     else:
         raise ValueError(f"Cannot parse arg datatype for arg of raw type {type(arg)}")

trilogy/core/optimization.py

@@ -1,10 +1,10 @@
 from trilogy.core.models import (
     CTE,
     SelectStatement,
-    PersistStatement,
     MultiSelectStatement,
+    Conditional,
 )
-from trilogy.core.enums import PurposeLineage
+from trilogy.core.enums import PurposeLineage, BooleanOperator
 from trilogy.constants import logger, CONFIG
 from trilogy.core.optimizations import (
     OptimizationRule,
@@ -42,34 +42,45 @@ def gen_inverse_map(input: list[CTE]) -> dict[str, list[CTE]]:
     return inverse_map
 
 
-def is_direct_return_eligible(
-    cte: CTE, select: SelectStatement | PersistStatement | MultiSelectStatement
-) -> bool:
-    if isinstance(select, (PersistStatement, MultiSelectStatement)):
-        return False
+def is_direct_return_eligible(cte: CTE) -> CTE | None:
+    # if isinstance(select, (PersistStatement, MultiSelectStatement)):
+    #     return False
+    if len(cte.parent_ctes) != 1:
+        return None
+    direct_parent = cte.parent_ctes[0]
+
+    output_addresses = set([x.address for x in cte.output_columns])
+    parent_output_addresses = set([x.address for x in direct_parent.output_columns])
+    if not output_addresses.issubset(parent_output_addresses):
+        return None
+    if not direct_parent.grain == cte.grain:
+        return None
     derived_concepts = [
         c
         for c in cte.source.output_concepts + cte.source.hidden_concepts
         if c not in cte.source.input_concepts
     ]
-    eligible = True
     conditions = (
-        set(x.address for x in select.where_clause.concept_arguments)
-        if select.where_clause
+        set(x.address for x in direct_parent.condition.concept_arguments)
+        if direct_parent.condition
         else set()
     )
     for x in derived_concepts:
         if x.derivation == PurposeLineage.WINDOW:
-            return False
+            return None
         if x.derivation == PurposeLineage.UNNEST:
-            return False
+            return None
         if x.derivation == PurposeLineage.AGGREGATE:
             if x.address in conditions:
-                return False
+                return None
+    # handling top level nodes that require unpacking
+    for x in cte.output_columns:
+        if x.derivation == PurposeLineage.UNNEST:
+            return None
     logger.info(
-        f"[Optimization][EarlyReturn] Upleveling output select to final CTE with derived_concepts {[x.address for x in derived_concepts]}"
+        f"[Optimization][EarlyReturn] Removing redundant output CTE with derived_concepts {[x.address for x in derived_concepts]}"
     )
-    return eligible
+    return direct_parent
 
 
 def sort_select_output(cte: CTE, query: SelectStatement | MultiSelectStatement):
@@ -90,23 +101,27 @@ def optimize_ctes(
     input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
 ) -> list[CTE]:
 
-    if CONFIG.optimizations.direct_return and is_direct_return_eligible(
-        root_cte, select
+    direct_parent: CTE | None = root_cte
+    while CONFIG.optimizations.direct_return and (
+        direct_parent := is_direct_return_eligible(root_cte)
     ):
-        root_cte.order_by = select.order_by
-        root_cte.limit = select.limit
-        # if select.where_clause:
-
-        #     if root_cte.condition:
-        #         root_cte.condition = Conditional(
-        #             left=root_cte.condition,
-        #             operator=BooleanOperator.AND,
-        #             right=select.where_clause.conditional,
-        #         )
-        #     else:
-        #         root_cte.condition = select.where_clause.conditional
-        root_cte.requires_nesting = False
-        sort_select_output(root_cte, select)
+        direct_parent.order_by = root_cte.order_by
+        direct_parent.limit = root_cte.limit
+        direct_parent.hidden_concepts = (
+            root_cte.hidden_concepts + direct_parent.hidden_concepts
+        )
+        if root_cte.condition:
+            if direct_parent.condition:
+                direct_parent.condition = Conditional(
+                    left=direct_parent.condition,
+                    operator=BooleanOperator.AND,
+                    right=root_cte.condition,
+                )
+            else:
+                direct_parent.condition = root_cte.condition
+        root_cte = direct_parent
+
+    sort_select_output(root_cte, select)
 
     REGISTERED_RULES: list["OptimizationRule"] = []
     if CONFIG.optimizations.constant_inlining:

trilogy/core/optimizations/predicate_pushdown.py

@@ -3,18 +3,43 @@ from trilogy.core.models import (
     Conditional,
     BooleanOperator,
     Datasource,
+    SubselectComparison,
+    Comparison,
+    Parenthetical,
+    Function,
+    FilterItem,
+    MagicConstants,
+    Concept,
+    WindowItem,
+    AggregateWrapper,
+    DataType,
 )
 from trilogy.core.optimizations.base_optimization import OptimizationRule
+from trilogy.core.enums import FunctionClass
 
 
-def decompose_condition(conditional: Conditional):
-    chunks = []
+def decompose_condition(
+    conditional: Conditional,
+) -> list[SubselectComparison | Comparison | Conditional | Parenthetical]:
+    chunks: list[SubselectComparison | Comparison | Conditional | Parenthetical] = []
     if conditional.operator == BooleanOperator.AND:
-        for val in [conditional.left, conditional.right]:
-            if isinstance(val, Conditional):
-                chunks.extend(decompose_condition(val))
-            else:
-                chunks.append(val)
+        if not (
+            isinstance(
+                conditional.left,
+                (SubselectComparison, Comparison, Conditional, Parenthetical),
+            )
+            and isinstance(
+                conditional.right,
+                (SubselectComparison, Comparison, Conditional, Parenthetical),
+            )
+        ):
+            chunks.append(conditional)
+        else:
+            for val in [conditional.left, conditional.right]:
+                if isinstance(val, Conditional):
+                    chunks.extend(decompose_condition(val))
+                else:
+                    chunks.append(val)
     else:
         chunks.append(conditional)
     return chunks
@@ -31,6 +56,40 @@ def is_child_of(a, comparison):
     return base
 
 
+def is_basic(
+    element: (
+        int
+        | str
+        | float
+        | list
+        | WindowItem
+        | FilterItem
+        | Concept
+        | Comparison
+        | Conditional
+        | Parenthetical
+        | Function
+        | AggregateWrapper
+        | MagicConstants
+        | DataType
+    ),
+) -> bool:
+    if isinstance(element, Parenthetical):
+        return is_basic(element.content)
+    elif isinstance(element, SubselectComparison):
+        return True
+    elif isinstance(element, Comparison):
+        return is_basic(element.left) and is_basic(element.right)
+    elif isinstance(element, Function):
+        if element.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
+            return False
+    elif isinstance(element, AggregateWrapper):
+        return is_basic(element.function)
+    elif isinstance(element, Conditional):
+        return is_basic(element.left) and is_basic(element.right)
+    return True
+
+
 class PredicatePushdown(OptimizationRule):
 
     def __init__(self, *args, **kwargs) -> None:
@@ -128,7 +187,14 @@ class PredicatePushdown(OptimizationRule):
         )
         optimized = False
         for candidate in candidates:
-            self.debug(f"Checking candidate {candidate}")
+            if not is_basic(candidate):
+                self.debug(
+                    f"Skipping {candidate} as not a basic [no aggregate, etc] condition"
+                )
+                continue
+            self.log(
+                f"Checking candidate {candidate}, {type(candidate)}, {is_basic(candidate)}"
+            )
             for parent_cte in cte.parent_ctes:
                 local_pushdown = self._check_parent(
                     parent_cte=parent_cte, candidate=candidate, inverse_map=inverse_map

trilogy/core/processing/concept_strategies_v3.py

@@ -336,8 +336,11 @@ def validate_concept(
     virtual_addresses: set[str],
     found_map: dict[str, set[Concept]],
     accept_partial: bool,
+    seen: set[str],
 ):
+
     found_map[str(node)].add(concept)
+    seen.add(concept.address)
     if concept not in node.partial_concepts:
 
         found_addresses.add(concept.address)
@@ -357,6 +360,8 @@ def validate_concept(
     for _, v in concept.pseudonyms.items():
         if v.address == concept.address:
             return
+        if v.address in seen:
+            return
         validate_concept(
             v,
             node,
@@ -366,6 +371,7 @@ def validate_concept(
             virtual_addresses,
             found_map,
             accept_partial,
+            seen=seen,
         )
 
 
@@ -379,8 +385,10 @@ def validate_stack(
     non_partial_addresses: set[str] = set()
     partial_addresses: set[str] = set()
     virtual_addresses: set[str] = set()
+    seen: set[str] = set()
     for node in stack:
         resolved = node.resolve()
+
         for concept in resolved.output_concepts:
             validate_concept(
                 concept,
@@ -391,6 +399,7 @@ def validate_stack(
                 virtual_addresses,
                 found_map,
                 accept_partial,
+                seen,
             )
         for concept in node.virtual_output_concepts:
             if concept.address in non_partial_addresses: