pytrilogy 0.0.2.3__py3-none-any.whl → 0.0.2.4__py3-none-any.whl

{pytrilogy-0.0.2.3.dist-info → pytrilogy-0.0.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.3
+Version: 0.0.2.4
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.3.dist-info → pytrilogy-0.0.2.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-trilogy/__init__.py,sha256=P6eUqLnB3qFOCWULAL337Zc9648DuD0E0pGbTqzHChw,290
+trilogy/__init__.py,sha256=sspl5Npt0npzVRC6wCwiYXa0nK0E7a_biwiL2T4kbjM,290
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=KIvi-cgU4R9urNgDdGiCsRkCrzjAfM4xGHhZb1SVy2w,881
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
@@ -8,30 +8,30 @@ trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=zf5VvxkXB9wz-cIMwGB4d00KXOTYfBhkbIp6Ff45YVs,5880
+trilogy/core/enums.py,sha256=ozgw8G0s4CdbL6IUsX5nNT_ZGd4mMkoo8_EGLZXeQYY,5902
 trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
 trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
 trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
-trilogy/core/functions.py,sha256=MSNe-OLAEvj72qn5mif3ce9ncRh3ZATpX3V300yAm98,10125
+trilogy/core/functions.py,sha256=ARJAyBjeS415-54k3G_bx807rkPZonEulMaLRxSP7vU,10371
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=eZKTOD4mXGCH78YE0xNlHex15t1Ur4d6CGAZ1QNcBCQ,136597
+trilogy/core/models.py,sha256=KVPUfupZVhQxJZCViMP7PELzvNR7WOwkjmqqgEM4oeM,137120
 trilogy/core/optimization.py,sha256=A8S9C9H5RcQcFSQLYtEEBnm-r1CW_e9GEWlLK7q3MqA,4930
-trilogy/core/query_processor.py,sha256=BrP4x96Nbzfq6MfxYzxnOL6zYCLIoHLEii65MBT-Daw,17731
+trilogy/core/query_processor.py,sha256=6JiX6YqsTgnsEkIxcmWFxJuM6kRJbXbPGRCQgi8g-B8,17941
 trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
 trilogy/core/optimizations/inline_datasource.py,sha256=KiwZ4fnRnSOVK8zzdwMAkafszo3fNn5LY8xgb0X1CbI,3194
-trilogy/core/optimizations/predicate_pushdown.py,sha256=HWHHG0hejBoIv985hVlsh3INPTrx8EGwCRWeeiPEHsc,7374
+trilogy/core/optimizations/predicate_pushdown.py,sha256=iVZV_BVaL3I6wlELPOuJAiuF0heOM7bCUnNqyCxxRVw,6370
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=Ssekd9sA7W-JgAif-Bp7ID1pZujaKJzlcP_KuoH2h4I,23938
+trilogy/core/processing/concept_strategies_v3.py,sha256=xCPfg1ySHP6C-u777kqnV8eqIK1X68hqwvRcw61q5X0,23968
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=L92nMZQqVSb_aOtYGi2rkvDD10e8WWg5-oQ7ApJBlI4,12374
+trilogy/core/processing/utility.py,sha256=rhnkn3KgB1MW17mZYSEZ8XFEbC__B7PPmZgpDql4Myg,13614
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
 trilogy/core/processing/node_generators/basic_node.py,sha256=4242PNGTCm2tklqMIkqVu5Iv4m_IeTnOYXxDveuCDZM,2856
-trilogy/core/processing/node_generators/common.py,sha256=liZDth7mvhkF_sUFXK7JitJsiaKD132w3ySLbF7l-nE,8956
-trilogy/core/processing/node_generators/filter_node.py,sha256=5B7UCK84A9lGgJ7EjCiC7YGWWM1xPyQGHkBwZgYypFM,4585
+trilogy/core/processing/node_generators/common.py,sha256=uLrqbEsuYREswiTfuTQuxldO9SsS6ylZEQT_q9iB2uY,8948
+trilogy/core/processing/node_generators/filter_node.py,sha256=kVsK9O9QWUgknui3ESHJJCnFY9dVMCHlGpHXn1QtXkU,4824
 trilogy/core/processing/node_generators/group_node.py,sha256=xWI1xNIXEOj6jlRGD9hcv2_vVNvY6lpzJl6pQ8HuFBE,2988
 trilogy/core/processing/node_generators/group_to_node.py,sha256=BzPdYwzoo8gRMH7BDffTTXq4z-mjfCEzvfB5I-P0_nw,2941
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
@@ -43,40 +43,40 @@ trilogy/core/processing/node_generators/window_node.py,sha256=9nXUXUgQrNczU1gaOq
 trilogy/core/processing/nodes/__init__.py,sha256=-8ONG7Nqa6sNLpst5wNNyMigyaHRfUKGdE3xce8E3mM,3998
 trilogy/core/processing/nodes/base_node.py,sha256=q1G4FO3Q4OuboWL8EaJgJ4jcnnqM9QsothMIcF2ot1g,11161
 trilogy/core/processing/nodes/filter_node.py,sha256=ImUHYj5G5eHbOtki-ZMR31Op38wKOjR1xu_ETiE_D6I,1932
-trilogy/core/processing/nodes/group_node.py,sha256=5dqGlonXG5Ou7HRSsTStbM1KyxeDSZROz9qKTjOUMEc,5166
+trilogy/core/processing/nodes/group_node.py,sha256=j3vtFQIn_t4EfCke8dGULlwVkG2KKPYfgGwo2rvgcww,6322
 trilogy/core/processing/nodes/merge_node.py,sha256=W7ryL9rceylCQGNgcuGY-dSiNoIz8qrbpJXKzg_Niwo,14056
 trilogy/core/processing/nodes/select_node_v2.py,sha256=COu-WPuyabGAc3HTkJB-_7eNzR4OgxIL4YDuZIRrST0,7179
 trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=bjoDwVAARpCVLjJvnfNpr1JtvoamkAAYZxw-G5QRV_A,27845
-trilogy/dialect/bigquery.py,sha256=BAN2o0SOuLvzLYLV7Sc11woapS5L7Cc36aMBeaphW5k,2905
+trilogy/dialect/base.py,sha256=aE8iJJhkGtZE04mYm_wQx3ebQKYbcRsKISUd9mbrZ2A,28782
+trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
 trilogy/dialect/common.py,sha256=5jdOHWIj3Xv8F8y5mnyWHLjxD_we2fncM-ZnNetJP7U,2781
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
-trilogy/dialect/duckdb.py,sha256=JQPnR5F39iDQXfTccKhbq7xWYGzYc0dPpn5vo0Qd_Vk,3076
+trilogy/dialect/duckdb.py,sha256=JEdhctDxZr-FEodot5V4De6JnrQDJT02zbjX48j6ay8,3125
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
-trilogy/dialect/postgres.py,sha256=r47xbCA7nfEYENofiVfLZ-SnReNfDmUmW4OSHVkkP4E,3206
-trilogy/dialect/presto.py,sha256=evb7Tq77l4iBrZnBeSPfEdmdObgrca-p5H72CIrEips,3221
-trilogy/dialect/snowflake.py,sha256=N3HknYgN-fjD7BLX1Ucj-ss_ku2Ox8DgLsF3BIHutHo,2941
-trilogy/dialect/sql_server.py,sha256=HX68vNTrcDaTnOxe6Zbx_PBgrO42e2VuThxO6CYQ2cY,3026
+trilogy/dialect/postgres.py,sha256=ev1RJZsC8BB3vJSxJ4q-TTYqZ4Hk1NXUtuRkLrQEBX0,3254
+trilogy/dialect/presto.py,sha256=InvHDXsGLDG3E3ffl7hGZ-9y3vH8RO6UhNE0OSra01g,3269
+trilogy/dialect/snowflake.py,sha256=_Bf4XO7-nImMv9XCSsTfVM3g2f_KHdO17VTa9J-HgSM,2989
+trilogy/dialect/sql_server.py,sha256=OtXbm1v6NIGyXeC5i18ojUvLeqescc_Pbv3EASUBB94,3074
 trilogy/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/hooks/base_hook.py,sha256=Xkb-A2qCHozYjum0A36zOy5PwTVwrP3NLDF0U2GpgHo,1100
 trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2467
 trilogy/hooks/query_debugger.py,sha256=NDChfkPmmW-KINa4TaQmDe_adGiwsKFdGLDSYpbodeU,4282
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=UZdG51pHNMGokvHjUxv7Zt5Xrh2snxfZuVOZVyg9yZ8,6091
+trilogy/parsing/common.py,sha256=aYmLtsEk8VJj5yYASzUcTf0i0lXVsZBbZJwphwokxK4,6092
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=-dXf-W6cOj-l2hDvgb7TWJKLZZEHAj0kwaw-DT0YKNU,58336
+trilogy/parsing/parse_engine.py,sha256=_Cpq7bq6QyuDcnmYX_fDLGNFNSQPj173cqYsh3V6jH8,60969
 trilogy/parsing/render.py,sha256=Gy_6wVYPwYLf35Iota08sbqveuWILtUhI8MYStcvtJM,12174
-trilogy/parsing/trilogy.lark,sha256=DiJe_XQZ9lXVUW5AF9eZl531lPAw7mm9MMeHCGkw7kc,11148
+trilogy/parsing/trilogy.lark,sha256=uQlyb4RQ6Tpp8lRlXLK_Qb8sD1qzYVYErRI97v22Zak,11317
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.3.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.3.dist-info/METADATA,sha256=9Q2murSlwgsOWkr6A3FkEWWtlPKoI-VTmkONLbcnNPU,7906
-pytrilogy-0.0.2.3.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-pytrilogy-0.0.2.3.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.3.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.3.dist-info/RECORD,,
+pytrilogy-0.0.2.4.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.4.dist-info/METADATA,sha256=R3D4ULv2Y2yJRO0sToJ5FlPPiqkU8H02X035M-vaG9s,7906
+pytrilogy-0.0.2.4.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+pytrilogy-0.0.2.4.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.4.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.4.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.3"
+__version__ = "0.0.2.4"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/enums.py

@@ -121,6 +121,7 @@ class FunctionType(Enum):
     INDEX_ACCESS = "index_access"
     MAP_ACCESS = "map_access"
     ATTR_ACCESS = "attr_access"
+    STRUCT = "struct"
 
     # TEXT AND MAYBE MORE
     SPLIT = "split"

trilogy/core/functions.py

@@ -251,11 +251,19 @@ def MapAccess(args: list[Concept]):
     )
 
 
+def get_attr_datatype(
+    arg: Concept, lookup
+) -> DataType | ListType | StructType | MapType | NumericType:
+    if isinstance(arg.datatype, StructType):
+        return arg_to_datatype(arg.datatype.fields_map[lookup])
+    return arg.datatype
+
+
 def AttrAccess(args: list[Concept]):
     return Function(
         operator=FunctionType.ATTR_ACCESS,
         arguments=args,
-        output_datatype=args[0].field_map[args[1]].datatype,  # type: ignore
+        output_datatype=get_attr_datatype(args[0], args[1]),  # type: ignore
         output_purpose=Purpose.PROPERTY,
         valid_inputs=[
             {DataType.STRUCT},

trilogy/core/models.py

@@ -296,7 +296,7 @@ class MapType(BaseModel):
 
 class StructType(BaseModel):
     fields: List[ALL_TYPES]
-    fields_map: Dict[str, Concept] = Field(default_factory=dict)
+    fields_map: Dict[str, Concept | int | float | str] = Field(default_factory=dict)
 
     @property
     def data_type(self):
@@ -419,6 +419,23 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     def __hash__(self):
         return hash(str(self))
 
+    @property
+    def is_aggregate(self):
+        if (
+            self.lineage
+            and isinstance(self.lineage, Function)
+            and self.lineage.operator in FunctionClass.AGGREGATE_FUNCTIONS.value
+        ):
+            return True
+        if (
+            self.lineage
+            and isinstance(self.lineage, AggregateWrapper)
+            and self.lineage.function.operator
+            in FunctionClass.AGGREGATE_FUNCTIONS.value
+        ):
+            return True
+        return False
+
     def with_merge(self, source: Concept, target: Concept, modifiers: List[Modifier]):
         if self.address == source.address:
             new = target.with_grain(self.grain.with_merge(source, target, modifiers))

trilogy/core/optimizations/predicate_pushdown.py

@@ -6,16 +6,9 @@ from trilogy.core.models import (
     SubselectComparison,
     Comparison,
     Parenthetical,
-    Function,
-    FilterItem,
-    MagicConstants,
-    Concept,
-    WindowItem,
-    AggregateWrapper,
-    DataType,
 )
 from trilogy.core.optimizations.base_optimization import OptimizationRule
-from trilogy.core.enums import FunctionClass
+from trilogy.core.processing.utility import is_scalar_condition
 
 
 def decompose_condition(
@@ -56,40 +49,6 @@ def is_child_of(a, comparison):
     return base
 
 
-def is_basic(
-    element: (
-        int
-        | str
-        | float
-        | list
-        | WindowItem
-        | FilterItem
-        | Concept
-        | Comparison
-        | Conditional
-        | Parenthetical
-        | Function
-        | AggregateWrapper
-        | MagicConstants
-        | DataType
-    ),
-) -> bool:
-    if isinstance(element, Parenthetical):
-        return is_basic(element.content)
-    elif isinstance(element, SubselectComparison):
-        return True
-    elif isinstance(element, Comparison):
-        return is_basic(element.left) and is_basic(element.right)
-    elif isinstance(element, Function):
-        if element.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
-            return False
-    elif isinstance(element, AggregateWrapper):
-        return is_basic(element.function)
-    elif isinstance(element, Conditional):
-        return is_basic(element.left) and is_basic(element.right)
-    return True
-
-
 class PredicatePushdown(OptimizationRule):
 
     def __init__(self, *args, **kwargs) -> None:
@@ -187,13 +146,13 @@ class PredicatePushdown(OptimizationRule):
         )
         optimized = False
         for candidate in candidates:
-            if not is_basic(candidate):
+            if not is_scalar_condition(candidate):
                 self.debug(
                     f"Skipping {candidate} as not a basic [no aggregate, etc] condition"
                 )
                 continue
             self.log(
-                f"Checking candidate {candidate}, {type(candidate)}, {is_basic(candidate)}"
+                f"Checking candidate {candidate}, {type(candidate)}, scalar: {is_scalar_condition(candidate)}"
             )
             for parent_cte in cte.parent_ctes:
                 local_pushdown = self._check_parent(

trilogy/core/processing/concept_strategies_v3.py

@@ -489,6 +489,7 @@ def _search_concepts(
 
     found: set[str] = set()
     skip: set[str] = set()
+    virtual: set[str] = set()
     stack: List[StrategyNode] = []
     complete = ValidationResult.INCOMPLETE
 

trilogy/core/processing/node_generators/common.py

@@ -47,7 +47,7 @@ def resolve_function_parent_concepts(concept: Concept) -> List[Concept]:
 
 def resolve_filter_parent_concepts(
     concept: Concept,
-) -> Tuple[Concept, List[Concept], List[Concept]]:
+) -> Tuple[Concept, List[Concept], List[Tuple[Concept, ...]]]:
     if not isinstance(concept.lineage, FilterItem):
         raise ValueError(
             f"Concept {concept} lineage is not filter item, is {type(concept.lineage)}"
@@ -58,7 +58,7 @@ def resolve_filter_parent_concepts(
     base_rows += concept.lineage.where.row_arguments
     # TODO: pass tuple groups through
     for ctuple in concept.lineage.where.existence_arguments:
-        base_existence += list(ctuple)
+        base_existence.append(ctuple)
     if direct_parent.grain:
         base_rows += direct_parent.grain.components_copy
     if (
@@ -71,7 +71,7 @@ def resolve_filter_parent_concepts(
         return (
             concept.lineage.content,
             unique(base_rows, "address"),
-            unique(base_existence, "address"),
+            base_existence,
         )
     return concept.lineage.content, unique(base_rows, "address"), []
 

trilogy/core/processing/node_generators/filter_node.py

@@ -74,27 +74,31 @@ def gen_filter_node(
         return parent
 
     core_parents.append(parent)
+
     if parent_existence_concepts:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} fetching filter node existence parents {[x.address for x in parent_existence_concepts]}"
-        )
-        parent_existence = source_concepts(
-            mandatory_list=parent_existence_concepts,
-            environment=environment,
-            g=g,
-            depth=depth + 1,
-            history=history,
-        )
-        if not parent_existence:
+        for existence_tuple in parent_existence_concepts:
+            if not existence_tuple:
+                continue
             logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} filter existence node parents could not be found"
+                f"{padding(depth)}{LOGGER_PREFIX} fetching filter node existence parents {[x.address for x in existence_tuple]}"
             )
-            return None
-        core_parents.append(parent_existence)
-
+            parent_existence = source_concepts(
+                mandatory_list=list(existence_tuple),
+                environment=environment,
+                g=g,
+                depth=depth + 1,
+                history=history,
+            )
+            if not parent_existence:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} filter existence node parents could not be found"
+                )
+                return None
+            core_parents.append(parent_existence)
+    flattened_existence = [x for y in parent_existence_concepts for x in y]
     filter_node = FilterNode(
         input_concepts=unique(
-            [immediate_parent] + parent_row_concepts + parent_existence_concepts,
+            [immediate_parent] + parent_row_concepts + flattened_existence,
             "address",
         ),
         output_concepts=[concept, immediate_parent] + parent_row_concepts,

trilogy/core/processing/nodes/group_node.py

@@ -19,7 +19,7 @@ from trilogy.core.processing.nodes.base_node import (
     concept_list_to_grain,
 )
 from trilogy.utility import unique
-
+from trilogy.core.processing.utility import is_scalar_condition
 
 LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
 
@@ -111,7 +111,8 @@ class GroupNode(StrategyNode):
                     f" {parent.grain}"
                 )
             source_type = SourceType.GROUP
-        return QueryDatasource(
+
+        base = QueryDatasource(
             input_concepts=self.input_concepts,
             output_concepts=self.output_concepts,
             datasources=parent_sources,
@@ -134,6 +135,31 @@ class GroupNode(StrategyNode):
             partial_concepts=self.partial_concepts,
             condition=self.conditions,
         )
+        # if there is a condition on a group node and it's not scalar
+        # inject an additional CTE
+        if self.conditions:
+            logger.info("CONDITIONS")
+            logger.info(str(self.conditions))
+            logger.info(is_scalar_condition(self.conditions))
+        if self.conditions and not is_scalar_condition(self.conditions):
+            base.condition = None
+            base.output_concepts = self.output_concepts + self.conditions.row_arguments
+            return QueryDatasource(
+                input_concepts=base.output_concepts,
+                output_concepts=self.output_concepts,
+                datasources=[base],
+                source_type=SourceType.SELECT,
+                source_map=resolve_concept_map(
+                    [base],
+                    targets=self.output_concepts,
+                    inherited_inputs=base.output_concepts,
+                ),
+                joins=[],
+                grain=grain,
+                partial_concepts=self.partial_concepts,
+                condition=self.conditions,
+            )
+        return base
 
     def copy(self) -> "GroupNode":
         return GroupNode(

trilogy/core/processing/utility.py

@@ -8,6 +8,16 @@ from trilogy.core.models import (
     QueryDatasource,
     LooseConceptList,
     Environment,
+    Conditional,
+    SubselectComparison,
+    Comparison,
+    Parenthetical,
+    Function,
+    FilterItem,
+    MagicConstants,
+    WindowItem,
+    AggregateWrapper,
+    DataType,
 )
 
 from trilogy.core.enums import Purpose, Granularity
@@ -18,6 +28,8 @@ from collections import defaultdict
 from logging import Logger
 from pydantic import BaseModel
 
+from trilogy.core.enums import FunctionClass
+
 
 class NodeType(Enum):
     CONCEPT = 1
@@ -332,3 +344,37 @@ def get_disconnected_components(
         x for x in sub_graphs if calculate_graph_relevance(graph, x, all_concepts) > 0
     ]
     return len(sub_graphs), sub_graphs
+
+
+def is_scalar_condition(
+    element: (
+        int
+        | str
+        | float
+        | list
+        | WindowItem
+        | FilterItem
+        | Concept
+        | Comparison
+        | Conditional
+        | Parenthetical
+        | Function
+        | AggregateWrapper
+        | MagicConstants
+        | DataType
+    ),
+) -> bool:
+    if isinstance(element, Parenthetical):
+        return is_scalar_condition(element.content)
+    elif isinstance(element, SubselectComparison):
+        return True
+    elif isinstance(element, Comparison):
+        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+    elif isinstance(element, Function):
+        if element.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
+            return False
+    elif isinstance(element, AggregateWrapper):
+        return is_scalar_condition(element.function)
+    elif isinstance(element, Conditional):
+        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+    return True

trilogy/core/query_processor.py

@@ -350,7 +350,11 @@ def get_query_datasources(
 
     search_concepts: list[Concept] = statement.output_components
     nest_where = statement.where_clause_category == SelectFiltering.IMPLICIT
-    if nest_where and statement.where_clause:
+
+    # if all are aggregates, we've pushed the filtering inside the aggregates anyway
+    all_aggregate = all([x.is_aggregate for x in search_concepts])
+
+    if nest_where and statement.where_clause and not all_aggregate:
         search_concepts = unique(
             statement.where_clause.row_arguments + search_concepts, "address"
         )
@@ -363,7 +367,8 @@ def get_query_datasources(
     )
     ds: GroupNode | SelectNode
     if nest_where and statement.where_clause:
-        ods.conditions = statement.where_clause.conditional
+        if not all_aggregate:
+            ods.conditions = statement.where_clause.conditional
         ods.output_concepts = search_concepts
         # ods.hidden_concepts = where_delta
         ods.rebuild_cache()

trilogy/dialect/base.py

@@ -2,6 +2,7 @@ from typing import List, Union, Optional, Dict, Any, Sequence, Callable
 
 from jinja2 import Template
 
+from trilogy.core.processing.utility import is_scalar_condition
 from trilogy.constants import CONFIG, logger, MagicConstants
 from trilogy.core.internal import DEFAULT_CONCEPTS
 from trilogy.core.enums import (
@@ -46,6 +47,7 @@ from trilogy.core.models import (
     ProcessedRawSQLStatement,
     NumericType,
     MapType,
+    StructType,
     MergeStatementV2,
 )
 from trilogy.core.query_processor import process_query, process_persist
@@ -106,6 +108,10 @@ def render_case(args):
     return "CASE\n\t" + "\n\t".join(args) + "\n\tEND"
 
 
+def struct_arg(args):
+    return [f"{x[0]}: {x[1]}" for x in zip(args[::2], args[1::2])]
+
+
 FUNCTION_MAP = {
     # generic types
     FunctionType.ALIAS: lambda x: f"{x[0]}",
@@ -120,6 +126,8 @@ FUNCTION_MAP = {
     FunctionType.INDEX_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
     FunctionType.MAP_ACCESS: lambda x: f"{x[0]}[{x[1]}][1]",
     FunctionType.UNNEST: lambda x: f"unnest({x[0]})",
+    FunctionType.ATTR_ACCESS: lambda x: f"""{x[0]}.{x[1].replace("'", "")}""",
+    FunctionType.STRUCT: lambda x: f"{{{', '.join(struct_arg(x))}}}",
     # math
     FunctionType.ADD: lambda x: f"{x[0]} + {x[1]}",
     FunctionType.SUBTRACT: lambda x: f"{x[0]} - {x[1]}",
@@ -164,7 +172,6 @@ FUNCTION_MAP = {
     # constant types
     FunctionType.CURRENT_DATE: lambda x: "current_date()",
     FunctionType.CURRENT_DATETIME: lambda x: "current_datetime()",
-    FunctionType.ATTR_ACCESS: lambda x: f"""{x[0]}.{x[1].replace("'", "")}""",
 }
 
 FUNCTION_GRAIN_MATCH_MAP = {
@@ -196,7 +203,9 @@ TOP {{ limit }}{% endif %}
 WHERE
 \t{{ where }}{% endif %}{%- if group_by %}
 GROUP BY {% for group in group_by %}
-\t{{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{%- if order_by %}
+\t{{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{% if having %}
+HAVING
+\t{{ having }}{% endif %}{%- if order_by %}
 ORDER BY{% for order in order_by %}
 \t{{ order }}{% if not loop.last %},{% endif %}{% endfor %}
 {% endif %}{% endif %}
@@ -347,6 +356,7 @@ class BaseDialect:
             MapWrapper[Any, Any],
             MapType,
             NumericType,
+            StructType,
             ListType,
             ListWrapper[Any],
             DatePart,
@@ -370,11 +380,18 @@ class BaseDialect:
                     lookup_cte = cte_map.get(e.right.address)
                 assert lookup_cte, "Subselects must be rendered with a CTE in context"
                 if e.right.address not in lookup_cte.existence_source_map:
-                    lookup = lookup_cte.source_map[e.right.address]
+                    lookup = lookup_cte.source_map.get(
+                        e.right.address,
+                        [
+                            INVALID_REFERENCE_STRING(
+                                f"Missing source reference to {e.right.name}"
+                            )
+                        ],
+                    )
                 else:
                     lookup = lookup_cte.existence_source_map[e.right.address]
 
-                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} (select {lookup[0]}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {lookup[0]})"
+                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} (select {lookup[0]}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {lookup[0]} where {lookup[0]}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
             elif isinstance(e.right, (ListWrapper, Parenthetical, list)):
                 return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map)}"
 
@@ -418,6 +435,7 @@ class BaseDialect:
         elif isinstance(e, CaseElse):
             return f"ELSE {self.render_expr(e.expr, cte=cte, cte_map=cte_map) }"
         elif isinstance(e, Function):
+
             if cte and cte.group_to_grain:
                 return self.FUNCTION_MAP[e.operator](
                     [self.render_expr(z, cte=cte, cte_map=cte_map) for z in e.arguments]
@@ -507,8 +525,15 @@ class BaseDialect:
                     if j
                 ],
                 where=(
-                    self.render_expr(cte.condition, cte) if cte.condition else None
-                ),  # source_map=cte_output_map)
+                    self.render_expr(cte.condition, cte)
+                    if cte.condition and is_scalar_condition(cte.condition)
+                    else None
+                ),
+                having=(
+                    self.render_expr(cte.condition, cte)
+                    if cte.condition and not is_scalar_condition(cte.condition)
+                    else None
+                ),
                 order_by=(
                     [self.render_order_item(i, cte) for i in cte.order_by.items]
                     if cte.order_by

trilogy/dialect/bigquery.py

@@ -59,7 +59,9 @@ SELECT
     {{ where }}
 {% endif %}
 {%- if group_by %}GROUP BY {% for group in group_by %}
-    {{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
+    {{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{% if having %}
+HAVING
+\t{{ having }}{% endif %}
 {%- if order_by %}
 ORDER BY {% for order in order_by %}
     {{ order }}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}

trilogy/dialect/duckdb.py

@@ -59,11 +59,12 @@ SELECT
     {{ join }}{% endfor %}{% endif %}
 {% if where %}WHERE
     {{ where }}
-{% endif -%}
-{%- if group_by %}
+{% endif -%}{%- if group_by %}
 GROUP BY {% for group in group_by %}
-    {{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
-{%- if order_by %}
+    {{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{% if having %}
+HAVING
+    {{ having }}
+{% endif %}{%- if order_by %}
 ORDER BY {% for order in order_by %}
     {{ order }}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
 {%- if limit is not none %}

trilogy/dialect/postgres.py

@@ -64,7 +64,9 @@ SELECT
     {{ where }}
 {% endif %}
 {%- if group_by %}GROUP BY {% for group in group_by %}
-    {{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
+    {{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{% if having %}
+HAVING
+\t{{ having }}{% endif %}
 {%- if order_by %}
 ORDER BY {% for order in order_by %}
     {{ order }}{% if not loop.last %},{% endif %}

trilogy/dialect/presto.py

@@ -63,7 +63,9 @@ SELECT
     {{ where }}
 {% endif %}
 {%- if group_by %}GROUP BY {% for group in group_by %}
-    {{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
+    {{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{% if having %}
+HAVING
+\t{{ having }}{% endif %}
 {%- if order_by %}
 ORDER BY {% for order in order_by %}
     {{ order }}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}

trilogy/dialect/snowflake.py

@@ -61,7 +61,9 @@ SELECT
     {{ where }}
 {% endif %}
 {%- if group_by %}GROUP BY {% for group in group_by %}
-    {{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
+    {{group}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{% if having %}
+HAVING
+\t{{ having }}{% endif %}
 {%- if order_by %}
 ORDER BY {% for order in order_by %}
     {{ order }}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}

trilogy/dialect/sql_server.py

@@ -60,7 +60,9 @@ TOP {{ limit }}{% endif %}
 {%- if group_by %}
 GROUP BY {% for group in group_by %}
     {{group}}{% if not loop.last %},{% endif %}
-{% endfor %}{% endif %}
+{% endfor %}{% endif %}{% if having %}
+HAVING
+\t{{ having }}{% endif %}
 {%- if order_by %}
 ORDER BY {% for order in order_by %}
     {{ order }}{% if not loop.last %},{% endif %}

trilogy/parsing/common.py

@@ -198,6 +198,7 @@ def arbitrary_to_concept(
     metadata: Metadata | None = None,
     purpose: Purpose | None = None,
 ) -> Concept:
+
     if isinstance(parent, AggregateWrapper):
         return agg_wrapper_to_concept(parent, namespace, name, metadata, purpose)
     elif isinstance(parent, WindowItem):

trilogy/parsing/parse_engine.py

@@ -500,7 +500,53 @@ class ParseToObjects(Transformer):
         # we need to strip off every parenthetical to see what is being assigned.
         while isinstance(source_value, Parenthetical):
             source_value = source_value.content
-        if isinstance(
+
+        if (
+            isinstance(source_value, Function)
+            and source_value.operator == FunctionType.STRUCT
+        ):
+            concept = arbitrary_to_concept(
+                source_value,
+                name=name,
+                namespace=namespace,
+                purpose=purpose,
+                metadata=metadata,
+            )
+
+            if concept.metadata:
+                concept.metadata.line_number = meta.line
+            self.environment.add_concept(concept, meta=meta)
+            assert isinstance(concept.datatype, StructType)
+            for key, value in concept.datatype.fields_map.items():
+                args = self.process_function_args([concept, key], meta=meta)
+                self.environment.add_concept(
+                    Concept(
+                        name=key,
+                        datatype=arg_to_datatype(value),
+                        purpose=Purpose.PROPERTY,
+                        namespace=self.environment.namespace + "." + name,
+                        lineage=AttrAccess(args),
+                    )
+                )
+            return ConceptDerivation(concept=concept)
+        elif (
+            isinstance(source_value, Function)
+            and source_value.operator == FunctionType.ALIAS
+        ):
+            concept = arbitrary_to_concept(
+                source_value,
+                name=name,
+                namespace=namespace,
+                purpose=purpose,
+                metadata=metadata,
+            )
+
+            if concept.metadata:
+                concept.metadata.line_number = meta.line
+            self.environment.add_concept(concept, meta=meta)
+            return ConceptDerivation(concept=concept)
+
+        elif isinstance(
             source_value, (FilterItem, WindowItem, AggregateWrapper, Function)
         ):
             concept = arbitrary_to_concept(
@@ -515,6 +561,7 @@ class ParseToObjects(Transformer):
                 concept.metadata.line_number = meta.line
             self.environment.add_concept(concept, meta=meta)
             return ConceptDerivation(concept=concept)
+
         elif isinstance(source_value, CONSTANT_TYPES):
             concept = constant_to_concept(
                 source_value,
@@ -950,7 +997,15 @@ class ParseToObjects(Transformer):
                     isinstance(orderitem.expr, Concept)
                     and orderitem.expr.purpose == Purpose.METRIC
                 ):
-                    orderitem.expr = orderitem.expr.with_grain(output.grain)
+                    orderitem.expr = orderitem.expr.with_select_context(
+                        output.grain,
+                        conditional=(
+                            output.where_clause.conditional
+                            if output.where_clause
+                            and output.where_clause_category == SelectFiltering.IMPLICIT
+                            else None
+                        ),
+                    )
         return output
 
     @v_args(meta=True)
@@ -1016,6 +1071,18 @@ class ParseToObjects(Transformer):
     def array_lit(self, args):
         return list_to_wrapper(args)
 
+    def struct_lit(self, args):
+
+        zipped = dict(zip(args[::2], args[1::2]))
+        types = [arg_to_datatype(x) for x in args[1::2]]
+        return Function(
+            operator=FunctionType.STRUCT,
+            output_datatype=StructType(fields=types, fields_map=zipped),
+            output_purpose=function_args_to_output_purpose(args),
+            arguments=args,
+            arg_count=-1,
+        )
+
     def map_lit(self, args):
         parsed = dict(zip(args[::2], args[1::2]))
         wrapped = dict_to_map_wrapper(parsed)
@@ -1162,6 +1229,11 @@ class ParseToObjects(Transformer):
             return MapAccess(args)
         return IndexAccess(args)
 
+    @v_args(meta=True)
+    def map_key_access(self, meta, args):
+        args = self.process_function_args(args, meta=meta)
+        return MapAccess(args)
+
     @v_args(meta=True)
     def attr_access(self, meta, args):
         args = self.process_function_args(args, meta=meta)

trilogy/parsing/trilogy.lark

@@ -153,9 +153,10 @@
     unnest: _UNNEST expr ")"
     //indexing into an expression is a function
     index_access: expr "[" int_lit "]"
-    attr_access: expr  "[" _string_lit "]"
+    map_key_access: expr "[" _string_lit "]"
+    attr_access: expr  "." _string_lit 
 
-    expr:  _constant_functions | window_item | filter_item | subselect_comparison | between_comparison |  fgroup |  aggregate_functions | unnest | _static_functions |  literal |  concept_lit  | index_access | attr_access | parenthetical | expr_tuple | comparison |  alt_like 
+    expr:  _constant_functions | window_item | filter_item | subselect_comparison | between_comparison |  fgroup |  aggregate_functions | unnest | _static_functions |  literal |  concept_lit  | index_access |  map_key_access | attr_access | parenthetical | expr_tuple | comparison |  alt_like 
     
     // functions
     
@@ -276,12 +277,15 @@
     array_lit: "[" (literal ",")* literal ","? "]"()
 
     map_lit: "{" (literal ":" literal ",")* literal ":" literal ","? "}"
+
+   
+    struct_lit: "struct" "(" (IDENTIFIER "=" literal ",")* IDENTIFIER "=" literal ","? ")"
     
     !bool_lit: "True"i | "False"i
 
     !null_lit.1: "null"i
     
-    literal: null_lit | _string_lit | int_lit | float_lit | bool_lit | array_lit | map_lit
+    literal: null_lit | _string_lit | int_lit | float_lit | bool_lit | array_lit | map_lit | struct_lit
 
     MODIFIER: "Optional"i | "Partial"i | "Nullable"i