pytrilogy 0.0.1.110__py3-none-any.whl → 0.0.1.112__py3-none-any.whl

{pytrilogy-0.0.1.110.dist-info → pytrilogy-0.0.1.112.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.110
+Version: 0.0.1.112
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.110.dist-info → pytrilogy-0.0.1.112.dist-info}/RECORD

@@ -1,8 +1,8 @@
-trilogy/__init__.py,sha256=zz6RmV7fp8n1Ezl-E5jUW92Ns-EMfac6jy3fX5zQuzc,292
+trilogy/__init__.py,sha256=DnY_8Giv8_IhdMGbFdiIMOk0nDhb3uLiFVA0tLyveas,292
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=C6mwa0BpVyENkcWi-dwqHorCP85ScTXDemfHFiSmLQ8,737
+trilogy/constants.py,sha256=DJi3ESttmvqgy6fPRXiaQzqJVye6jYwf6XM89NHv0_M,735
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
-trilogy/executor.py,sha256=xF6wzbhP6a3wz4nrxsRCKeKF7qytUQEL75oI3BGJ2hQ,8744
+trilogy/executor.py,sha256=_ZbjrKsUdWL52tWgpxqZnmccAuPXcIPEPN_dDSLNeAQ,9696
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
@@ -16,36 +16,36 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=zkRReytiotOBAW-a3Ri5eoejZDYTt2-7Op80ZxZxUmw,9129
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=FNAMbqJrHh-KfC8QbBe3anWVdNxP-uMjs--MrbJM8QM,109943
-trilogy/core/optimization.py,sha256=5n5HMGGdTGkDZf479lZPpv2angLSxklZJ0D4DBrFDeA,8411
-trilogy/core/query_processor.py,sha256=6BqLYPwyFkRtueTIRFZi3IcVFTpbpGRNowayhSn3_AY,11805
+trilogy/core/models.py,sha256=r8Wn-cvcSgkZ399TGqAYlJURqdiWi5zxzHMY2CUnbP4,110175
+trilogy/core/optimization.py,sha256=942MnGRzscAHcG9LsfMslIRRQBslbIiPHnAvJ3w8YRg,9157
+trilogy/core/query_processor.py,sha256=clIRJ6IcsqIVBPKFsxt8bqCLsLyajvAu02MUIcKQhTo,15713
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=27lZXFLgDEF3sh2MUR7HX_atVz7TC1fJB7z3oxa1TcY,22610
+trilogy/core/processing/concept_strategies_v3.py,sha256=MYrpNMidqvPOg123RekOcqVTjcj03i_538gBo0MzoWE,23432
 trilogy/core/processing/graph_utils.py,sha256=ulCJ4hYAISbUxLD6VM2fah9RBPGIXSEHEPeRBSFl0Rs,1197
-trilogy/core/processing/utility.py,sha256=Gk35HgyIG2SSUyI5OHZcB0bw1PZUVC_aNc9Sre6xPQU,10535
+trilogy/core/processing/utility.py,sha256=acxH5448-j8JXqxMRibyAxjz1Wqu7QudbR0PfMuucww,9902
 trilogy/core/processing/node_generators/__init__.py,sha256=LIs6uBEum8LDc-26zjyAwjxa-ay2ok9tKtPjDNvbVkE,757
-trilogy/core/processing/node_generators/basic_node.py,sha256=tVPmg0r0kDdABkmn6z4sxsk1hKy9yTT_Xvl1eVN2Zck,2162
-trilogy/core/processing/node_generators/common.py,sha256=A0zB4xr1etbEexaiSH6mVTecXY_wd7pSwWUGUt-u0eg,8882
-trilogy/core/processing/node_generators/concept_merge_node.py,sha256=TRbOIjLWfLB0Nl6YmMV1ao0qhPP6OQDd9M3UViWkCBU,6621
-trilogy/core/processing/node_generators/filter_node.py,sha256=CGALiTzKhPAvXPFAguIQfjf6I3pjlafY0uaaM9MTkIE,3414
+trilogy/core/processing/node_generators/basic_node.py,sha256=HJnIhZLgkUdorKYcofe-QnKSM3Lf_3QO91cbSJhsqf4,2242
+trilogy/core/processing/node_generators/common.py,sha256=liZDth7mvhkF_sUFXK7JitJsiaKD132w3ySLbF7l-nE,8956
+trilogy/core/processing/node_generators/concept_merge_node.py,sha256=x4M8VVZZmBcqHDY1uq7M9KGKCBwjU6mcE_x2BOEk2Mg,7328
+trilogy/core/processing/node_generators/filter_node.py,sha256=y_tqYe2So18vWHASMwVPLzDO-PnyQCO-MAlI4B-rY3Y,4526
 trilogy/core/processing/node_generators/group_node.py,sha256=xWI1xNIXEOj6jlRGD9hcv2_vVNvY6lpzJl6pQ8HuFBE,2988
 trilogy/core/processing/node_generators/group_to_node.py,sha256=BzPdYwzoo8gRMH7BDffTTXq4z-mjfCEzvfB5I-P0_nw,2941
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=sQQ9jhw1oAJh649DBAJX6U7r_E_piFS95mxKvm7pxqQ,5818
-trilogy/core/processing/node_generators/rowset_node.py,sha256=BYTpXyiFJwoFp_n8kKE-HUdnwgaK_58n0rSFOz2jOVM,5141
-trilogy/core/processing/node_generators/select_node.py,sha256=xeCqIUEubrf3u_QQfbGdf1BG4fO0HYQ64hiFur8NUqY,20080
-trilogy/core/processing/node_generators/unnest_node.py,sha256=s1VXQZSf1LnX3ISeQ5JzmzmCKUw30-5OK_f0YTB9_48,1031
-trilogy/core/processing/node_generators/window_node.py,sha256=ekazi5eXxnShpcp-qukXNG4DHFdULoXrX-YWUWLNEpM,2527
-trilogy/core/processing/nodes/__init__.py,sha256=gzKxGSduIQ5QwpMWrmwSYiE8sg2mWejwVn0VvjYc6s0,3879
-trilogy/core/processing/nodes/base_node.py,sha256=Du7hRjVVOAiGb0okytzKIa_TQqhwTNYGU8PGNnrE1xs,9142
-trilogy/core/processing/nodes/filter_node.py,sha256=DqSRv8voEajPZqzeeiIsxuv4ubvsmeQcCW6x_v2CmOk,1359
-trilogy/core/processing/nodes/group_node.py,sha256=Y_NWB_AwFrE-YithjZ7lYYDN4e0el4su3ICq2EIr3HA,3837
-trilogy/core/processing/nodes/merge_node.py,sha256=uo1AfLjA02EiIpJku8T0TBjN2D-IyP6NVq5m7BfiZbQ,12413
-trilogy/core/processing/nodes/select_node_v2.py,sha256=tAADeVruch-flFiedbY1zi7ukMG2RpWecvxxZ5aL3ZU,6354
-trilogy/core/processing/nodes/unnest_node.py,sha256=t4kY3a_dR3iXistPemStfdw0uJfnxwTcoQg1HiDa3xo,1501
-trilogy/core/processing/nodes/window_node.py,sha256=QjAWgqBZqFSRCPwc7JBmgQJobWW50rsHI0pjJe0Zzg0,926
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=wNDHAbRrKSjsns-EROM_G12mRyOMjbcWpYav2uefXOE,6045
+trilogy/core/processing/node_generators/rowset_node.py,sha256=eNG6rfLifUKraoRGxE8pesQMy5cKT6R5XNIaa3Wuiwk,6081
+trilogy/core/processing/node_generators/select_node.py,sha256=Qb00Kizsv-877UMkGfusl5jXKXMZtZTtLks5pxU07SU,20698
+trilogy/core/processing/node_generators/unnest_node.py,sha256=6CH66eGwpadNX7TzUhWZ8aqIisOtQeHINbLV6X3QBUk,1779
+trilogy/core/processing/node_generators/window_node.py,sha256=9nXUXUgQrNczU1gaOqhOZPNzCUxw-lkxt0R7HORI6ss,2582
+trilogy/core/processing/nodes/__init__.py,sha256=baODkJfvUoWEEbu843GEd7snubwLeOG5FQ8l-CwIaC8,3928
+trilogy/core/processing/nodes/base_node.py,sha256=yhjmsAUmhHDqgbQjz_9YdfP-M5pj4xbrPRDF6Y4XVuw,10498
+trilogy/core/processing/nodes/filter_node.py,sha256=rDw4vfE6tqWxuKT0arihVmIOoOWDDCyzRA-2yONX_Ek,1860
+trilogy/core/processing/nodes/group_node.py,sha256=vzeU9J4xMhRrPj4-KPJTgNbH-KFu2ZS8b57SOynsdw0,4448
+trilogy/core/processing/nodes/merge_node.py,sha256=FvSiTWKOzaUsXBkf6wJD8QQqQxp_aphS_I5VzNRw8Yo,13600
+trilogy/core/processing/nodes/select_node_v2.py,sha256=ERCflBFzKpD5SzweMevnJLyQnxmF_-IQ6VRu5yVeiBg,6552
+trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
+trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=xPB5mh6471VJLHxNdXrYvi7q7vJC_tioVR1LrLcoZc0,29394
+trilogy/dialect/base.py,sha256=ii9P_OO8BhKsQVAr9A13rhx_dzRZd4wxnkL-Ul5OS74,30398
 trilogy/dialect/bigquery.py,sha256=9vxQn2BMv_oTGQSWQpoN5ho_OgqMWaHH9e-5vQVf44c,2906
 trilogy/dialect/common.py,sha256=zWrYmvevlXznocw9uGHmY5Ws1rp_kICm9zA_ulTe4eg,2165
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
@@ -65,13 +65,13 @@ trilogy/parsing/common.py,sha256=lz0IyVA8v-u-DGFgzkmdb4_00I--Kegmo9HNF7CrajI,579
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=iOqKUCyLeHyFVwwAt-XTSJGHia4zzLUN6bYDuIfJ1Pg,63938
+trilogy/parsing/parse_engine.py,sha256=LdxdYQpv-9pBNXXs1QdnD08fflnQ82tG6H6mVgpVeIY,65532
 trilogy/parsing/render.py,sha256=fxjpq2FZLgllw_d4cru-t_IXNPAz2DmYkT7v9ED0XRI,11540
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.1.110.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.1.110.dist-info/METADATA,sha256=FbH2jc_Eg9QjfxlbLhTAqBzFoCxe4aM-6EbE4HDC4TM,7882
-pytrilogy-0.0.1.110.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-pytrilogy-0.0.1.110.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.1.110.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.1.110.dist-info/RECORD,,
+pytrilogy-0.0.1.112.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.1.112.dist-info/METADATA,sha256=jhGh-RxeT78DDSvj9YY4K_ww6-C00dhYHyD5NMs1EFY,7882
+pytrilogy-0.0.1.112.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
+pytrilogy-0.0.1.112.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.1.112.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.1.112.dist-info/RECORD,,

{pytrilogy-0.0.1.110.dist-info → pytrilogy-0.0.1.112.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.3.0)
+Generator: setuptools (71.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.1.110"
+__version__ = "0.0.1.112"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -30,7 +30,7 @@ class Optimizations:
 class Config:
     strict_mode: bool = True
     human_identifiers: bool = True
-    inline_datasources: bool = True
+    validate_missing: bool = True
     optimizations: Optimizations = field(default_factory=Optimizations)
 
 

trilogy/core/models.py

@@ -33,7 +33,13 @@ from pydantic import (
 )
 from lark.tree import Meta
 from pathlib import Path
-from trilogy.constants import logger, DEFAULT_NAMESPACE, ENV_CACHE_NAME, MagicConstants
+from trilogy.constants import (
+    logger,
+    DEFAULT_NAMESPACE,
+    ENV_CACHE_NAME,
+    MagicConstants,
+    CONFIG,
+)
 from trilogy.core.constants import (
     ALL_ROWS_CONCEPT,
     INTERNAL_NAMESPACE,
@@ -61,7 +67,6 @@ from trilogy.core.enums import (
 from trilogy.core.exceptions import UndefinedConceptException, InvalidSyntaxException
 from trilogy.utility import unique
 from collections import UserList
-from trilogy.utility import string_to_hash
 from functools import cached_property
 from abc import ABC
 
@@ -129,7 +134,7 @@ class ConceptArgs(ABC):
         raise NotImplementedError
 
     @property
-    def existence_arguments(self) -> List["Concept"]:
+    def existence_arguments(self) -> list[tuple["Concept", ...]]:
         return []
 
     @property
@@ -281,9 +286,6 @@ class Concept(Namespaced, SelectGrain, BaseModel):
             MultiSelectStatement | MergeStatement,
         ]
     ] = None
-    # lineage: Annotated[Optional[
-    #     Union[Function, WindowItem, FilterItem, AggregateWrapper]
-    # ], WrapValidator(lineage_validator)] = None
     namespace: Optional[str] = Field(default=DEFAULT_NAMESPACE, validate_default=True)
     keys: Optional[Tuple["Concept", ...]] = None
     grain: "Grain" = Field(default=None, validate_default=True)
@@ -621,6 +623,12 @@ class Grain(BaseModel):
             if sub.purpose in (Purpose.PROPERTY, Purpose.METRIC) and sub.keys:
                 if all([c in v2 for c in sub.keys]):
                     continue
+            elif sub.derivation == PurposeLineage.MERGE and isinstance(
+                sub.lineage, MergeStatement
+            ):
+                parents = sub.lineage.concepts
+                if any([p in v2 for p in parents]):
+                    continue
             final.append(sub)
         v2 = sorted(final, key=lambda x: x.name)
         return v2
@@ -966,23 +974,6 @@ class ConceptTransform(Namespaced, BaseModel):
             modifiers=self.modifiers,
         )
 
-    def with_filter(self, where: "WhereClause") -> "ConceptTransform":
-        id_hash = string_to_hash(str(where))
-        new_parent_concept = Concept(
-            name=f"_anon_concept_transform_filter_input_{id_hash}",
-            datatype=self.output.datatype,
-            purpose=self.output.purpose,
-            lineage=self.output.lineage,
-            namespace=DEFAULT_NAMESPACE,
-            grain=self.output.grain,
-            keys=self.output.keys,
-        )
-        new_parent = FilterItem(content=new_parent_concept, where=where)
-        self.output.lineage = new_parent
-        return ConceptTransform(
-            function=new_parent, output=self.output, modifiers=self.modifiers
-        )
-
 
 class Window(BaseModel):
     count: int
@@ -1611,13 +1602,15 @@ class Datasource(Namespaced, BaseModel):
     def __add__(self, other):
         if not other == self:
             raise ValueError(
-                "Attempted to add two datasources that are not identical, this should"
-                " never happen"
+                "Attempted to add two datasources that are not identical, this is not a valid operation"
             )
         return self
 
+    def __repr__(self):
+        return f"Datasource<{self.namespace}.{self.identifier}@<{self.grain}>"
+
     def __str__(self):
-        return f"{self.namespace}.{self.identifier}@<{self.grain}>"
+        return self.__repr__()
 
     def __hash__(self):
         return (self.namespace + self.identifier).__hash__()
@@ -1786,6 +1779,7 @@ class QueryDatasource(BaseModel):
     input_concepts: List[Concept]
     output_concepts: List[Concept]
     source_map: Dict[str, Set[Union[Datasource, "QueryDatasource", "UnnestJoin"]]]
+
     datasources: List[Union[Datasource, "QueryDatasource"]]
     grain: Grain
     joins: List[BaseJoin | UnnestJoin]
@@ -1799,6 +1793,12 @@ class QueryDatasource(BaseModel):
     join_derived_concepts: List[Concept] = Field(default_factory=list)
     hidden_concepts: List[Concept] = Field(default_factory=list)
     force_group: bool | None = None
+    existence_source_map: Dict[str, Set[Union[Datasource, "QueryDatasource"]]] = Field(
+        default_factory=dict
+    )
+
+    def __repr__(self):
+        return f"{self.identifier}@<{self.grain}>"
 
     @property
     def non_partial_concept_addresses(self) -> List[str]:
@@ -1841,14 +1841,14 @@ class QueryDatasource(BaseModel):
         for k, _ in v.items():
             seen.add(k)
         for x in expected:
-            if x not in seen:
+            if x not in seen and CONFIG.validate_missing:
                 raise SyntaxError(
                     f"source map missing {x} on (expected {expected}, have {seen})"
                 )
         return v
 
     def __str__(self):
-        return f"{self.identifier}@<{self.grain}>"
+        return self.__repr__()
 
     def __hash__(self):
         return (self.identifier).__hash__()
@@ -2010,10 +2010,11 @@ class CTE(BaseModel):
     name: str
     source: "QueryDatasource"
     output_columns: List[Concept]
-    source_map: Dict[str, str | list[str]]
+    source_map: Dict[str, list[str]]
     grain: Grain
     base: bool = False
     group_to_grain: bool = False
+    existence_source_map: Dict[str, list[str]] = Field(default_factory=dict)
     parent_ctes: List["CTE"] = Field(default_factory=list)
     joins: List[Union["Join", "InstantiatedUnnestJoin"]] = Field(default_factory=list)
     condition: Optional[Union["Conditional", "Comparison", "Parenthetical"]] = None
@@ -2024,6 +2025,7 @@ class CTE(BaseModel):
     limit: Optional[int] = None
     requires_nesting: bool = True
     base_name_override: Optional[str] = None
+    base_alias_override: Optional[str] = None
 
     @computed_field  # type: ignore
     @property
@@ -2034,11 +2036,19 @@ class CTE(BaseModel):
     def validate_output_columns(cls, v):
         return unique(v, "address")
 
-    def inline_parent_datasource(self, parent: CTE) -> bool:
+    def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
         qds_being_inlined = parent.source
         ds_being_inlined = qds_being_inlined.datasources[0]
         if not isinstance(ds_being_inlined, Datasource):
             return False
+        if any(
+            [
+                x.identifier == ds_being_inlined.identifier
+                for x in self.source.datasources
+            ]
+        ):
+            return False
+
         self.source.datasources = [
             ds_being_inlined,
             *[
@@ -2050,6 +2060,7 @@ class CTE(BaseModel):
         # need to identify this before updating joins
         if self.base_name == parent.name:
             self.base_name_override = ds_being_inlined.safe_location
+            self.base_alias_override = ds_being_inlined.identifier
 
         for join in self.joins:
             if isinstance(join, InstantiatedUnnestJoin):
@@ -2066,6 +2077,8 @@ class CTE(BaseModel):
             elif v == parent.name:
                 self.source_map[k] = ds_being_inlined.name
         self.parent_ctes = [x for x in self.parent_ctes if x.name != parent.name]
+        if force_group:
+            self.group_to_grain = True
         return True
 
     def __add__(self, other: "CTE"):
@@ -2126,9 +2139,6 @@ class CTE(BaseModel):
         if self.base_name_override:
             return self.base_name_override
         # if this cte selects from a single datasource, select right from it
-        valid_joins: List[Join] = [
-            join for join in self.joins if isinstance(join, Join)
-        ]
         if self.is_root_datasource:
             return self.source.datasources[0].safe_location
 
@@ -2136,33 +2146,16 @@ class CTE(BaseModel):
         # as the root
         elif len(self.source.datasources) == 1 and len(self.parent_ctes) == 1:
             return self.parent_ctes[0].name
-        elif valid_joins and len(valid_joins) > 0:
-            candidates = [x.left_cte.name for x in valid_joins]
-            disallowed = [x.right_cte.name for x in valid_joins]
-            try:
-                return [y for y in candidates if y not in disallowed][0]
-            except IndexError:
-                raise SyntaxError(
-                    f"Invalid join configuration {candidates} {disallowed} with all parents {[x.base_name for x in self.parent_ctes]}"
-                )
         elif self.relevant_base_ctes:
             return self.relevant_base_ctes[0].name
-        elif self.parent_ctes:
-            raise SyntaxError(
-                f"{self.name} has no relevant base CTEs, {self.source_map},"
-                f" {[x.name for x in self.parent_ctes]}, outputs"
-                f" {[x.address for x in self.output_columns]}"
-            )
         return self.source.name
 
     @property
     def base_alias(self) -> str:
-
+        if self.base_alias_override:
+            return self.base_alias_override
         if self.is_root_datasource:
             return self.source.datasources[0].identifier
-        relevant_joins = [j for j in self.joins if isinstance(j, Join)]
-        if relevant_joins:
-            return relevant_joins[0].left_cte.name
         elif self.relevant_base_ctes:
             return self.relevant_base_ctes[0].name
         elif self.parent_ctes:
@@ -2492,9 +2485,17 @@ class Environment(BaseModel):
         for datasource in self.datasources.values():
             for concept in datasource.output_concepts:
                 concrete_addresses.add(concept.address)
+        current_mat = [x.address for x in self.materialized_concepts]
         self.materialized_concepts = [
             c for c in self.concepts.values() if c.address in concrete_addresses
         ]
+        new = [
+            x.address
+            for x in self.materialized_concepts
+            if x.address not in current_mat
+        ]
+        if new:
+            logger.info(f"Environment added new materialized concepts {new}")
         for concept in self.concepts.values():
             if concept.derivation == PurposeLineage.MERGE:
                 ms = concept.lineage
@@ -2653,6 +2654,17 @@ class Environment(BaseModel):
         self.gen_concept_list_caches()
         return datasource
 
+    def delete_datasource(
+        self,
+        address: str,
+        meta: Meta | None = None,
+    ) -> bool:
+        if address in self.datasources:
+            del self.datasources[address]
+            self.gen_concept_list_caches()
+            return True
+        return False
+
 
 class LazyEnvironment(Environment):
     """Variant of environment to defer parsing of a path"""
@@ -2759,11 +2771,8 @@ class Comparison(ConceptArgs, Namespaced, SelectGrain, BaseModel):
                 if isinstance(self.left, SelectGrain)
                 else self.left
             ),
-            right=(
-                self.right.with_select_grain(grain)
-                if isinstance(self.right, SelectGrain)
-                else self.right
-            ),
+            # the right side does NOT need to inherit select grain
+            right=self.right,
             operator=self.operator,
         )
 
@@ -2809,8 +2818,8 @@ class SubselectComparison(Comparison):
         return get_concept_arguments(self.left)
 
     @property
-    def existence_arguments(self) -> List[Concept]:
-        return get_concept_arguments(self.right)
+    def existence_arguments(self) -> list[tuple["Concept", ...]]:
+        return [tuple(get_concept_arguments(self.right))]
 
     def with_select_grain(self, grain: Grain):
         # there's no need to pass the select grain through to a subselect comparison
@@ -3002,18 +3011,26 @@ class Conditional(ConceptArgs, Namespaced, SelectGrain, BaseModel):
         return output
 
     @property
-    def existence_arguments(self) -> List[Concept]:
+    def existence_arguments(self) -> list[tuple["Concept", ...]]:
         output = []
         if isinstance(self.left, ConceptArgs):
             output += self.left.existence_arguments
-        else:
-            output += get_concept_arguments(self.left)
         if isinstance(self.right, ConceptArgs):
             output += self.right.existence_arguments
-        else:
-            output += get_concept_arguments(self.right)
         return output
 
+    def decompose(self):
+        chunks = []
+        if self.operator == BooleanOperator.AND:
+            for val in [self.left, self.right]:
+                if isinstance(val, Conditional):
+                    chunks.extend(val.decompose())
+                else:
+                    chunks.append(val)
+        else:
+            chunks.append(self)
+        return chunks
+
 
 class AggregateWrapper(Namespaced, SelectGrain, BaseModel):
     function: Function
@@ -3073,7 +3090,7 @@ class WhereClause(ConceptArgs, Namespaced, SelectGrain, BaseModel):
         return self.conditional.row_arguments
 
     @property
-    def existence_arguments(self) -> List[Concept]:
+    def existence_arguments(self) -> list[tuple["Concept", ...]]:
         return self.conditional.existence_arguments
 
     def with_namespace(self, namespace: str) -> WhereClause:
@@ -3314,10 +3331,10 @@ class Parenthetical(ConceptArgs, Namespaced, SelectGrain, BaseModel):
         return self.concept_arguments
 
     @property
-    def existence_arguments(self) -> List[Concept]:
+    def existence_arguments(self) -> list[tuple["Concept", ...]]:
         if isinstance(self.content, ConceptArgs):
             return self.content.existence_arguments
-        return self.concept_arguments
+        return []
 
     @property
     def input(self):
@@ -3386,6 +3403,12 @@ Function.model_rebuild()
 Grain.model_rebuild()
 
 
+def list_to_wrapper(args):
+    types = [arg_to_datatype(arg) for arg in args]
+    assert len(set(types)) == 1
+    return ListWrapper(args, type=types[0])
+
+
 def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType:
     if isinstance(arg, Function):
         return arg.output_datatype
@@ -3409,5 +3432,8 @@ def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType:
         if arg.type in (WindowType.RANK, WindowType.ROW_NUMBER):
             return DataType.INTEGER
         return arg_to_datatype(arg.content)
+    elif isinstance(arg, list):
+        wrapper = list_to_wrapper(arg)
+        return ListType(type=wrapper.type)
     else:
         raise ValueError(f"Cannot parse arg datatype for arg of raw type {type(arg)}")

trilogy/core/optimization.py

@@ -12,9 +12,6 @@ from trilogy.constants import logger, CONFIG
 from abc import ABC
 
 
-REGISTERED_RULES: list["OptimizationRule"] = []
-
-
 class OptimizationRule(ABC):
 
     def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
@@ -38,6 +35,7 @@ class InlineDatasource(OptimizationRule):
             f"Checking {cte.name} for consolidating inline tables with {len(cte.parent_ctes)} parents"
         )
         to_inline: list[CTE] = []
+        force_group = False
         for parent_cte in cte.parent_ctes:
             if not parent_cte.is_root_datasource:
                 self.log(f"parent {parent_cte.name} is not root")
@@ -55,20 +53,25 @@ class InlineDatasource(OptimizationRule):
                 continue
             root_outputs = {x.address for x in root.output_concepts}
             cte_outputs = {x.address for x in parent_cte.output_columns}
+            grain_components = {x.address for x in root.grain.components}
             if not cte_outputs.issubset(root_outputs):
                 self.log(f"Not all {parent_cte.name} outputs are found on datasource")
                 continue
-
+            if not grain_components.issubset(cte_outputs):
+                self.log("Not all datasource components in cte outputs, forcing group")
+                force_group = True
             to_inline.append(parent_cte)
 
         for replaceable in to_inline:
-            self.log(f"Inlining parent {replaceable.name}")
-            cte.inline_parent_datasource(replaceable)
 
+            result = cte.inline_parent_datasource(replaceable, force_group=force_group)
+            if result:
+                self.log(f"Inlined parent {replaceable.name}")
+            else:
+                self.log(f"Failed to inline {replaceable.name}")
         return optimized
 
 
-# This will be used in the future for more complex condition decomposition
 def decompose_condition(conditional: Conditional):
     chunks = []
     if conditional.operator == BooleanOperator.AND:
@@ -107,14 +110,14 @@ class PredicatePushdown(OptimizationRule):
             f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
         )
         if isinstance(cte.condition, Conditional):
-            candidates = decompose_condition(cte.condition)
+            candidates = cte.condition.decompose()
         else:
             candidates = [cte.condition]
         logger.info(f"Have {len(candidates)} candidates to try to push down")
         for candidate in candidates:
             conditions = {x.address for x in candidate.concept_arguments}
             for parent_cte in cte.parent_ctes:
-                materialized = {k for k, v in parent_cte.source_map.items() if v != ""}
+                materialized = {k for k, v in parent_cte.source_map.items() if v != []}
                 if conditions.issubset(materialized):
                     if all(
                         [
@@ -150,12 +153,6 @@ class PredicatePushdown(OptimizationRule):
         return optimized
 
 
-if CONFIG.optimizations.datasource_inlining:
-    REGISTERED_RULES.append(InlineDatasource())
-if CONFIG.optimizations.predicate_pushdown:
-    REGISTERED_RULES.append(PredicatePushdown())
-
-
 def filter_irrelevant_ctes(
     input: list[CTE],
     root_cte: CTE,
@@ -200,6 +197,8 @@ def is_direct_return_eligible(
     for x in derived_concepts:
         if x.derivation == PurposeLineage.WINDOW:
             return False
+        if x.derivation == PurposeLineage.UNNEST:
+            return False
         if x.derivation == PurposeLineage.AGGREGATE:
             if x.address in conditions:
                 return False
@@ -227,6 +226,12 @@ def optimize_ctes(
     input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
 ):
     complete = False
+    REGISTERED_RULES: list["OptimizationRule"] = []
+
+    if CONFIG.optimizations.datasource_inlining:
+        REGISTERED_RULES.append(InlineDatasource())
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdown())
 
     while not complete:
         actions_taken = False
@@ -236,12 +241,21 @@ def optimize_ctes(
                 actions_taken = rule.optimize(cte, inverse_map)
         complete = not actions_taken
 
-    if is_direct_return_eligible(root_cte, select):
+    if CONFIG.optimizations.direct_return and is_direct_return_eligible(
+        root_cte, select
+    ):
         root_cte.order_by = select.order_by
         root_cte.limit = select.limit
-        root_cte.condition = (
-            select.where_clause.conditional if select.where_clause else None
-        )
+        if select.where_clause:
+
+            if root_cte.condition:
+                root_cte.condition = Conditional(
+                    left=root_cte.condition,
+                    operator=BooleanOperator.AND,
+                    right=select.where_clause.conditional,
+                )
+            else:
+                root_cte.condition = select.where_clause.conditional
         root_cte.requires_nesting = False
         sort_select_output(root_cte, select)