pytrilogy 0.0.2.12__py3-none-any.whl → 0.0.2.14__py3-none-any.whl

{pytrilogy-0.0.2.12.dist-info → pytrilogy-0.0.2.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.12
+Version: 0.0.2.14
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.12.dist-info → pytrilogy-0.0.2.14.dist-info}/RECORD

@@ -1,14 +1,14 @@
-trilogy/__init__.py,sha256=qXNp3R3OFRd_QUIgMqKJ4RGQoBEMVcm6s2DHSehlVWU,291
+trilogy/__init__.py,sha256=PE4Dy9UVhAPcaUZRWLaQUxC1U27eYhtTOKd-DOtVDh8,291
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=HRQq4i3cpSEJCywt61QKEzRO1jd4tEPZNSBuxUA_7yg,922
+trilogy/constants.py,sha256=Ijos7_TEajKhZ7OJ_TreEYFddW1V33AVymDDrxP-ZHk,1234
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
-trilogy/executor.py,sha256=5cRbU4Rj7p1pNV76rfp1pz704Hx_0q8_O8HFURjgXxQ,11016
+trilogy/executor.py,sha256=PZr7IF8wS1Oi2WJGE-B3lp70Y8ue2uuauODw02chjdQ,11175
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=WwQPOLfSdL27qhqXw4JTkMyUNvj57T3Xz9M3JUZzhZ8,5940
+trilogy/core/enums.py,sha256=BRYqy-NgIacCYTJo0B11m5XQWSHq5pfxhoLd8pzA3ho,6025
 trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
 trilogy/core/environment_helpers.py,sha256=1miP4is4FEoci01KSAy2VZVYmlmT5TOCOALBekd2muQ,7211
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
@@ -16,42 +16,42 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=ARJAyBjeS415-54k3G_bx807rkPZonEulMaLRxSP7vU,10371
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=kJcSNv6JX79KBEDTZlIyc1nAsn34fWUQLWZN3y7oTVs,143710
+trilogy/core/models.py,sha256=bWCklm8A9I0vx8fXWoN0jKJjLkXuyMUUQetT_zbhyRc,149031
 trilogy/core/optimization.py,sha256=7E-Ol51u6ZAxF56F_bzLxgRO-Hu6Yl1ZbPopZJB2tqk,7533
-trilogy/core/query_processor.py,sha256=Y8C03J9PSyXQoARiMmFomYhnP13L61XjRKNOD7nIops,19520
+trilogy/core/query_processor.py,sha256=qMVkaK1Lvr9jEftJwAidMdkb_tRx12G07qynEyl91C8,18801
 trilogy/core/optimizations/__init__.py,sha256=bWQecbeiwiDx9LJnLsa7dkWxdbl2wcnkcTN69JyP8iI,356
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
 trilogy/core/optimizations/inline_datasource.py,sha256=AATzQ6YrtW_1-aQFjQyTYqEYKBoMFhek7ADfBr4uUdQ,3634
-trilogy/core/optimizations/predicate_pushdown.py,sha256=3hSS1i1itR5lEmLwebuhz4FiPIyFjr2pBeydZfHHPSk,8433
+trilogy/core/optimizations/predicate_pushdown.py,sha256=1l9WnFOSv79e341typG3tTdk0XGl1J_ToQih3LYoGIY,8435
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/concept_strategies_v3.py,sha256=ae6FmwiKNiEbOU2GhnzggFMh82MhqxBj9bgr0ituT2w,25633
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=QKaZL5yJzGJBWCirgB1cAKgcDOibhyk7ETvHveb3GOE,14604
+trilogy/core/processing/utility.py,sha256=jFLZmzxHq94q29FInr8XjS5YiqJOPPBYqh8Tlgs432Y,17722
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
-trilogy/core/processing/node_generators/basic_node.py,sha256=EfCCYleCXVWeoCOUih1VtfUXewg1oyG7EdUMRQOyyMk,3135
-trilogy/core/processing/node_generators/common.py,sha256=lDBRq9X6dQ_xSwXxLLNDq2pW8D-XwAY-ylTJLMugkLw,9525
-trilogy/core/processing/node_generators/filter_node.py,sha256=Ij2WqyOsu-TFxhAcL50PLMGpghsSWXJnWEJ8yTqOwrY,8228
-trilogy/core/processing/node_generators/group_node.py,sha256=Du-9uFXD0M-aHq2MV7v5R3QCrAL0JZBFMW-YQwgb6Bw,3135
-trilogy/core/processing/node_generators/group_to_node.py,sha256=nzITnhaALIT7FMonyo16nNo-kSrLfefa9sZBYecrvkU,2887
-trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
+trilogy/core/processing/node_generators/basic_node.py,sha256=IHj5jEloUe5yojGRLAzt35FcfHqGviWQdS8ETyvr39Q,3292
+trilogy/core/processing/node_generators/common.py,sha256=3_Ivrq_wersDZ5pnvyHvsAUc07mRggxRGTiDq47O0Rk,8840
+trilogy/core/processing/node_generators/filter_node.py,sha256=Tq9_Q6Xe95_-0S5H1EfCgwmKwVfXNoiclqV8QHKhvig,7723
+trilogy/core/processing/node_generators/group_node.py,sha256=G7SrU2X5kjgzeqzzpnPscQBTDcFMc4m7TR6n8VHLC_A,3762
+trilogy/core/processing/node_generators/group_to_node.py,sha256=yX0uw6YMxhyWVRMZoMFzEkJe3tB5ByFqrTnuRWVcRh4,2446
+trilogy/core/processing/node_generators/multiselect_node.py,sha256=OUjndYjA8xR6yKr-J7R-JxDeYfO6DxmMNNcJiFJzk7g,6138
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=D_jsnfoLMrQc08_JvT0wEDvjyzJAxBpdcZFyDN-feV0,13192
-trilogy/core/processing/node_generators/rowset_node.py,sha256=6KVnuk75mRzWJ-jIk7e8azN8BIPPuCn-VxPlxDqfPVE,4616
-trilogy/core/processing/node_generators/select_node.py,sha256=E8bKOAUpwLwZy1iiaFVD5sM4XK-eFpHgijdyIWLMyH4,18904
+trilogy/core/processing/node_generators/rowset_node.py,sha256=qYt0Ngk-WpmAA_gZZORXR-WpLdFqyYAuJiZnrxBmNvw,4201
+trilogy/core/processing/node_generators/select_node.py,sha256=XSMA4kvFdoXlfCpbciXXkbexXkemwUorcAU6P3EwuZY,19843
 trilogy/core/processing/node_generators/unnest_node.py,sha256=aZeixbOzMtXi7BPahKr9bOkIhTciyD9Klsj0kZ56F6s,2189
-trilogy/core/processing/node_generators/window_node.py,sha256=lFfmEjX_mLB7MuOM6CuKNnks1CabokGImpwhbQzjnkE,3283
+trilogy/core/processing/node_generators/window_node.py,sha256=LSlXe41elFGVRlxRX3MEFimhduGn3o5WE0kLx2JtA4M,3322
 trilogy/core/processing/nodes/__init__.py,sha256=jyduHk96j5fpju72sc8swOiBjR3Md866kt8JZGkp3ZU,4866
-trilogy/core/processing/nodes/base_node.py,sha256=szquAzrIkCTXlVhAVSHt6HSJ7rw3b8lfjeO5eFIcEU8,13067
+trilogy/core/processing/nodes/base_node.py,sha256=11Evv2LErwlzCU9ebLWlzldz7VbVMgYiR_sUkVYylKQ,13916
 trilogy/core/processing/nodes/filter_node.py,sha256=DBOSGFfkiILrZa1BlLv2uxUSkgWtSIKiZplqyKXPjg8,2132
-trilogy/core/processing/nodes/group_node.py,sha256=wE6tgyCUL74v76O8jACDm4oYMov4dAlwzLa5xMYReAA,6294
-trilogy/core/processing/nodes/merge_node.py,sha256=34eEH-denk9kkzD8FcZvxgDSMUB9K5e4lSeNpbqSt7I,14456
-trilogy/core/processing/nodes/select_node_v2.py,sha256=QuXNcwgjTRYamOoIooGrp4ie6INcqA9whtC5LZWjD8s,7180
+trilogy/core/processing/nodes/group_node.py,sha256=3zyEs11hv9CoGpO62cUKfClcS58clTUB0IMIkmOV998,6897
+trilogy/core/processing/nodes/merge_node.py,sha256=bn7CwZwbYFx-OjNLb9oQuYL_abwAAd_KSSYJGFSEiP8,15022
+trilogy/core/processing/nodes/select_node_v2.py,sha256=yoU2PWu-BkiUDECd7V7CKAPjznB_LObpl52HU9Sk5Yc,7433
 trilogy/core/processing/nodes/unnest_node.py,sha256=mAmFluzm2yeeiQ6NfIB7BU_8atRGh-UJfPf9ROwbhr8,2152
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/dialect/base.py,sha256=kQek_ufZC9HDVOKlWasvIx6xyew8wv3JNIU6r53_IR4,32842
 trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
-trilogy/dialect/common.py,sha256=LnOtsq4vUTTKB5QUk594QSoNDfOoOF08KQJZZamou84,3359
+trilogy/dialect/common.py,sha256=QCsqo5morOOL6kwaCYh1RBmaInaoPI6lKtzdgroWvuM,3440
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
 trilogy/dialect/duckdb.py,sha256=u_gpL35kouWxoBLas1h0ABYY2QzlVtEh22hm5h0lCOM,3182
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
@@ -65,18 +65,18 @@ trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2
 trilogy/hooks/query_debugger.py,sha256=NDChfkPmmW-KINa4TaQmDe_adGiwsKFdGLDSYpbodeU,4282
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=rLF7SFj_qtLh92ox-cHrtVSyjgl1aTaa7qZJdR1RDuA,8182
+trilogy/parsing/common.py,sha256=-4LM71ocidA8DI2RngqFEOmhzBrIt8VdBTO4x2BpD8E,9502
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=upL2pmq34vwNqzOtmb0EB22tbEZ4TTiK46J5qPq-_yw,62841
+trilogy/parsing/parse_engine.py,sha256=_DNpdZOQoZZio_mW8nxcCKoJHny7hnQQtOGG1msoCFU,63265
 trilogy/parsing/render.py,sha256=Gy_6wVYPwYLf35Iota08sbqveuWILtUhI8MYStcvtJM,12174
-trilogy/parsing/trilogy.lark,sha256=QNJnExOdvJyKTrQA4ffh-SGIz7rYd93kf2Ccs0m3cn4,11498
+trilogy/parsing/trilogy.lark,sha256=-9y4oVAIlKi-6pE58G4RwGGTBeG7P3T_V4gV8mILd8w,11549
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.12.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.12.dist-info/METADATA,sha256=i7Cd69-1p3XsssoHNy1IqbvtZaNeaKpQErvJp940JMw,7907
-pytrilogy-0.0.2.12.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-pytrilogy-0.0.2.12.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.12.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.12.dist-info/RECORD,,
+pytrilogy-0.0.2.14.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.14.dist-info/METADATA,sha256=MqQy1FLZVBpbDh36uTVMwYujLjDB4JkYdz9r5cytOgA,7907
+pytrilogy-0.0.2.14.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+pytrilogy-0.0.2.14.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.14.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.14.dist-info/RECORD,,

{pytrilogy-0.0.2.12.dist-info → pytrilogy-0.0.2.14.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.2)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.12"
+__version__ = "0.0.2.14"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -28,15 +28,30 @@ class Optimizations:
     direct_return: bool = True
 
 
+@dataclass
+class Comments:
+    """Control what is placed in CTE comments"""
+
+    show: bool = False
+    basic: bool = True
+    joins: bool = True
+    nullable: bool = False
+    partial: bool = False
+
+
 # TODO: support loading from environments
 @dataclass
 class Config:
     strict_mode: bool = True
     human_identifiers: bool = True
     validate_missing: bool = True
-    show_comments: bool = False
+    comments: Comments = field(default_factory=Comments)
     optimizations: Optimizations = field(default_factory=Optimizations)
 
+    @property
+    def show_comments(self) -> bool:
+        return self.comments.show
+
     def set_random_seed(self, seed: int):
         random.seed(seed)
 

trilogy/core/enums.py

@@ -62,6 +62,8 @@ class Modifier(Enum):
         strval = str(value)
         if strval == "~":
             return Modifier.PARTIAL
+        elif strval == "?":
+            return Modifier.NULLABLE
         return super()._missing_(value=strval.capitalize())
 
 
@@ -273,6 +275,7 @@ class SourceType(Enum):
     CONSTANT = "constant"
     ROWSET = "rowset"
     MERGE = "merge"
+    BASIC = "basic"
 
 
 class ShowCategory(Enum):

trilogy/core/models.py

@@ -79,6 +79,18 @@ VT = TypeVar("VT")
 LT = TypeVar("LT")
 
 
+def is_compatible_datatype(left, right):
+    if left == right:
+        return True
+    if {left, right} == {DataType.NUMERIC, DataType.FLOAT}:
+        return True
+    if {left, right} == {DataType.NUMERIC, DataType.INTEGER}:
+        return True
+    if {left, right} == {DataType.FLOAT, DataType.INTEGER}:
+        return True
+    return False
+
+
 def get_version():
     from trilogy import __version__
 
@@ -220,6 +232,7 @@ class DataType(Enum):
     ARRAY = "array"
     DATE_PART = "date_part"
     STRUCT = "struct"
+    NULL = "null"
 
     # GRANULAR
     UNIX_SECONDS = "unix_seconds"
@@ -960,6 +973,10 @@ class ColumnAssignment(BaseModel):
     def is_complete(self) -> bool:
         return Modifier.PARTIAL not in self.modifiers
 
+    @property
+    def is_nullable(self) -> bool:
+        return Modifier.NULLABLE in self.modifiers
+
     def with_namespace(self, namespace: str) -> "ColumnAssignment":
         return ColumnAssignment(
             alias=(
@@ -1065,9 +1082,12 @@ class Function(Mergeable, Namespaced, SelectContext, BaseModel):
         ]
     ]
 
-    def __str__(self):
+    def __repr__(self):
         return f'{self.operator.value}({",".join([str(a) for a in self.arguments])})'
 
+    def __str__(self):
+        return self.__repr__()
+
     @property
     def datatype(self):
         return self.output_datatype
@@ -2072,6 +2092,10 @@ class Datasource(Namespaced, BaseModel):
     def full_concepts(self) -> List[Concept]:
         return [c.concept for c in self.columns if Modifier.PARTIAL not in c.modifiers]
 
+    @property
+    def nullable_concepts(self) -> List[Concept]:
+        return [c.concept for c in self.columns if Modifier.NULLABLE in c.modifiers]
+
     @property
     def output_concepts(self) -> List[Concept]:
         return self.concepts
@@ -2135,13 +2159,27 @@ class InstantiatedUnnestJoin(BaseModel):
     alias: str = "unnest"
 
 
+class ConceptPair(BaseModel):
+    left: Concept
+    right: Concept
+    modifiers: List[Modifier] = Field(default_factory=list)
+
+    @property
+    def is_partial(self):
+        return Modifier.PARTIAL in self.modifiers
+
+    @property
+    def is_nullable(self):
+        return Modifier.NULLABLE in self.modifiers
+
+
 class BaseJoin(BaseModel):
     left_datasource: Union[Datasource, "QueryDatasource"]
     right_datasource: Union[Datasource, "QueryDatasource"]
     concepts: List[Concept]
     join_type: JoinType
     filter_to_mutual: bool = False
-    concept_pairs: list[tuple[Concept, Concept]] | None = None
+    concept_pairs: list[ConceptPair] | None = None
 
     def __init__(self, **data: Any):
         super().__init__(**data)
@@ -2219,7 +2257,7 @@ class BaseJoin(BaseModel):
             return (
                 f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
                 f" {self.right_datasource.identifier} on"
-                f" {','.join([str(k[0])+'='+str(k[1]) for k in self.concept_pairs])}"
+                f" {','.join([str(k.left)+'='+str(k.right) for k in self.concept_pairs])}"
             )
         return (
             f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
@@ -2243,8 +2281,9 @@ class QueryDatasource(BaseModel):
     filter_concepts: List[Concept] = Field(default_factory=list)
     source_type: SourceType = SourceType.SELECT
     partial_concepts: List[Concept] = Field(default_factory=list)
-    join_derived_concepts: List[Concept] = Field(default_factory=list)
     hidden_concepts: List[Concept] = Field(default_factory=list)
+    nullable_concepts: List[Concept] = Field(default_factory=list)
+    join_derived_concepts: List[Concept] = Field(default_factory=list)
     force_group: bool | None = None
     existence_source_map: Dict[str, Set[Union[Datasource, "QueryDatasource"]]] = Field(
         default_factory=dict
@@ -2264,6 +2303,7 @@ class QueryDatasource(BaseModel):
     @field_validator("joins")
     @classmethod
     def validate_joins(cls, v):
+        unique_pairs = set()
         for join in v:
             if not isinstance(join, BaseJoin):
                 continue
@@ -2271,7 +2311,16 @@ class QueryDatasource(BaseModel):
                 raise SyntaxError(
                     f"Cannot join a datasource to itself, joining {join.left_datasource}"
                 )
-
+            pairing = "".join(
+                sorted(
+                    [join.left_datasource.identifier, join.right_datasource.identifier]
+                )
+            )
+            if pairing in unique_pairs:
+                raise SyntaxError(
+                    f"Duplicate join {join.left_datasource.identifier} and {join.right_datasource.identifier}"
+                )
+            unique_pairs.add(pairing)
         return v
 
     @field_validator("input_concepts")
@@ -2386,6 +2435,11 @@ class QueryDatasource(BaseModel):
                 final_source_map[key] = other.source_map[key]
         for k, v in final_source_map.items():
             final_source_map[k] = set(merged_datasources[x.full_name] for x in list(v))
+        self_hidden = self.hidden_concepts or []
+        other_hidden = other.hidden_concepts or []
+        hidden = [
+            x for x in self_hidden if x.address in [y.address for y in other_hidden]
+        ]
         qds = QueryDatasource(
             input_concepts=unique(
                 self.input_concepts + other.input_concepts, "address"
@@ -2409,9 +2463,7 @@ class QueryDatasource(BaseModel):
             ),
             join_derived_concepts=self.join_derived_concepts,
             force_group=self.force_group,
-            hidden_concepts=unique(
-                self.hidden_concepts + other.hidden_concepts, "address"
-            ),
+            hidden_concepts=hidden,
         )
 
         return qds
@@ -2487,6 +2539,7 @@ class CTE(BaseModel):
     joins: List[Union["Join", "InstantiatedUnnestJoin"]] = Field(default_factory=list)
     condition: Optional[Union["Conditional", "Comparison", "Parenthetical"]] = None
     partial_concepts: List[Concept] = Field(default_factory=list)
+    nullable_concepts: List[Concept] = Field(default_factory=list)
     join_derived_concepts: List[Concept] = Field(default_factory=list)
     hidden_concepts: List[Concept] = Field(default_factory=list)
     order_by: Optional[OrderBy] = None
@@ -2557,6 +2610,7 @@ class CTE(BaseModel):
     @property
     def comment(self) -> str:
         base = f"Target: {str(self.grain)}."
+        base += f" Source: {self.source.source_type}."
         if self.parent_ctes:
             base += f" References: {', '.join([x.name for x in self.parent_ctes])}."
         if self.joins:
@@ -2565,6 +2619,15 @@ class CTE(BaseModel):
             base += (
                 f"\n-- Partials: {', '.join([str(x) for x in self.partial_concepts])}."
             )
+        base += f"\n-- Source Map: {self.source_map}."
+        base += f"\n-- Output: {', '.join([str(x) for x in self.output_columns])}."
+        if self.hidden_concepts:
+            base += f"\n-- Hidden: {', '.join([str(x) for x in self.hidden_concepts])}."
+        if self.nullable_concepts:
+            base += (
+                f"\n-- Nullable: {', '.join([str(x) for x in self.nullable_concepts])}."
+            )
+
         return base
 
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
@@ -2632,6 +2695,10 @@ class CTE(BaseModel):
                 f" {self.name} {other.name} conditions {self.condition} {other.condition}"
             )
             raise ValueError(error)
+        mutually_hidden = []
+        for concept in self.hidden_concepts:
+            if concept in other.hidden_concepts:
+                mutually_hidden.append(concept)
         self.partial_concepts = unique(
             self.partial_concepts + other.partial_concepts, "address"
         )
@@ -2654,9 +2721,10 @@ class CTE(BaseModel):
         self.source.output_concepts = unique(
             self.source.output_concepts + other.source.output_concepts, "address"
         )
-        self.hidden_concepts = unique(
-            self.hidden_concepts + other.hidden_concepts, "address"
+        self.nullable_concepts = unique(
+            self.nullable_concepts + other.nullable_concepts, "address"
         )
+        self.hidden_concepts = mutually_hidden
         self.existence_source_map = {
             **self.existence_source_map,
             **other.existence_source_map,
@@ -2821,7 +2889,7 @@ class Join(BaseModel):
     right_cte: CTE | Datasource
     jointype: JoinType
     joinkeys: List[JoinKey]
-    joinkey_pairs: List[tuple[Concept, Concept]] | None = None
+    joinkey_pairs: List[ConceptPair] | None = None
 
     @property
     def left_name(self) -> str:
@@ -2856,7 +2924,7 @@ class Join(BaseModel):
             return (
                 f"{self.jointype.value} JOIN {self.left_name} and"
                 f" {self.right_name} on"
-                f" {','.join([str(k[0])+'='+str(k[1]) for k in self.joinkey_pairs])}"
+                f" {','.join([str(k.left)+'='+str(k.right)+str(k.modifiers) for k in self.joinkey_pairs])}"
             )
         return (
             f"{self.jointype.value} JOIN {self.left_name} and"
@@ -3012,6 +3080,9 @@ class EnvironmentDatasourceDict(dict):
     def values(self) -> ValuesView[Datasource]:  # type: ignore
         return super().values()
 
+    def items(self) -> ItemsView[str, Datasource]:  # type: ignore
+        return super().items()
+
 
 class EnvironmentConceptDict(dict):
     def __init__(self, *args, **kwargs) -> None:
@@ -3426,11 +3497,41 @@ class Comparison(
     ]
     operator: ComparisonOperator
 
-    def __post_init__(self):
-        if arg_to_datatype(self.left) != arg_to_datatype(self.right):
-            raise SyntaxError(
-                f"Cannot compare {self.left} and {self.right} of different types"
-            )
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        if self.operator in (ComparisonOperator.IS, ComparisonOperator.IS_NOT):
+            if self.right != MagicConstants.NULL and DataType.BOOL != arg_to_datatype(
+                self.right
+            ):
+                raise SyntaxError(
+                    f"Cannot use {self.operator.value} with non-null or boolean value {self.right}"
+                )
+        elif self.operator in (ComparisonOperator.IN, ComparisonOperator.NOT_IN):
+            right = arg_to_datatype(self.right)
+            if not isinstance(self.right, Concept) and not isinstance(right, ListType):
+                raise SyntaxError(
+                    f"Cannot use {self.operator.value} with non-list type {right} in {str(self)}"
+                )
+
+            elif isinstance(right, ListType) and not is_compatible_datatype(
+                arg_to_datatype(self.left), right.value_data_type
+            ):
+                raise SyntaxError(
+                    f"Cannot compare {arg_to_datatype(self.left)} and {right} with operator {self.operator} in {str(self)}"
+                )
+            elif isinstance(self.right, Concept) and not is_compatible_datatype(
+                arg_to_datatype(self.left), arg_to_datatype(self.right)
+            ):
+                raise SyntaxError(
+                    f"Cannot compare {arg_to_datatype(self.left)} and {arg_to_datatype(self.right)} with operator {self.operator} in {str(self)}"
+                )
+        else:
+            if not is_compatible_datatype(
+                arg_to_datatype(self.left), arg_to_datatype(self.right)
+            ):
+                raise SyntaxError(
+                    f"Cannot compare {arg_to_datatype(self.left)} and {arg_to_datatype(self.right)} of different types with operator {self.operator} in {str(self)}"
+                )
 
     def __add__(self, other):
         if other is None:
@@ -3631,6 +3732,12 @@ class CaseWhen(Namespaced, SelectContext, BaseModel):
     comparison: Conditional | SubselectComparison | Comparison
     expr: "Expr"
 
+    def __str__(self):
+        return self.__repr__()
+
+    def __repr__(self):
+        return f"WHEN {str(self.comparison)} THEN {str(self.expr)}"
+
     @property
     def concept_arguments(self):
         return get_concept_arguments(self.comparison) + get_concept_arguments(self.expr)
@@ -4323,6 +4430,7 @@ class ShowStatement(BaseModel):
 
 Expr = (
     bool
+    | MagicConstants
     | int
     | str
     | float
@@ -4377,9 +4485,34 @@ def dict_to_map_wrapper(arg):
     return MapWrapper(arg, key_type=key_types[0], value_type=value_types[0])
 
 
+def merge_datatypes(
+    inputs: list[DataType | ListType | StructType | MapType | NumericType],
+) -> DataType | ListType | StructType | MapType | NumericType:
+    """This is a temporary hack for doing between
+    allowable datatype transformation matrix"""
+    if len(inputs) == 1:
+        return inputs[0]
+    if set(inputs) == {DataType.INTEGER, DataType.FLOAT}:
+        return DataType.FLOAT
+    if set(inputs) == {DataType.INTEGER, DataType.NUMERIC}:
+        return DataType.NUMERIC
+    if any(isinstance(x, NumericType) for x in inputs) and all(
+        isinstance(x, NumericType)
+        or x in (DataType.INTEGER, DataType.FLOAT, DataType.NUMERIC)
+        for x in inputs
+    ):
+        candidate = next(x for x in inputs if isinstance(x, NumericType))
+        return candidate
+    return inputs[0]
+
+
 def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | NumericType:
     if isinstance(arg, Function):
         return arg.output_datatype
+    elif isinstance(arg, MagicConstants):
+        if arg == MagicConstants.NULL:
+            return DataType.NULL
+        raise ValueError(f"Cannot parse arg datatype for arg of type {arg}")
     elif isinstance(arg, Concept):
         return arg.datatype
     elif isinstance(arg, bool):

trilogy/core/optimizations/predicate_pushdown.py

@@ -135,7 +135,7 @@ class PredicatePushdown(OptimizationRule):
                     f"Skipping {candidate} as not a basic [no aggregate, etc] condition"
                 )
                 continue
-            self.log(
+            self.debug(
                 f"Checking candidate {candidate}, {type(candidate)}, scalar: {is_scalar_condition(candidate)}"
             )
             for parent_cte in cte.parent_ctes:

trilogy/core/processing/node_generators/basic_node.py

@@ -10,6 +10,7 @@ from trilogy.core.processing.node_generators.common import (
 )
 from trilogy.utility import unique
 from trilogy.constants import logger
+from trilogy.core.enums import SourceType
 from itertools import combinations
 
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
@@ -35,7 +36,11 @@ def gen_basic_node(
     attempts: List[tuple[list[Concept], list[Concept]]] = [
         (parent_concepts, [concept] + local_optional_redundant)
     ]
-    equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
+    equivalent_optional = [
+        x
+        for x in local_optional
+        if x.lineage == concept.lineage and x.address != concept.address
+    ]
     non_equivalent_optional = [
         x for x in local_optional if x not in equivalent_optional
     ]
@@ -61,8 +66,10 @@ def gen_basic_node(
             depth=depth + 1,
             history=history,
         )
+
         if not parent_node:
             continue
+        parent_node.source_type = SourceType.BASIC
         parents: List[StrategyNode] = [parent_node]
         for x in basic_output:
             sources = [p for p in parents if x in p.output_concepts]