pytrilogy 0.0.2.11__py3-none-any.whl → 0.0.2.13__py3-none-any.whl

{pytrilogy-0.0.2.11.dist-info → pytrilogy-0.0.2.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.11
+Version: 0.0.2.13
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.11.dist-info → pytrilogy-0.0.2.13.dist-info}/RECORD

@@ -1,57 +1,57 @@
-trilogy/__init__.py,sha256=rSlG7hlvpZjtLdU_7j6ldvO5bmiafhgPB40sdcOu3j0,291
+trilogy/__init__.py,sha256=U1mVKIIGVmjHu6kwRMyNhb3buZHWWzbk-c9Hu7FiBGQ,291
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=HRQq4i3cpSEJCywt61QKEzRO1jd4tEPZNSBuxUA_7yg,922
+trilogy/constants.py,sha256=rZJh3fAx3ljxf_QZNECR-devR6QXkYc9mpLCxIWNqB0,960
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
-trilogy/executor.py,sha256=5cRbU4Rj7p1pNV76rfp1pz704Hx_0q8_O8HFURjgXxQ,11016
+trilogy/executor.py,sha256=PZr7IF8wS1Oi2WJGE-B3lp70Y8ue2uuauODw02chjdQ,11175
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=D5HUKiknRcx_OBoX21IAM0JFQ-vQNBgad2GOvJfXi3o,5964
+trilogy/core/enums.py,sha256=BRYqy-NgIacCYTJo0B11m5XQWSHq5pfxhoLd8pzA3ho,6025
 trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
-trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
+trilogy/core/environment_helpers.py,sha256=1miP4is4FEoci01KSAy2VZVYmlmT5TOCOALBekd2muQ,7211
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
 trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
 trilogy/core/functions.py,sha256=ARJAyBjeS415-54k3G_bx807rkPZonEulMaLRxSP7vU,10371
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=rIeYw173UjfTfc7194y-jqv7u51Uy6LufwbiwKI2Cgs,143834
-trilogy/core/optimization.py,sha256=RJmDr2f9hyFePF-B6LpmHkj69-AzUm-nNtvO59899O8,6601
-trilogy/core/query_processor.py,sha256=50FHJ9Rw89ZNBflZK8R9jn7O_WdDHplS1hZSNXCozN8,19352
-trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
+trilogy/core/models.py,sha256=9-1kYfq4NqAthe50Jgs0WjBnZtEWJw0ReCBfFhVxD0Y,144799
+trilogy/core/optimization.py,sha256=7E-Ol51u6ZAxF56F_bzLxgRO-Hu6Yl1ZbPopZJB2tqk,7533
+trilogy/core/query_processor.py,sha256=JJFBVBmT5QNsJ9rSDKNJBINLm7YW7i5fjAp98H0Wcd8,19281
+trilogy/core/optimizations/__init__.py,sha256=bWQecbeiwiDx9LJnLsa7dkWxdbl2wcnkcTN69JyP8iI,356
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
 trilogy/core/optimizations/inline_datasource.py,sha256=AATzQ6YrtW_1-aQFjQyTYqEYKBoMFhek7ADfBr4uUdQ,3634
-trilogy/core/optimizations/predicate_pushdown.py,sha256=4Y6zfJN3VVexkD6p9IYWN4BTL8RwW6hyNb0VQZ8ETdI,8020
+trilogy/core/optimizations/predicate_pushdown.py,sha256=3hSS1i1itR5lEmLwebuhz4FiPIyFjr2pBeydZfHHPSk,8433
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=5yZQRWlEyNR2sddq7kx12_PRUNhekEu6bWO2yIvjjUA,25511
+trilogy/core/processing/concept_strategies_v3.py,sha256=ae6FmwiKNiEbOU2GhnzggFMh82MhqxBj9bgr0ituT2w,25633
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=15Qi68ktmnQmMt5jxrpueQ34oG5fSSgst3kIEejwF8A,14519
+trilogy/core/processing/utility.py,sha256=QKaZL5yJzGJBWCirgB1cAKgcDOibhyk7ETvHveb3GOE,14604
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
-trilogy/core/processing/node_generators/basic_node.py,sha256=4242PNGTCm2tklqMIkqVu5Iv4m_IeTnOYXxDveuCDZM,2856
-trilogy/core/processing/node_generators/common.py,sha256=lDBRq9X6dQ_xSwXxLLNDq2pW8D-XwAY-ylTJLMugkLw,9525
+trilogy/core/processing/node_generators/basic_node.py,sha256=IHj5jEloUe5yojGRLAzt35FcfHqGviWQdS8ETyvr39Q,3292
+trilogy/core/processing/node_generators/common.py,sha256=WY41zjxSCG13n3mdUHmcE0mUQ0gtWxz4nugBgxHONd8,9327
 trilogy/core/processing/node_generators/filter_node.py,sha256=Ij2WqyOsu-TFxhAcL50PLMGpghsSWXJnWEJ8yTqOwrY,8228
 trilogy/core/processing/node_generators/group_node.py,sha256=Du-9uFXD0M-aHq2MV7v5R3QCrAL0JZBFMW-YQwgb6Bw,3135
 trilogy/core/processing/node_generators/group_to_node.py,sha256=nzITnhaALIT7FMonyo16nNo-kSrLfefa9sZBYecrvkU,2887
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=qS8VKppi10NKfqNb7BXTKyLBBn4wJuMmFYhrtMUkYb8,12580
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=D_jsnfoLMrQc08_JvT0wEDvjyzJAxBpdcZFyDN-feV0,13192
 trilogy/core/processing/node_generators/rowset_node.py,sha256=6KVnuk75mRzWJ-jIk7e8azN8BIPPuCn-VxPlxDqfPVE,4616
 trilogy/core/processing/node_generators/select_node.py,sha256=E8bKOAUpwLwZy1iiaFVD5sM4XK-eFpHgijdyIWLMyH4,18904
-trilogy/core/processing/node_generators/unnest_node.py,sha256=gHjurhr86JFkbq6vxTHDQGDKt95EdotSFHVKgrR_Z3Q,1860
+trilogy/core/processing/node_generators/unnest_node.py,sha256=aZeixbOzMtXi7BPahKr9bOkIhTciyD9Klsj0kZ56F6s,2189
 trilogy/core/processing/node_generators/window_node.py,sha256=lFfmEjX_mLB7MuOM6CuKNnks1CabokGImpwhbQzjnkE,3283
 trilogy/core/processing/nodes/__init__.py,sha256=jyduHk96j5fpju72sc8swOiBjR3Md866kt8JZGkp3ZU,4866
-trilogy/core/processing/nodes/base_node.py,sha256=KVLAPLlo5QQq9Cm6lTLsqLN7l8EyXGxQkKrZFHdyRp8,12871
+trilogy/core/processing/nodes/base_node.py,sha256=P4VyOhbgPgTLaXEIftbVVmmEPviScJJhi9v6hSxjC7M,13155
 trilogy/core/processing/nodes/filter_node.py,sha256=DBOSGFfkiILrZa1BlLv2uxUSkgWtSIKiZplqyKXPjg8,2132
-trilogy/core/processing/nodes/group_node.py,sha256=RKqODYxbY8wmClvI8-k3UgXWL7qQv5Hxhz5AB-RVUu0,6344
-trilogy/core/processing/nodes/merge_node.py,sha256=ZUcFIm5v4hip_Ml_oJXqVMrevUkE62wvPrvCrO9FWAc,14272
+trilogy/core/processing/nodes/group_node.py,sha256=wE6tgyCUL74v76O8jACDm4oYMov4dAlwzLa5xMYReAA,6294
+trilogy/core/processing/nodes/merge_node.py,sha256=KOTdYli_T3c5RwMaK_73W5UNtKPA0F8hjDJF9jQx2fs,14491
 trilogy/core/processing/nodes/select_node_v2.py,sha256=QuXNcwgjTRYamOoIooGrp4ie6INcqA9whtC5LZWjD8s,7180
-trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
+trilogy/core/processing/nodes/unnest_node.py,sha256=mAmFluzm2yeeiQ6NfIB7BU_8atRGh-UJfPf9ROwbhr8,2152
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=xbZXcWvwdRadyb77J2qat67R_gRAmKeeeI4JRH4P928,30726
+trilogy/dialect/base.py,sha256=kQek_ufZC9HDVOKlWasvIx6xyew8wv3JNIU6r53_IR4,32842
 trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
-trilogy/dialect/common.py,sha256=HVNPL8dGyQjT2REruV5C2YPpBbVR7KlD0akVM7GcuPI,3329
+trilogy/dialect/common.py,sha256=LnOtsq4vUTTKB5QUk594QSoNDfOoOF08KQJZZamou84,3359
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
 trilogy/dialect/duckdb.py,sha256=u_gpL35kouWxoBLas1h0ABYY2QzlVtEh22hm5h0lCOM,3182
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
@@ -65,18 +65,18 @@ trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2
 trilogy/hooks/query_debugger.py,sha256=NDChfkPmmW-KINa4TaQmDe_adGiwsKFdGLDSYpbodeU,4282
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=zNd5buKxK4z9WSszOk4zOI0GexDFukzZtdNfQxg3kVw,6218
+trilogy/parsing/common.py,sha256=fa3R0xHXQkU8BUN8quPe7qCzez6qaRbnctkYaLgHLxY,8863
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=riXOhMqYGvzyy1aeTZTjPIz3jOszue2VTEH7hn59zD4,63808
+trilogy/parsing/parse_engine.py,sha256=yCHd6RVRijNctZJZ3iRJG2263UZL2n8EoFA_-Qfr88E,62966
 trilogy/parsing/render.py,sha256=Gy_6wVYPwYLf35Iota08sbqveuWILtUhI8MYStcvtJM,12174
-trilogy/parsing/trilogy.lark,sha256=QNJnExOdvJyKTrQA4ffh-SGIz7rYd93kf2Ccs0m3cn4,11498
+trilogy/parsing/trilogy.lark,sha256=00j0D77gpTBsyet9WGN4Ir9Nc-YvNwXFr2toRiASb_M,11525
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.11.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.11.dist-info/METADATA,sha256=mZ6V2gwihh9nv-bIGkimK2-bKNvsY_3JIUSNzP_m2Wg,7907
-pytrilogy-0.0.2.11.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-pytrilogy-0.0.2.11.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.11.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.11.dist-info/RECORD,,
+pytrilogy-0.0.2.13.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.13.dist-info/METADATA,sha256=AJXWyDVSN9EhYPYJlrkXhMVULqg3HyscbZGDhgVd-FA,7907
+pytrilogy-0.0.2.13.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+pytrilogy-0.0.2.13.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.13.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.13.dist-info/RECORD,,

{pytrilogy-0.0.2.11.dist-info → pytrilogy-0.0.2.13.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.2)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.11"
+__version__ = "0.0.2.13"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -28,6 +28,11 @@ class Optimizations:
     direct_return: bool = True
 
 
+@dataclass
+class Comments:
+    pass
+
+
 # TODO: support loading from environments
 @dataclass
 class Config:

trilogy/core/enums.py

@@ -62,6 +62,8 @@ class Modifier(Enum):
         strval = str(value)
         if strval == "~":
             return Modifier.PARTIAL
+        elif strval == "?":
+            return Modifier.NULLABLE
         return super()._missing_(value=strval.capitalize())
 
 
@@ -229,7 +231,6 @@ class ComparisonOperator(Enum):
     ILIKE = "ilike"
     CONTAINS = "contains"
     ELSE = "else"
-    BETWEEN = "between"
 
     @classmethod
     def _missing_(cls, value):
@@ -274,6 +275,7 @@ class SourceType(Enum):
     CONSTANT = "constant"
     ROWSET = "rowset"
     MERGE = "merge"
+    BASIC = "basic"
 
 
 class ShowCategory(Enum):

trilogy/core/environment_helpers.py

@@ -1,6 +1,15 @@
-from trilogy.core.models import DataType, Concept, Environment, Function, Metadata
+from trilogy.core.models import (
+    DataType,
+    Concept,
+    Environment,
+    Function,
+    Metadata,
+    StructType,
+)
+from trilogy.core.functions import AttrAccess
 from trilogy.core.enums import Purpose, FunctionType, ConceptSource
 from trilogy.constants import DEFAULT_NAMESPACE
+from trilogy.parsing.common import process_function_args, arg_to_datatype, Meta
 
 
 def generate_date_concepts(concept: Concept, environment: Environment):
@@ -142,15 +151,44 @@ def generate_key_concepts(concept: Concept, environment: Environment):
         environment.add_concept(new_concept, add_derived=False)
 
 
-def generate_related_concepts(concept: Concept, environment: Environment):
+def generate_related_concepts(
+    concept: Concept,
+    environment: Environment,
+    meta: Meta | None = None,
+    add_derived: bool = False,
+):
     """Auto populate common derived concepts on types"""
-    if concept.purpose == Purpose.KEY:
+    if concept.purpose == Purpose.KEY and add_derived:
         generate_key_concepts(concept, environment)
-    if concept.datatype == DataType.DATE:
+
+    # datatype types
+    if concept.datatype == DataType.DATE and add_derived:
         generate_date_concepts(concept, environment)
-    elif concept.datatype == DataType.DATETIME:
+    elif concept.datatype == DataType.DATETIME and add_derived:
         generate_date_concepts(concept, environment)
         generate_datetime_concepts(concept, environment)
-    elif concept.datatype == DataType.TIMESTAMP:
+    elif concept.datatype == DataType.TIMESTAMP and add_derived:
         generate_date_concepts(concept, environment)
         generate_datetime_concepts(concept, environment)
+
+    if isinstance(concept.datatype, StructType):
+        for key, value in concept.datatype.fields_map.items():
+            args = process_function_args(
+                [concept, key], meta=meta, environment=environment
+            )
+            auto = Concept(
+                name=key,
+                datatype=arg_to_datatype(value),
+                purpose=Purpose.PROPERTY,
+                namespace=(
+                    environment.namespace + "." + concept.name
+                    if environment.namespace
+                    and environment.namespace != DEFAULT_NAMESPACE
+                    else concept.name
+                ),
+                lineage=AttrAccess(args),
+            )
+            environment.add_concept(auto, meta=meta)
+            if isinstance(value, Concept):
+                environment.merge_concept(auto, value, modifiers=[])
+                assert value.pseudonyms is not None

trilogy/core/models.py

@@ -300,7 +300,7 @@ class MapType(BaseModel):
 
 class StructType(BaseModel):
     fields: List[ALL_TYPES]
-    fields_map: Dict[str, Concept | int | float | str] = Field(default_factory=dict)
+    fields_map: Dict[str, Concept | int | float | str]
 
     @property
     def data_type(self):
@@ -2119,16 +2119,19 @@ class Datasource(Namespaced, BaseModel):
 
 
 class UnnestJoin(BaseModel):
-    concept: Concept
+    concepts: list[Concept]
+    parent: Function
     alias: str = "unnest"
     rendering_required: bool = True
 
     def __hash__(self):
-        return (self.alias + self.concept.address).__hash__()
+        return (
+            self.alias + "".join([str(s.address) for s in self.concepts])
+        ).__hash__()
 
 
 class InstantiatedUnnestJoin(BaseModel):
-    concept: Concept
+    concept_to_unnest: Concept
     alias: str = "unnest"
 
 
@@ -2261,6 +2264,7 @@ class QueryDatasource(BaseModel):
     @field_validator("joins")
     @classmethod
     def validate_joins(cls, v):
+        unique_pairs = set()
         for join in v:
             if not isinstance(join, BaseJoin):
                 continue
@@ -2268,6 +2272,16 @@ class QueryDatasource(BaseModel):
                 raise SyntaxError(
                     f"Cannot join a datasource to itself, joining {join.left_datasource}"
                 )
+            pairing = "".join(
+                sorted(
+                    [join.left_datasource.identifier, join.right_datasource.identifier]
+                )
+            )
+            if pairing in unique_pairs:
+                raise SyntaxError(
+                    f"Duplicate join {join.left_datasource.identifier} and {join.right_datasource.identifier}"
+                )
+            unique_pairs.add(pairing)
         return v
 
     @field_validator("input_concepts")
@@ -2287,8 +2301,13 @@ class QueryDatasource(BaseModel):
         for key in ("input_concepts", "output_concepts"):
             if not values.get(key):
                 continue
+            concept: Concept
             for concept in values[key]:
-                if concept.address not in v and CONFIG.validate_missing:
+                if (
+                    concept.address not in v
+                    and not any(x in v for x in concept.pseudonyms)
+                    and CONFIG.validate_missing
+                ):
                     raise SyntaxError(
                         f"Missing source map for {concept.address} on {key}, have {v}"
                     )
@@ -2377,6 +2396,11 @@ class QueryDatasource(BaseModel):
                 final_source_map[key] = other.source_map[key]
         for k, v in final_source_map.items():
             final_source_map[k] = set(merged_datasources[x.full_name] for x in list(v))
+        self_hidden = self.hidden_concepts or []
+        other_hidden = other.hidden_concepts or []
+        hidden = [
+            x for x in self_hidden if x.address in [y.address for y in other_hidden]
+        ]
         qds = QueryDatasource(
             input_concepts=unique(
                 self.input_concepts + other.input_concepts, "address"
@@ -2400,9 +2424,7 @@ class QueryDatasource(BaseModel):
             ),
             join_derived_concepts=self.join_derived_concepts,
             force_group=self.force_group,
-            hidden_concepts=unique(
-                self.hidden_concepts + other.hidden_concepts, "address"
-            ),
+            hidden_concepts=hidden,
         )
 
         return qds
@@ -2533,7 +2555,7 @@ class CTE(BaseModel):
                     )
                 ]
                 for join in self.joins:
-                    if isinstance(join, UnnestJoin) and join.concept == concept:
+                    if isinstance(join, UnnestJoin) and concept in join.concepts:
                         join.rendering_required = False
 
                 self.parent_ctes = [
@@ -2548,6 +2570,7 @@ class CTE(BaseModel):
     @property
     def comment(self) -> str:
         base = f"Target: {str(self.grain)}."
+        base += f" Source: {self.source.source_type}."
         if self.parent_ctes:
             base += f" References: {', '.join([x.name for x in self.parent_ctes])}."
         if self.joins:
@@ -2556,6 +2579,11 @@ class CTE(BaseModel):
             base += (
                 f"\n-- Partials: {', '.join([str(x) for x in self.partial_concepts])}."
             )
+        base += f"\n-- Source Map: {self.source_map}."
+        base += f"\n-- Output: {', '.join([str(x) for x in self.output_columns])}."
+        if self.hidden_concepts:
+            base += f"\n-- Hidden: {', '.join([str(x) for x in self.hidden_concepts])}."
+
         return base
 
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
@@ -2623,6 +2651,10 @@ class CTE(BaseModel):
                 f" {self.name} {other.name} conditions {self.condition} {other.condition}"
             )
             raise ValueError(error)
+        mutually_hidden = []
+        for concept in self.hidden_concepts:
+            if concept in other.hidden_concepts:
+                mutually_hidden.append(concept)
         self.partial_concepts = unique(
             self.partial_concepts + other.partial_concepts, "address"
         )
@@ -2645,9 +2677,7 @@ class CTE(BaseModel):
         self.source.output_concepts = unique(
             self.source.output_concepts + other.source.output_concepts, "address"
         )
-        self.hidden_concepts = unique(
-            self.hidden_concepts + other.hidden_concepts, "address"
-        )
+        self.hidden_concepts = mutually_hidden
         self.existence_source_map = {
             **self.existence_source_map,
             **other.existence_source_map,
@@ -2996,13 +3026,16 @@ class EnvironmentDatasourceDict(dict):
         except KeyError:
             if DEFAULT_NAMESPACE + "." + key in self:
                 return self.__getitem__(DEFAULT_NAMESPACE + "." + key)
-            if "." in key and key.split(".")[0] == DEFAULT_NAMESPACE:
-                return self.__getitem__(key.split(".")[1])
+            if "." in key and key.split(".", 1)[0] == DEFAULT_NAMESPACE:
+                return self.__getitem__(key.split(".", 1)[1])
             raise
 
     def values(self) -> ValuesView[Datasource]:  # type: ignore
         return super().values()
 
+    def items(self) -> ItemsView[str, Datasource]:  # type: ignore
+        return super().items()
+
 
 class EnvironmentConceptDict(dict):
     def __init__(self, *args, **kwargs) -> None:
@@ -3027,8 +3060,8 @@ class EnvironmentConceptDict(dict):
             return super(EnvironmentConceptDict, self).__getitem__(key)
 
         except KeyError:
-            if "." in key and key.split(".")[0] == DEFAULT_NAMESPACE:
-                return self.__getitem__(key.split(".")[1], line_no)
+            if "." in key and key.split(".", 1)[0] == DEFAULT_NAMESPACE:
+                return self.__getitem__(key.split(".", 1)[1], line_no)
             if DEFAULT_NAMESPACE + "." + key in self:
                 return self.__getitem__(DEFAULT_NAMESPACE + "." + key, line_no)
             if not self.fail_on_missing:
@@ -3293,10 +3326,9 @@ class Environment(BaseModel):
             self.concepts[concept.name] = concept
         else:
             self.concepts[concept.address] = concept
-        if add_derived:
-            from trilogy.core.environment_helpers import generate_related_concepts
+        from trilogy.core.environment_helpers import generate_related_concepts
 
-            generate_related_concepts(concept, self)
+        generate_related_concepts(concept, self, meta=meta, add_derived=add_derived)
         self.gen_concept_list_caches()
         return concept
 
@@ -3423,14 +3455,6 @@ class Comparison(
             raise SyntaxError(
                 f"Cannot compare {self.left} and {self.right} of different types"
             )
-        if self.operator == ComparisonOperator.BETWEEN:
-            if (
-                not isinstance(self.right, ComparisonOperator)
-                and self.right.operator == BooleanOperator.AND
-            ):
-                raise SyntaxError(
-                    f"Between operator must have two operands with and, not {self.right}"
-                )
 
     def __add__(self, other):
         if other is None:

trilogy/core/optimization.py

@@ -10,6 +10,7 @@ from trilogy.core.optimizations import (
     OptimizationRule,
     InlineConstant,
     PredicatePushdown,
+    PredicatePushdownRemove,
     InlineDatasource,
 )
 
@@ -34,6 +35,31 @@ MAX_OPTIMIZATION_LOOPS = 100
 #     return parent
 
 
+def reorder_ctes(
+    input: list[CTE],
+):
+    import networkx as nx
+
+    # Create a directed graph
+    G = nx.DiGraph()
+    mapping: dict[str, CTE] = {}
+    for cte in input:
+        mapping[cte.name] = cte
+        for parent in cte.parent_ctes:
+            G.add_edge(parent.name, cte.name)
+    # Perform topological sort (only works for DAGs)
+    try:
+        topological_order = list(nx.topological_sort(G))
+        if not topological_order:
+            return input
+        return [mapping[x] for x in topological_order]
+    except nx.NetworkXUnfeasible as e:
+        print(
+            "The graph is not a DAG (contains cycles) and cannot be topologically sorted."
+        )
+        raise e
+
+
 def filter_irrelevant_ctes(
     input: list[CTE],
     root_cte: CTE,
@@ -169,20 +195,22 @@ def optimize_ctes(
         REGISTERED_RULES.append(InlineDatasource())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdown())
-
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdownRemove())
     for rule in REGISTERED_RULES:
         loops = 0
         complete = False
         while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
             actions_taken = False
             # assume we go through all CTEs once
-            look_at = [root_cte, *input]
+            look_at = [root_cte, *reversed(input)]
             inverse_map = gen_inverse_map(look_at)
             for cte in look_at:
                 opt = rule.optimize(cte, inverse_map)
                 actions_taken = actions_taken or opt
             complete = not actions_taken
             loops += 1
+            input = reorder_ctes(filter_irrelevant_ctes(input, root_cte))
         logger.info(f"finished checking for {type(rule).__name__} in {loops} loops")
 
-    return filter_irrelevant_ctes(input, root_cte)
+    return reorder_ctes(filter_irrelevant_ctes(input, root_cte))

trilogy/core/optimizations/__init__.py

@@ -1,6 +1,6 @@
 from .inline_constant import InlineConstant
 from .inline_datasource import InlineDatasource
-from .predicate_pushdown import PredicatePushdown
+from .predicate_pushdown import PredicatePushdown, PredicatePushdownRemove
 from .base_optimization import OptimizationRule
 
 __all__ = [
@@ -8,4 +8,5 @@ __all__ = [
     "InlineConstant",
     "InlineDatasource",
     "PredicatePushdown",
+    "PredicatePushdownRemove",
 ]

trilogy/core/optimizations/predicate_pushdown.py

@@ -114,48 +114,6 @@ class PredicatePushdown(OptimizationRule):
         if not cte.condition:
             self.debug(f"No CTE condition for {cte.name}")
             return False
-
-        parent_filter_status = {
-            parent.name: is_child_of(cte.condition, parent.condition)
-            for parent in cte.parent_ctes
-        }
-        # flatten existnce argument tuples to a list
-
-        flattened_existence = [
-            x.address for y in cte.condition.existence_arguments for x in y
-        ]
-
-        existence_only = [
-            parent.name
-            for parent in cte.parent_ctes
-            if all([x.address in flattened_existence for x in parent.output_columns])
-            and len(flattened_existence) > 0
-        ]
-        if all(
-            [
-                value
-                for key, value in parent_filter_status.items()
-                if key not in existence_only
-            ]
-        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
-            self.log(
-                f"All parents of {cte.name} have same filter or are existence only inputs, removing filter from {cte.name}"
-            )
-            cte.condition = None
-            # remove any "parent" CTEs that provided only existence inputs
-            if existence_only:
-                original = [y.name for y in cte.parent_ctes]
-                cte.parent_ctes = [
-                    x for x in cte.parent_ctes if x.name not in existence_only
-                ]
-                self.log(
-                    f"new parents for {cte.name} are {[x.name for x in cte.parent_ctes]}, vs {original}"
-                )
-            return True
-        else:
-            self.log(
-                f"Could not remove filter from {cte.name}, as not all parents have the same filter: {parent_filter_status}"
-            )
         if self.complete.get(cte.name):
             self.debug("Have done this CTE before")
             return False
@@ -197,3 +155,63 @@ class PredicatePushdown(OptimizationRule):
 
         self.complete[cte.name] = True
         return optimized
+
+
+class PredicatePushdownRemove(OptimizationRule):
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.complete: dict[str, bool] = {}
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        optimized = False
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+
+            return False
+
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+
+        parent_filter_status = {
+            parent.name: is_child_of(cte.condition, parent.condition)
+            for parent in cte.parent_ctes
+        }
+        # flatten existnce argument tuples to a list
+
+        flattened_existence = [
+            x.address for y in cte.condition.existence_arguments for x in y
+        ]
+
+        existence_only = [
+            parent.name
+            for parent in cte.parent_ctes
+            if all([x.address in flattened_existence for x in parent.output_columns])
+            and len(flattened_existence) > 0
+        ]
+        if all(
+            [
+                value
+                for key, value in parent_filter_status.items()
+                if key not in existence_only
+            ]
+        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
+            self.log(
+                f"All parents of {cte.name} have same filter or are existence only inputs, removing filter from {cte.name}"
+            )
+            cte.condition = None
+            # remove any "parent" CTEs that provided only existence inputs
+            if existence_only:
+                original = [y.name for y in cte.parent_ctes]
+                cte.parent_ctes = [
+                    x for x in cte.parent_ctes if x.name not in existence_only
+                ]
+                self.log(
+                    f"new parents for {cte.name} are {[x.name for x in cte.parent_ctes]}, vs {original}"
+                )
+            return True
+
+        self.complete[cte.name] = True
+        return optimized

trilogy/core/processing/concept_strategies_v3.py

@@ -180,7 +180,10 @@ def generate_candidates_restrictive(
     local_candidates = [
         x
         for x in list(candidates)
-        if x.address not in exhausted and x.granularity != Granularity.SINGLE_ROW
+        if x.address not in exhausted
+        and x.granularity != Granularity.SINGLE_ROW
+        and x.address not in priority_concept.pseudonyms
+        and priority_concept.address not in x.pseudonyms
     ]
     combos: list[list[Concept]] = []
     grain_check = Grain(components=[*local_candidates]).components_copy
@@ -608,7 +611,7 @@ def _search_concepts(
         if len(stack) == 1:
             output = stack[0]
             logger.info(
-                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)} with output {[x.address for x in output.output_concepts]}"
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)} with output {[x.address for x in output.output_concepts]} and {output.resolve().source_map}"
             )
             return output
 
@@ -658,8 +661,7 @@ def _search_concepts(
                 if x.address not in [y.address for y in mandatory_list]
                 and x not in ex_resolve.grain.components
             ]
-            expanded.output_concepts = mandatory_list
-            expanded.rebuild_cache()
+            expanded.set_output_concepts(mandatory_list)
 
             logger.info(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in mandatory_list]} via concept addition; removing extra {[c.address for c in extra]}"