pytrilogy 0.0.2.11__py3-none-any.whl → 0.0.2.12__py3-none-any.whl

{pytrilogy-0.0.2.11.dist-info → pytrilogy-0.0.2.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.11
+Version: 0.0.2.12
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.11.dist-info → pytrilogy-0.0.2.12.dist-info}/RECORD

@@ -1,4 +1,4 @@
-trilogy/__init__.py,sha256=rSlG7hlvpZjtLdU_7j6ldvO5bmiafhgPB40sdcOu3j0,291
+trilogy/__init__.py,sha256=qXNp3R3OFRd_QUIgMqKJ4RGQoBEMVcm6s2DHSehlVWU,291
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=HRQq4i3cpSEJCywt61QKEzRO1jd4tEPZNSBuxUA_7yg,922
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
@@ -8,50 +8,50 @@ trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=D5HUKiknRcx_OBoX21IAM0JFQ-vQNBgad2GOvJfXi3o,5964
+trilogy/core/enums.py,sha256=WwQPOLfSdL27qhqXw4JTkMyUNvj57T3Xz9M3JUZzhZ8,5940
 trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
-trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
+trilogy/core/environment_helpers.py,sha256=1miP4is4FEoci01KSAy2VZVYmlmT5TOCOALBekd2muQ,7211
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
 trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
 trilogy/core/functions.py,sha256=ARJAyBjeS415-54k3G_bx807rkPZonEulMaLRxSP7vU,10371
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=rIeYw173UjfTfc7194y-jqv7u51Uy6LufwbiwKI2Cgs,143834
-trilogy/core/optimization.py,sha256=RJmDr2f9hyFePF-B6LpmHkj69-AzUm-nNtvO59899O8,6601
-trilogy/core/query_processor.py,sha256=50FHJ9Rw89ZNBflZK8R9jn7O_WdDHplS1hZSNXCozN8,19352
-trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
+trilogy/core/models.py,sha256=kJcSNv6JX79KBEDTZlIyc1nAsn34fWUQLWZN3y7oTVs,143710
+trilogy/core/optimization.py,sha256=7E-Ol51u6ZAxF56F_bzLxgRO-Hu6Yl1ZbPopZJB2tqk,7533
+trilogy/core/query_processor.py,sha256=Y8C03J9PSyXQoARiMmFomYhnP13L61XjRKNOD7nIops,19520
+trilogy/core/optimizations/__init__.py,sha256=bWQecbeiwiDx9LJnLsa7dkWxdbl2wcnkcTN69JyP8iI,356
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
 trilogy/core/optimizations/inline_datasource.py,sha256=AATzQ6YrtW_1-aQFjQyTYqEYKBoMFhek7ADfBr4uUdQ,3634
-trilogy/core/optimizations/predicate_pushdown.py,sha256=4Y6zfJN3VVexkD6p9IYWN4BTL8RwW6hyNb0VQZ8ETdI,8020
+trilogy/core/optimizations/predicate_pushdown.py,sha256=3hSS1i1itR5lEmLwebuhz4FiPIyFjr2pBeydZfHHPSk,8433
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=5yZQRWlEyNR2sddq7kx12_PRUNhekEu6bWO2yIvjjUA,25511
+trilogy/core/processing/concept_strategies_v3.py,sha256=ae6FmwiKNiEbOU2GhnzggFMh82MhqxBj9bgr0ituT2w,25633
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=15Qi68ktmnQmMt5jxrpueQ34oG5fSSgst3kIEejwF8A,14519
+trilogy/core/processing/utility.py,sha256=QKaZL5yJzGJBWCirgB1cAKgcDOibhyk7ETvHveb3GOE,14604
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
-trilogy/core/processing/node_generators/basic_node.py,sha256=4242PNGTCm2tklqMIkqVu5Iv4m_IeTnOYXxDveuCDZM,2856
+trilogy/core/processing/node_generators/basic_node.py,sha256=EfCCYleCXVWeoCOUih1VtfUXewg1oyG7EdUMRQOyyMk,3135
 trilogy/core/processing/node_generators/common.py,sha256=lDBRq9X6dQ_xSwXxLLNDq2pW8D-XwAY-ylTJLMugkLw,9525
 trilogy/core/processing/node_generators/filter_node.py,sha256=Ij2WqyOsu-TFxhAcL50PLMGpghsSWXJnWEJ8yTqOwrY,8228
 trilogy/core/processing/node_generators/group_node.py,sha256=Du-9uFXD0M-aHq2MV7v5R3QCrAL0JZBFMW-YQwgb6Bw,3135
 trilogy/core/processing/node_generators/group_to_node.py,sha256=nzITnhaALIT7FMonyo16nNo-kSrLfefa9sZBYecrvkU,2887
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=qS8VKppi10NKfqNb7BXTKyLBBn4wJuMmFYhrtMUkYb8,12580
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=D_jsnfoLMrQc08_JvT0wEDvjyzJAxBpdcZFyDN-feV0,13192
 trilogy/core/processing/node_generators/rowset_node.py,sha256=6KVnuk75mRzWJ-jIk7e8azN8BIPPuCn-VxPlxDqfPVE,4616
 trilogy/core/processing/node_generators/select_node.py,sha256=E8bKOAUpwLwZy1iiaFVD5sM4XK-eFpHgijdyIWLMyH4,18904
-trilogy/core/processing/node_generators/unnest_node.py,sha256=gHjurhr86JFkbq6vxTHDQGDKt95EdotSFHVKgrR_Z3Q,1860
+trilogy/core/processing/node_generators/unnest_node.py,sha256=aZeixbOzMtXi7BPahKr9bOkIhTciyD9Klsj0kZ56F6s,2189
 trilogy/core/processing/node_generators/window_node.py,sha256=lFfmEjX_mLB7MuOM6CuKNnks1CabokGImpwhbQzjnkE,3283
 trilogy/core/processing/nodes/__init__.py,sha256=jyduHk96j5fpju72sc8swOiBjR3Md866kt8JZGkp3ZU,4866
-trilogy/core/processing/nodes/base_node.py,sha256=KVLAPLlo5QQq9Cm6lTLsqLN7l8EyXGxQkKrZFHdyRp8,12871
+trilogy/core/processing/nodes/base_node.py,sha256=szquAzrIkCTXlVhAVSHt6HSJ7rw3b8lfjeO5eFIcEU8,13067
 trilogy/core/processing/nodes/filter_node.py,sha256=DBOSGFfkiILrZa1BlLv2uxUSkgWtSIKiZplqyKXPjg8,2132
-trilogy/core/processing/nodes/group_node.py,sha256=RKqODYxbY8wmClvI8-k3UgXWL7qQv5Hxhz5AB-RVUu0,6344
-trilogy/core/processing/nodes/merge_node.py,sha256=ZUcFIm5v4hip_Ml_oJXqVMrevUkE62wvPrvCrO9FWAc,14272
+trilogy/core/processing/nodes/group_node.py,sha256=wE6tgyCUL74v76O8jACDm4oYMov4dAlwzLa5xMYReAA,6294
+trilogy/core/processing/nodes/merge_node.py,sha256=34eEH-denk9kkzD8FcZvxgDSMUB9K5e4lSeNpbqSt7I,14456
 trilogy/core/processing/nodes/select_node_v2.py,sha256=QuXNcwgjTRYamOoIooGrp4ie6INcqA9whtC5LZWjD8s,7180
-trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
+trilogy/core/processing/nodes/unnest_node.py,sha256=mAmFluzm2yeeiQ6NfIB7BU_8atRGh-UJfPf9ROwbhr8,2152
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=xbZXcWvwdRadyb77J2qat67R_gRAmKeeeI4JRH4P928,30726
+trilogy/dialect/base.py,sha256=kQek_ufZC9HDVOKlWasvIx6xyew8wv3JNIU6r53_IR4,32842
 trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
-trilogy/dialect/common.py,sha256=HVNPL8dGyQjT2REruV5C2YPpBbVR7KlD0akVM7GcuPI,3329
+trilogy/dialect/common.py,sha256=LnOtsq4vUTTKB5QUk594QSoNDfOoOF08KQJZZamou84,3359
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
 trilogy/dialect/duckdb.py,sha256=u_gpL35kouWxoBLas1h0ABYY2QzlVtEh22hm5h0lCOM,3182
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
@@ -65,18 +65,18 @@ trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2
 trilogy/hooks/query_debugger.py,sha256=NDChfkPmmW-KINa4TaQmDe_adGiwsKFdGLDSYpbodeU,4282
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=zNd5buKxK4z9WSszOk4zOI0GexDFukzZtdNfQxg3kVw,6218
+trilogy/parsing/common.py,sha256=rLF7SFj_qtLh92ox-cHrtVSyjgl1aTaa7qZJdR1RDuA,8182
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=riXOhMqYGvzyy1aeTZTjPIz3jOszue2VTEH7hn59zD4,63808
+trilogy/parsing/parse_engine.py,sha256=upL2pmq34vwNqzOtmb0EB22tbEZ4TTiK46J5qPq-_yw,62841
 trilogy/parsing/render.py,sha256=Gy_6wVYPwYLf35Iota08sbqveuWILtUhI8MYStcvtJM,12174
 trilogy/parsing/trilogy.lark,sha256=QNJnExOdvJyKTrQA4ffh-SGIz7rYd93kf2Ccs0m3cn4,11498
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.11.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.11.dist-info/METADATA,sha256=mZ6V2gwihh9nv-bIGkimK2-bKNvsY_3JIUSNzP_m2Wg,7907
-pytrilogy-0.0.2.11.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-pytrilogy-0.0.2.11.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.11.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.11.dist-info/RECORD,,
+pytrilogy-0.0.2.12.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.12.dist-info/METADATA,sha256=i7Cd69-1p3XsssoHNy1IqbvtZaNeaKpQErvJp940JMw,7907
+pytrilogy-0.0.2.12.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+pytrilogy-0.0.2.12.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.12.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.12.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.11"
+__version__ = "0.0.2.12"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/enums.py

@@ -229,7 +229,6 @@ class ComparisonOperator(Enum):
     ILIKE = "ilike"
     CONTAINS = "contains"
     ELSE = "else"
-    BETWEEN = "between"
 
     @classmethod
     def _missing_(cls, value):

trilogy/core/environment_helpers.py

@@ -1,6 +1,15 @@
-from trilogy.core.models import DataType, Concept, Environment, Function, Metadata
+from trilogy.core.models import (
+    DataType,
+    Concept,
+    Environment,
+    Function,
+    Metadata,
+    StructType,
+)
+from trilogy.core.functions import AttrAccess
 from trilogy.core.enums import Purpose, FunctionType, ConceptSource
 from trilogy.constants import DEFAULT_NAMESPACE
+from trilogy.parsing.common import process_function_args, arg_to_datatype, Meta
 
 
 def generate_date_concepts(concept: Concept, environment: Environment):
@@ -142,15 +151,44 @@ def generate_key_concepts(concept: Concept, environment: Environment):
         environment.add_concept(new_concept, add_derived=False)
 
 
-def generate_related_concepts(concept: Concept, environment: Environment):
+def generate_related_concepts(
+    concept: Concept,
+    environment: Environment,
+    meta: Meta | None = None,
+    add_derived: bool = False,
+):
     """Auto populate common derived concepts on types"""
-    if concept.purpose == Purpose.KEY:
+    if concept.purpose == Purpose.KEY and add_derived:
         generate_key_concepts(concept, environment)
-    if concept.datatype == DataType.DATE:
+
+    # datatype types
+    if concept.datatype == DataType.DATE and add_derived:
         generate_date_concepts(concept, environment)
-    elif concept.datatype == DataType.DATETIME:
+    elif concept.datatype == DataType.DATETIME and add_derived:
         generate_date_concepts(concept, environment)
         generate_datetime_concepts(concept, environment)
-    elif concept.datatype == DataType.TIMESTAMP:
+    elif concept.datatype == DataType.TIMESTAMP and add_derived:
         generate_date_concepts(concept, environment)
         generate_datetime_concepts(concept, environment)
+
+    if isinstance(concept.datatype, StructType):
+        for key, value in concept.datatype.fields_map.items():
+            args = process_function_args(
+                [concept, key], meta=meta, environment=environment
+            )
+            auto = Concept(
+                name=key,
+                datatype=arg_to_datatype(value),
+                purpose=Purpose.PROPERTY,
+                namespace=(
+                    environment.namespace + "." + concept.name
+                    if environment.namespace
+                    and environment.namespace != DEFAULT_NAMESPACE
+                    else concept.name
+                ),
+                lineage=AttrAccess(args),
+            )
+            environment.add_concept(auto, meta=meta)
+            if isinstance(value, Concept):
+                environment.merge_concept(auto, value, modifiers=[])
+                assert value.pseudonyms is not None

trilogy/core/models.py

@@ -300,7 +300,7 @@ class MapType(BaseModel):
 
 class StructType(BaseModel):
     fields: List[ALL_TYPES]
-    fields_map: Dict[str, Concept | int | float | str] = Field(default_factory=dict)
+    fields_map: Dict[str, Concept | int | float | str]
 
     @property
     def data_type(self):
@@ -2119,16 +2119,19 @@ class Datasource(Namespaced, BaseModel):
 
 
 class UnnestJoin(BaseModel):
-    concept: Concept
+    concepts: list[Concept]
+    parent: Function
     alias: str = "unnest"
     rendering_required: bool = True
 
     def __hash__(self):
-        return (self.alias + self.concept.address).__hash__()
+        return (
+            self.alias + "".join([str(s.address) for s in self.concepts])
+        ).__hash__()
 
 
 class InstantiatedUnnestJoin(BaseModel):
-    concept: Concept
+    concept_to_unnest: Concept
     alias: str = "unnest"
 
 
@@ -2268,6 +2271,7 @@ class QueryDatasource(BaseModel):
                 raise SyntaxError(
                     f"Cannot join a datasource to itself, joining {join.left_datasource}"
                 )
+
         return v
 
     @field_validator("input_concepts")
@@ -2287,8 +2291,13 @@ class QueryDatasource(BaseModel):
         for key in ("input_concepts", "output_concepts"):
             if not values.get(key):
                 continue
+            concept: Concept
             for concept in values[key]:
-                if concept.address not in v and CONFIG.validate_missing:
+                if (
+                    concept.address not in v
+                    and not any(x in v for x in concept.pseudonyms)
+                    and CONFIG.validate_missing
+                ):
                     raise SyntaxError(
                         f"Missing source map for {concept.address} on {key}, have {v}"
                     )
@@ -2533,7 +2542,7 @@ class CTE(BaseModel):
                     )
                 ]
                 for join in self.joins:
-                    if isinstance(join, UnnestJoin) and join.concept == concept:
+                    if isinstance(join, UnnestJoin) and concept in join.concepts:
                         join.rendering_required = False
 
                 self.parent_ctes = [
@@ -2996,8 +3005,8 @@ class EnvironmentDatasourceDict(dict):
         except KeyError:
             if DEFAULT_NAMESPACE + "." + key in self:
                 return self.__getitem__(DEFAULT_NAMESPACE + "." + key)
-            if "." in key and key.split(".")[0] == DEFAULT_NAMESPACE:
-                return self.__getitem__(key.split(".")[1])
+            if "." in key and key.split(".", 1)[0] == DEFAULT_NAMESPACE:
+                return self.__getitem__(key.split(".", 1)[1])
             raise
 
     def values(self) -> ValuesView[Datasource]:  # type: ignore
@@ -3027,8 +3036,8 @@ class EnvironmentConceptDict(dict):
             return super(EnvironmentConceptDict, self).__getitem__(key)
 
         except KeyError:
-            if "." in key and key.split(".")[0] == DEFAULT_NAMESPACE:
-                return self.__getitem__(key.split(".")[1], line_no)
+            if "." in key and key.split(".", 1)[0] == DEFAULT_NAMESPACE:
+                return self.__getitem__(key.split(".", 1)[1], line_no)
             if DEFAULT_NAMESPACE + "." + key in self:
                 return self.__getitem__(DEFAULT_NAMESPACE + "." + key, line_no)
             if not self.fail_on_missing:
@@ -3293,10 +3302,9 @@ class Environment(BaseModel):
             self.concepts[concept.name] = concept
         else:
             self.concepts[concept.address] = concept
-        if add_derived:
-            from trilogy.core.environment_helpers import generate_related_concepts
+        from trilogy.core.environment_helpers import generate_related_concepts
 
-            generate_related_concepts(concept, self)
+        generate_related_concepts(concept, self, meta=meta, add_derived=add_derived)
         self.gen_concept_list_caches()
         return concept
 
@@ -3423,14 +3431,6 @@ class Comparison(
             raise SyntaxError(
                 f"Cannot compare {self.left} and {self.right} of different types"
             )
-        if self.operator == ComparisonOperator.BETWEEN:
-            if (
-                not isinstance(self.right, ComparisonOperator)
-                and self.right.operator == BooleanOperator.AND
-            ):
-                raise SyntaxError(
-                    f"Between operator must have two operands with and, not {self.right}"
-                )
 
     def __add__(self, other):
         if other is None:

trilogy/core/optimization.py

@@ -10,6 +10,7 @@ from trilogy.core.optimizations import (
     OptimizationRule,
     InlineConstant,
     PredicatePushdown,
+    PredicatePushdownRemove,
     InlineDatasource,
 )
 
@@ -34,6 +35,31 @@ MAX_OPTIMIZATION_LOOPS = 100
 #     return parent
 
 
+def reorder_ctes(
+    input: list[CTE],
+):
+    import networkx as nx
+
+    # Create a directed graph
+    G = nx.DiGraph()
+    mapping: dict[str, CTE] = {}
+    for cte in input:
+        mapping[cte.name] = cte
+        for parent in cte.parent_ctes:
+            G.add_edge(parent.name, cte.name)
+    # Perform topological sort (only works for DAGs)
+    try:
+        topological_order = list(nx.topological_sort(G))
+        if not topological_order:
+            return input
+        return [mapping[x] for x in topological_order]
+    except nx.NetworkXUnfeasible as e:
+        print(
+            "The graph is not a DAG (contains cycles) and cannot be topologically sorted."
+        )
+        raise e
+
+
 def filter_irrelevant_ctes(
     input: list[CTE],
     root_cte: CTE,
@@ -169,20 +195,22 @@ def optimize_ctes(
         REGISTERED_RULES.append(InlineDatasource())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdown())
-
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdownRemove())
     for rule in REGISTERED_RULES:
         loops = 0
         complete = False
         while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
             actions_taken = False
             # assume we go through all CTEs once
-            look_at = [root_cte, *input]
+            look_at = [root_cte, *reversed(input)]
             inverse_map = gen_inverse_map(look_at)
             for cte in look_at:
                 opt = rule.optimize(cte, inverse_map)
                 actions_taken = actions_taken or opt
             complete = not actions_taken
             loops += 1
+            input = reorder_ctes(filter_irrelevant_ctes(input, root_cte))
         logger.info(f"finished checking for {type(rule).__name__} in {loops} loops")
 
-    return filter_irrelevant_ctes(input, root_cte)
+    return reorder_ctes(filter_irrelevant_ctes(input, root_cte))

trilogy/core/optimizations/__init__.py

@@ -1,6 +1,6 @@
 from .inline_constant import InlineConstant
 from .inline_datasource import InlineDatasource
-from .predicate_pushdown import PredicatePushdown
+from .predicate_pushdown import PredicatePushdown, PredicatePushdownRemove
 from .base_optimization import OptimizationRule
 
 __all__ = [
@@ -8,4 +8,5 @@ __all__ = [
     "InlineConstant",
     "InlineDatasource",
     "PredicatePushdown",
+    "PredicatePushdownRemove",
 ]

trilogy/core/optimizations/predicate_pushdown.py

@@ -114,48 +114,6 @@ class PredicatePushdown(OptimizationRule):
         if not cte.condition:
             self.debug(f"No CTE condition for {cte.name}")
             return False
-
-        parent_filter_status = {
-            parent.name: is_child_of(cte.condition, parent.condition)
-            for parent in cte.parent_ctes
-        }
-        # flatten existnce argument tuples to a list
-
-        flattened_existence = [
-            x.address for y in cte.condition.existence_arguments for x in y
-        ]
-
-        existence_only = [
-            parent.name
-            for parent in cte.parent_ctes
-            if all([x.address in flattened_existence for x in parent.output_columns])
-            and len(flattened_existence) > 0
-        ]
-        if all(
-            [
-                value
-                for key, value in parent_filter_status.items()
-                if key not in existence_only
-            ]
-        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
-            self.log(
-                f"All parents of {cte.name} have same filter or are existence only inputs, removing filter from {cte.name}"
-            )
-            cte.condition = None
-            # remove any "parent" CTEs that provided only existence inputs
-            if existence_only:
-                original = [y.name for y in cte.parent_ctes]
-                cte.parent_ctes = [
-                    x for x in cte.parent_ctes if x.name not in existence_only
-                ]
-                self.log(
-                    f"new parents for {cte.name} are {[x.name for x in cte.parent_ctes]}, vs {original}"
-                )
-            return True
-        else:
-            self.log(
-                f"Could not remove filter from {cte.name}, as not all parents have the same filter: {parent_filter_status}"
-            )
         if self.complete.get(cte.name):
             self.debug("Have done this CTE before")
             return False
@@ -197,3 +155,63 @@ class PredicatePushdown(OptimizationRule):
 
         self.complete[cte.name] = True
         return optimized
+
+
+class PredicatePushdownRemove(OptimizationRule):
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.complete: dict[str, bool] = {}
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        optimized = False
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+
+            return False
+
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+
+        parent_filter_status = {
+            parent.name: is_child_of(cte.condition, parent.condition)
+            for parent in cte.parent_ctes
+        }
+        # flatten existnce argument tuples to a list
+
+        flattened_existence = [
+            x.address for y in cte.condition.existence_arguments for x in y
+        ]
+
+        existence_only = [
+            parent.name
+            for parent in cte.parent_ctes
+            if all([x.address in flattened_existence for x in parent.output_columns])
+            and len(flattened_existence) > 0
+        ]
+        if all(
+            [
+                value
+                for key, value in parent_filter_status.items()
+                if key not in existence_only
+            ]
+        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
+            self.log(
+                f"All parents of {cte.name} have same filter or are existence only inputs, removing filter from {cte.name}"
+            )
+            cte.condition = None
+            # remove any "parent" CTEs that provided only existence inputs
+            if existence_only:
+                original = [y.name for y in cte.parent_ctes]
+                cte.parent_ctes = [
+                    x for x in cte.parent_ctes if x.name not in existence_only
+                ]
+                self.log(
+                    f"new parents for {cte.name} are {[x.name for x in cte.parent_ctes]}, vs {original}"
+                )
+            return True
+
+        self.complete[cte.name] = True
+        return optimized

trilogy/core/processing/concept_strategies_v3.py

@@ -180,7 +180,10 @@ def generate_candidates_restrictive(
     local_candidates = [
         x
         for x in list(candidates)
-        if x.address not in exhausted and x.granularity != Granularity.SINGLE_ROW
+        if x.address not in exhausted
+        and x.granularity != Granularity.SINGLE_ROW
+        and x.address not in priority_concept.pseudonyms
+        and priority_concept.address not in x.pseudonyms
     ]
     combos: list[list[Concept]] = []
     grain_check = Grain(components=[*local_candidates]).components_copy
@@ -608,7 +611,7 @@ def _search_concepts(
         if len(stack) == 1:
             output = stack[0]
             logger.info(
-                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)} with output {[x.address for x in output.output_concepts]}"
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)} with output {[x.address for x in output.output_concepts]} and {output.resolve().source_map}"
             )
             return output
 
@@ -658,8 +661,7 @@ def _search_concepts(
                 if x.address not in [y.address for y in mandatory_list]
                 and x not in ex_resolve.grain.components
             ]
-            expanded.output_concepts = mandatory_list
-            expanded.rebuild_cache()
+            expanded.set_output_concepts(mandatory_list)
 
             logger.info(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in mandatory_list]} via concept addition; removing extra {[c.address for c in extra]}"

trilogy/core/processing/node_generators/basic_node.py

@@ -10,6 +10,7 @@ from trilogy.core.processing.node_generators.common import (
 )
 from trilogy.utility import unique
 from trilogy.constants import logger
+from itertools import combinations
 
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
 
@@ -31,12 +32,17 @@ def gen_basic_node(
     )
 
     local_optional_redundant = [x for x in local_optional if x in parent_concepts]
-    attempts = [(parent_concepts, [concept] + local_optional_redundant)]
-    from itertools import combinations
+    attempts: List[tuple[list[Concept], list[Concept]]] = [
+        (parent_concepts, [concept] + local_optional_redundant)
+    ]
+    equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
+    non_equivalent_optional = [
+        x for x in local_optional if x not in equivalent_optional
+    ]
 
     if local_optional:
-        for combo in range(1, len(local_optional) + 1):
-            combos = combinations(local_optional, combo)
+        for combo in range(1, len(non_equivalent_optional) + 1):
+            combos = combinations(non_equivalent_optional, combo)
             for optional_set in combos:
                 attempts.append(
                     (
@@ -64,13 +70,10 @@ def gen_basic_node(
                 continue
             if all(x in source.partial_concepts for source in sources):
                 partials.append(x)
-        outputs = parent_node.output_concepts + [concept]
-        logger.info(
-            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in outputs]}"
-        )
-        # parents.resolve()
 
         parent_node.add_output_concept(concept)
+        for x in equivalent_optional:
+            parent_node.add_output_concept(x)
 
         parent_node.remove_output_concepts(
             [
@@ -79,6 +82,9 @@ def gen_basic_node(
                 if x.address not in [y.address for y in basic_output]
             ]
         )
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in parent_node.output_concepts]}"
+        )
         return parent_node
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"

trilogy/core/processing/node_generators/node_merge_node.py

@@ -86,7 +86,7 @@ def determine_induced_minimal_nodes(
 
     for node in G.nodes:
         if concepts.get(node):
-            lookup = concepts[node]
+            lookup: Concept = concepts[node]
             if lookup.derivation not in (PurposeLineage.BASIC, PurposeLineage.ROOT):
                 nodes_to_remove.append(node)
             elif lookup.derivation == PurposeLineage.BASIC and G.out_degree(node) == 0:
@@ -155,6 +155,26 @@ def detect_ambiguity_and_raise(all_concepts, reduced_concept_sets) -> None:
         )
 
 
+def has_synonym(concept: Concept, others: list[list[Concept]]) -> bool:
+    return any(
+        c.address in concept.pseudonyms or concept.address in c.pseudonyms
+        for sublist in others
+        for c in sublist
+    )
+
+
+def filter_relevant_subgraphs(subgraphs: list[list[Concept]]) -> list[list[Concept]]:
+    return [
+        subgraph
+        for subgraph in subgraphs
+        if len(subgraph) > 1
+        or (
+            len(subgraph) == 1
+            and not has_synonym(subgraph[0], [x for x in subgraphs if x != subgraph])
+        )
+    ]
+
+
 def resolve_weak_components(
     all_concepts: List[Concept],
     environment: Environment,
@@ -249,6 +269,7 @@ def resolve_weak_components(
             continue
         subgraphs.append(sub_component)
     return subgraphs
+    # return filter_relevant_subgraphs(subgraphs)
 
 
 def subgraphs_to_merge_node(