pytrilogy 0.0.2.12__py3-none-any.whl → 0.0.2.13__py3-none-any.whl

{pytrilogy-0.0.2.12.dist-info → pytrilogy-0.0.2.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.12
+Version: 0.0.2.13
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.12.dist-info → pytrilogy-0.0.2.13.dist-info}/RECORD

@@ -1,14 +1,14 @@
-trilogy/__init__.py,sha256=qXNp3R3OFRd_QUIgMqKJ4RGQoBEMVcm6s2DHSehlVWU,291
+trilogy/__init__.py,sha256=U1mVKIIGVmjHu6kwRMyNhb3buZHWWzbk-c9Hu7FiBGQ,291
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=HRQq4i3cpSEJCywt61QKEzRO1jd4tEPZNSBuxUA_7yg,922
+trilogy/constants.py,sha256=rZJh3fAx3ljxf_QZNECR-devR6QXkYc9mpLCxIWNqB0,960
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
-trilogy/executor.py,sha256=5cRbU4Rj7p1pNV76rfp1pz704Hx_0q8_O8HFURjgXxQ,11016
+trilogy/executor.py,sha256=PZr7IF8wS1Oi2WJGE-B3lp70Y8ue2uuauODw02chjdQ,11175
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=WwQPOLfSdL27qhqXw4JTkMyUNvj57T3Xz9M3JUZzhZ8,5940
+trilogy/core/enums.py,sha256=BRYqy-NgIacCYTJo0B11m5XQWSHq5pfxhoLd8pzA3ho,6025
 trilogy/core/env_processor.py,sha256=l7TAB0LalxjTYJdTlcmFIkLXuyxa9lrenWLeZfa9qw0,2276
 trilogy/core/environment_helpers.py,sha256=1miP4is4FEoci01KSAy2VZVYmlmT5TOCOALBekd2muQ,7211
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
@@ -16,9 +16,9 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=ARJAyBjeS415-54k3G_bx807rkPZonEulMaLRxSP7vU,10371
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=kJcSNv6JX79KBEDTZlIyc1nAsn34fWUQLWZN3y7oTVs,143710
+trilogy/core/models.py,sha256=9-1kYfq4NqAthe50Jgs0WjBnZtEWJw0ReCBfFhVxD0Y,144799
 trilogy/core/optimization.py,sha256=7E-Ol51u6ZAxF56F_bzLxgRO-Hu6Yl1ZbPopZJB2tqk,7533
-trilogy/core/query_processor.py,sha256=Y8C03J9PSyXQoARiMmFomYhnP13L61XjRKNOD7nIops,19520
+trilogy/core/query_processor.py,sha256=JJFBVBmT5QNsJ9rSDKNJBINLm7YW7i5fjAp98H0Wcd8,19281
 trilogy/core/optimizations/__init__.py,sha256=bWQecbeiwiDx9LJnLsa7dkWxdbl2wcnkcTN69JyP8iI,356
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
@@ -29,8 +29,8 @@ trilogy/core/processing/concept_strategies_v3.py,sha256=ae6FmwiKNiEbOU2GhnzggFMh
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
 trilogy/core/processing/utility.py,sha256=QKaZL5yJzGJBWCirgB1cAKgcDOibhyk7ETvHveb3GOE,14604
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
-trilogy/core/processing/node_generators/basic_node.py,sha256=EfCCYleCXVWeoCOUih1VtfUXewg1oyG7EdUMRQOyyMk,3135
-trilogy/core/processing/node_generators/common.py,sha256=lDBRq9X6dQ_xSwXxLLNDq2pW8D-XwAY-ylTJLMugkLw,9525
+trilogy/core/processing/node_generators/basic_node.py,sha256=IHj5jEloUe5yojGRLAzt35FcfHqGviWQdS8ETyvr39Q,3292
+trilogy/core/processing/node_generators/common.py,sha256=WY41zjxSCG13n3mdUHmcE0mUQ0gtWxz4nugBgxHONd8,9327
 trilogy/core/processing/node_generators/filter_node.py,sha256=Ij2WqyOsu-TFxhAcL50PLMGpghsSWXJnWEJ8yTqOwrY,8228
 trilogy/core/processing/node_generators/group_node.py,sha256=Du-9uFXD0M-aHq2MV7v5R3QCrAL0JZBFMW-YQwgb6Bw,3135
 trilogy/core/processing/node_generators/group_to_node.py,sha256=nzITnhaALIT7FMonyo16nNo-kSrLfefa9sZBYecrvkU,2887
@@ -41,10 +41,10 @@ trilogy/core/processing/node_generators/select_node.py,sha256=E8bKOAUpwLwZy1iiaF
 trilogy/core/processing/node_generators/unnest_node.py,sha256=aZeixbOzMtXi7BPahKr9bOkIhTciyD9Klsj0kZ56F6s,2189
 trilogy/core/processing/node_generators/window_node.py,sha256=lFfmEjX_mLB7MuOM6CuKNnks1CabokGImpwhbQzjnkE,3283
 trilogy/core/processing/nodes/__init__.py,sha256=jyduHk96j5fpju72sc8swOiBjR3Md866kt8JZGkp3ZU,4866
-trilogy/core/processing/nodes/base_node.py,sha256=szquAzrIkCTXlVhAVSHt6HSJ7rw3b8lfjeO5eFIcEU8,13067
+trilogy/core/processing/nodes/base_node.py,sha256=P4VyOhbgPgTLaXEIftbVVmmEPviScJJhi9v6hSxjC7M,13155
 trilogy/core/processing/nodes/filter_node.py,sha256=DBOSGFfkiILrZa1BlLv2uxUSkgWtSIKiZplqyKXPjg8,2132
 trilogy/core/processing/nodes/group_node.py,sha256=wE6tgyCUL74v76O8jACDm4oYMov4dAlwzLa5xMYReAA,6294
-trilogy/core/processing/nodes/merge_node.py,sha256=34eEH-denk9kkzD8FcZvxgDSMUB9K5e4lSeNpbqSt7I,14456
+trilogy/core/processing/nodes/merge_node.py,sha256=KOTdYli_T3c5RwMaK_73W5UNtKPA0F8hjDJF9jQx2fs,14491
 trilogy/core/processing/nodes/select_node_v2.py,sha256=QuXNcwgjTRYamOoIooGrp4ie6INcqA9whtC5LZWjD8s,7180
 trilogy/core/processing/nodes/unnest_node.py,sha256=mAmFluzm2yeeiQ6NfIB7BU_8atRGh-UJfPf9ROwbhr8,2152
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
@@ -65,18 +65,18 @@ trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2
 trilogy/hooks/query_debugger.py,sha256=NDChfkPmmW-KINa4TaQmDe_adGiwsKFdGLDSYpbodeU,4282
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=rLF7SFj_qtLh92ox-cHrtVSyjgl1aTaa7qZJdR1RDuA,8182
+trilogy/parsing/common.py,sha256=fa3R0xHXQkU8BUN8quPe7qCzez6qaRbnctkYaLgHLxY,8863
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=upL2pmq34vwNqzOtmb0EB22tbEZ4TTiK46J5qPq-_yw,62841
+trilogy/parsing/parse_engine.py,sha256=yCHd6RVRijNctZJZ3iRJG2263UZL2n8EoFA_-Qfr88E,62966
 trilogy/parsing/render.py,sha256=Gy_6wVYPwYLf35Iota08sbqveuWILtUhI8MYStcvtJM,12174
-trilogy/parsing/trilogy.lark,sha256=QNJnExOdvJyKTrQA4ffh-SGIz7rYd93kf2Ccs0m3cn4,11498
+trilogy/parsing/trilogy.lark,sha256=00j0D77gpTBsyet9WGN4Ir9Nc-YvNwXFr2toRiASb_M,11525
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.12.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.12.dist-info/METADATA,sha256=i7Cd69-1p3XsssoHNy1IqbvtZaNeaKpQErvJp940JMw,7907
-pytrilogy-0.0.2.12.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-pytrilogy-0.0.2.12.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.12.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.12.dist-info/RECORD,,
+pytrilogy-0.0.2.13.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.13.dist-info/METADATA,sha256=AJXWyDVSN9EhYPYJlrkXhMVULqg3HyscbZGDhgVd-FA,7907
+pytrilogy-0.0.2.13.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+pytrilogy-0.0.2.13.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.13.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.13.dist-info/RECORD,,

{pytrilogy-0.0.2.12.dist-info → pytrilogy-0.0.2.13.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.2)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.12"
+__version__ = "0.0.2.13"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -28,6 +28,11 @@ class Optimizations:
     direct_return: bool = True
 
 
+@dataclass
+class Comments:
+    pass
+
+
 # TODO: support loading from environments
 @dataclass
 class Config:

trilogy/core/enums.py

@@ -62,6 +62,8 @@ class Modifier(Enum):
         strval = str(value)
         if strval == "~":
             return Modifier.PARTIAL
+        elif strval == "?":
+            return Modifier.NULLABLE
         return super()._missing_(value=strval.capitalize())
 
 
@@ -273,6 +275,7 @@ class SourceType(Enum):
     CONSTANT = "constant"
     ROWSET = "rowset"
     MERGE = "merge"
+    BASIC = "basic"
 
 
 class ShowCategory(Enum):

trilogy/core/models.py

@@ -2264,6 +2264,7 @@ class QueryDatasource(BaseModel):
     @field_validator("joins")
     @classmethod
     def validate_joins(cls, v):
+        unique_pairs = set()
         for join in v:
             if not isinstance(join, BaseJoin):
                 continue
@@ -2271,7 +2272,16 @@ class QueryDatasource(BaseModel):
                 raise SyntaxError(
                     f"Cannot join a datasource to itself, joining {join.left_datasource}"
                 )
-
+            pairing = "".join(
+                sorted(
+                    [join.left_datasource.identifier, join.right_datasource.identifier]
+                )
+            )
+            if pairing in unique_pairs:
+                raise SyntaxError(
+                    f"Duplicate join {join.left_datasource.identifier} and {join.right_datasource.identifier}"
+                )
+            unique_pairs.add(pairing)
         return v
 
     @field_validator("input_concepts")
@@ -2386,6 +2396,11 @@ class QueryDatasource(BaseModel):
                 final_source_map[key] = other.source_map[key]
         for k, v in final_source_map.items():
             final_source_map[k] = set(merged_datasources[x.full_name] for x in list(v))
+        self_hidden = self.hidden_concepts or []
+        other_hidden = other.hidden_concepts or []
+        hidden = [
+            x for x in self_hidden if x.address in [y.address for y in other_hidden]
+        ]
         qds = QueryDatasource(
             input_concepts=unique(
                 self.input_concepts + other.input_concepts, "address"
@@ -2409,9 +2424,7 @@ class QueryDatasource(BaseModel):
             ),
             join_derived_concepts=self.join_derived_concepts,
             force_group=self.force_group,
-            hidden_concepts=unique(
-                self.hidden_concepts + other.hidden_concepts, "address"
-            ),
+            hidden_concepts=hidden,
         )
 
         return qds
@@ -2557,6 +2570,7 @@ class CTE(BaseModel):
     @property
     def comment(self) -> str:
         base = f"Target: {str(self.grain)}."
+        base += f" Source: {self.source.source_type}."
         if self.parent_ctes:
             base += f" References: {', '.join([x.name for x in self.parent_ctes])}."
         if self.joins:
@@ -2565,6 +2579,11 @@ class CTE(BaseModel):
             base += (
                 f"\n-- Partials: {', '.join([str(x) for x in self.partial_concepts])}."
             )
+        base += f"\n-- Source Map: {self.source_map}."
+        base += f"\n-- Output: {', '.join([str(x) for x in self.output_columns])}."
+        if self.hidden_concepts:
+            base += f"\n-- Hidden: {', '.join([str(x) for x in self.hidden_concepts])}."
+
         return base
 
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
@@ -2632,6 +2651,10 @@ class CTE(BaseModel):
                 f" {self.name} {other.name} conditions {self.condition} {other.condition}"
             )
             raise ValueError(error)
+        mutually_hidden = []
+        for concept in self.hidden_concepts:
+            if concept in other.hidden_concepts:
+                mutually_hidden.append(concept)
         self.partial_concepts = unique(
             self.partial_concepts + other.partial_concepts, "address"
         )
@@ -2654,9 +2677,7 @@ class CTE(BaseModel):
         self.source.output_concepts = unique(
             self.source.output_concepts + other.source.output_concepts, "address"
         )
-        self.hidden_concepts = unique(
-            self.hidden_concepts + other.hidden_concepts, "address"
-        )
+        self.hidden_concepts = mutually_hidden
         self.existence_source_map = {
             **self.existence_source_map,
             **other.existence_source_map,
@@ -3012,6 +3033,9 @@ class EnvironmentDatasourceDict(dict):
     def values(self) -> ValuesView[Datasource]:  # type: ignore
         return super().values()
 
+    def items(self) -> ItemsView[str, Datasource]:  # type: ignore
+        return super().items()
+
 
 class EnvironmentConceptDict(dict):
     def __init__(self, *args, **kwargs) -> None:

trilogy/core/processing/node_generators/basic_node.py

@@ -10,6 +10,7 @@ from trilogy.core.processing.node_generators.common import (
 )
 from trilogy.utility import unique
 from trilogy.constants import logger
+from trilogy.core.enums import SourceType
 from itertools import combinations
 
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
@@ -35,7 +36,11 @@ def gen_basic_node(
     attempts: List[tuple[list[Concept], list[Concept]]] = [
         (parent_concepts, [concept] + local_optional_redundant)
     ]
-    equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
+    equivalent_optional = [
+        x
+        for x in local_optional
+        if x.lineage == concept.lineage and x.address != concept.address
+    ]
     non_equivalent_optional = [
         x for x in local_optional if x not in equivalent_optional
     ]
@@ -61,8 +66,10 @@ def gen_basic_node(
             depth=depth + 1,
             history=history,
         )
+
         if not parent_node:
             continue
+        parent_node.source_type = SourceType.BASIC
         parents: List[StrategyNode] = [parent_node]
         for x in basic_output:
             sources = [p for p in parents if x in p.output_concepts]

trilogy/core/processing/node_generators/common.py

@@ -1,4 +1,4 @@
-from typing import List, Tuple
+from typing import List, Tuple, Callable
 
 
 from trilogy.core.enums import PurposeLineage, Purpose
@@ -96,6 +96,7 @@ def gen_property_enrichment_node(
     g,
     depth: int,
     source_concepts,
+    log_lambda: Callable,
     history: History | None = None,
     conditions: WhereClause | None = None,
 ):
@@ -106,8 +107,8 @@ def gen_property_enrichment_node(
         keys = "-".join([y.address for y in x.keys])
         required_keys[keys].add(x.address)
     final_nodes = []
-    node_joins = []
     for _k, vs in required_keys.items():
+        log_lambda(f"Generating enrichment node for {_k} with {vs}")
         ks = _k.split("-")
         enrich_node: StrategyNode = source_concepts(
             mandatory_list=[environment.concepts[k] for k in ks]
@@ -119,17 +120,6 @@ def gen_property_enrichment_node(
             conditions=conditions,
         )
         final_nodes.append(enrich_node)
-        node_joins.append(
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=base_node,
-                concepts=concept_to_relevant_joins(
-                    [environment.concepts[k] for k in ks]
-                ),
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        )
     return MergeNode(
         input_concepts=unique(
             base_node.output_concepts
@@ -146,9 +136,8 @@ def gen_property_enrichment_node(
         g=g,
         parents=[
             base_node,
-            enrich_node,
-        ],
-        node_joins=node_joins,
+        ]
+        + final_nodes,
     )
 
 
@@ -197,6 +186,7 @@ def gen_enrichment_node(
                 source_concepts,
                 history=history,
                 conditions=conditions,
+                log_lambda=log_lambda,
             )
 
     enrich_node: StrategyNode = source_concepts(  # this fetches the parent + join keys
@@ -216,14 +206,14 @@ def gen_enrichment_node(
     log_lambda(
         f"{str(type(base_node).__name__)} returning merge node with group node + enrichment node"
     )
-
+    non_hidden = [
+        x
+        for x in base_node.output_concepts
+        if x.address not in [y.address for y in base_node.hidden_concepts]
+    ]
     return MergeNode(
-        input_concepts=unique(
-            join_keys + extra_required + base_node.output_concepts, "address"
-        ),
-        output_concepts=unique(
-            join_keys + extra_required + base_node.output_concepts, "address"
-        ),
+        input_concepts=unique(join_keys + extra_required + non_hidden, "address"),
+        output_concepts=unique(join_keys + extra_required + non_hidden, "address"),
         environment=environment,
         g=g,
         parents=[enrich_node, base_node],

trilogy/core/processing/nodes/base_node.py

@@ -61,11 +61,9 @@ def resolve_concept_map(
         for concept in input.output_concepts:
             if concept.address not in input.non_partial_concept_addresses:
                 continue
-
-            if (
-                isinstance(input, QueryDatasource)
-                and concept.address in input.hidden_concepts
-            ):
+            if isinstance(input, QueryDatasource) and concept.address in [
+                x.address for x in input.hidden_concepts
+            ]:
                 continue
             if concept.address in full_addresses:
 
@@ -82,10 +80,9 @@ def resolve_concept_map(
         for concept in input.output_concepts:
             if concept.address not in [t.address for t in inherited_inputs]:
                 continue
-            if (
-                isinstance(input, QueryDatasource)
-                and concept.address in input.hidden_concepts
-            ):
+            if isinstance(input, QueryDatasource) and concept.address in [
+                x.address for x in input.hidden_concepts
+            ]:
                 continue
             if len(concept_map.get(concept.address, [])) == 0:
                 concept_map[concept.address].add(input)
@@ -94,6 +91,7 @@ def resolve_concept_map(
         if target.address not in inherited:
             # an empty source means it is defined in this CTE
             concept_map[target.address] = set()
+
     return concept_map
 
 
@@ -226,7 +224,10 @@ class StrategyNode:
     def remove_output_concepts(self, concepts: List[Concept]):
         for x in concepts:
             self.hidden_concepts.append(x)
-        self.output_concepts = [x for x in self.output_concepts if x not in concepts]
+        addresses = [x.address for x in concepts]
+        self.output_concepts = [
+            x for x in self.output_concepts if x.address not in addresses
+        ]
         self.rebuild_cache()
 
     @property
@@ -257,6 +258,7 @@ class StrategyNode:
             targets=self.output_concepts,
             inherited_inputs=self.input_concepts + self.existence_concepts,
         )
+
         return QueryDatasource(
             input_concepts=self.input_concepts,
             output_concepts=self.output_concepts,

trilogy/core/processing/nodes/merge_node.py

@@ -371,7 +371,7 @@ class MergeNode(StrategyNode):
             conditions=self.conditions,
             hidden_concepts=list(self.hidden_concepts),
             virtual_output_concepts=list(self.virtual_output_concepts),
-            node_joins=self.node_joins,
+            node_joins=list(self.node_joins) if self.node_joins else None,
             join_concepts=list(self.join_concepts) if self.join_concepts else None,
             force_join_type=self.force_join_type,
             existence_concepts=list(self.existence_concepts),

trilogy/core/query_processor.py

@@ -35,7 +35,6 @@ from trilogy.core.ergonomics import CTE_NAMES
 from trilogy.core.optimization import optimize_ctes
 from math import ceil
 from collections import defaultdict
-from random import shuffle
 
 LOGGER_PREFIX = "[QUERY BUILD]"
 
@@ -128,8 +127,6 @@ def generate_source_map(
                 if qdk in output_address:
                     source_map[qdk].append(cte.name)
             # now do a pass that accepts partials
-            # TODO: move this into a second loop by first creationg all sub sources
-            # then loop through this
             for cte in matches:
                 if qdk not in source_map:
                     source_map[qdk] = [cte.name]
@@ -180,7 +177,6 @@ def generate_cte_name(full_name: str, name_map: dict[str, str]) -> str:
             int = ceil(idx / len(CTE_NAMES))
             suffix = f"_{int}"
         valid = [x for x in CTE_NAMES if x + suffix not in name_map.values()]
-        shuffle(valid)
         lookup = valid[0]
         new_name = f"{lookup}{suffix}"
         name_map[full_name] = new_name
@@ -334,12 +330,13 @@ def append_existence_check(
         for subselect in where.existence_arguments:
             if not subselect:
                 continue
-            logger.info(
-                f"{LOGGER_PREFIX} fetching existance clause inputs {[str(c) for c in subselect]}"
-            )
+
             eds = source_query_concepts(
                 [*subselect], environment=environment, g=graph, history=history
             )
+            logger.info(
+                f"{LOGGER_PREFIX} fetching existence clause inputs {[str(c) for c in subselect]}"
+            )
             node.add_parents([eds])
             node.add_existence_concepts([*subselect])
 
@@ -384,9 +381,7 @@ def get_query_node(
     if nest_where and statement.where_clause:
         if not all_aggregate:
             ods.conditions = statement.where_clause.conditional
-        ods.output_concepts = statement.output_components
-        # ods.hidden_concepts = where_delta
-        ods.rebuild_cache()
+        ods.set_output_concepts(statement.output_components)
         append_existence_check(ods, environment, graph, history)
         ds = GroupNode(
             output_concepts=statement.output_components,

trilogy/executor.py

@@ -300,10 +300,16 @@ class Executor(object):
                 self.environment.add_datasource(x.datasource)
             yield x
 
-    def execute_raw_sql(self, command: str) -> CursorResult:
+    def execute_raw_sql(
+        self, command: str, variables: dict | None = None
+    ) -> CursorResult:
         """Run a command against the raw underlying
         execution engine"""
-        return self.connection.execute(text(command))
+        if variables:
+            return self.connection.execute(text(command), variables)
+        return self.connection.execute(
+            text(command),
+        )
 
     def execute_text(self, command: str) -> List[CursorResult]:
         """Run a preql text command"""

trilogy/parsing/common.py

@@ -50,7 +50,7 @@ def process_function_args(
             id_hash = string_to_hash(str(arg))
             concept = function_to_concept(
                 arg,
-                name=f"{VIRTUAL_CONCEPT_PREFIX}_{id_hash}",
+                name=f"{VIRTUAL_CONCEPT_PREFIX}_{arg.operator.value}_{id_hash}",
                 namespace=environment.namespace,
             )
             # to satisfy mypy, concept will always have metadata
@@ -255,20 +255,34 @@ def arbitrary_to_concept(
         | str
     ),
     namespace: str,
-    name: str,
+    name: str | None = None,
     metadata: Metadata | None = None,
     purpose: Purpose | None = None,
 ) -> Concept:
 
     if isinstance(parent, AggregateWrapper):
+        if not name:
+            name = (
+                f"_agg_{parent.function.operator.value}_{string_to_hash(str(parent))}"
+            )
         return agg_wrapper_to_concept(parent, namespace, name, metadata, purpose)
     elif isinstance(parent, WindowItem):
+        if not name:
+            name = f"_window_{parent.type.value}_{string_to_hash(str(parent))}"
         return window_item_to_concept(parent, name, namespace, purpose, metadata)
     elif isinstance(parent, FilterItem):
+        if not name:
+            name = f"_filter_{parent.content.name}_{string_to_hash(str(parent))}"
         return filter_item_to_concept(parent, name, namespace, purpose, metadata)
     elif isinstance(parent, Function):
+        if not name:
+            name = f"_func_{parent.operator.value}_{string_to_hash(str(parent))}"
         return function_to_concept(parent, name, namespace)
     elif isinstance(parent, ListWrapper):
+        if not name:
+            name = f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"
         return constant_to_concept(parent, name, namespace, purpose, metadata)
     else:
+        if not name:
+            name = f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"
         return constant_to_concept(parent, name, namespace, purpose, metadata)

trilogy/parsing/parse_engine.py

@@ -16,7 +16,6 @@ from trilogy.core.internal import INTERNAL_NAMESPACE, ALL_ROWS_CONCEPT
 from trilogy.constants import (
     DEFAULT_NAMESPACE,
     NULL_VALUE,
-    VIRTUAL_CONCEPT_PREFIX,
     MagicConstants,
 )
 from trilogy.core.enums import (
@@ -109,7 +108,6 @@ from trilogy.core.models import (
     HavingClause,
 )
 from trilogy.parsing.exceptions import ParseError
-from trilogy.utility import string_to_hash
 from trilogy.parsing.common import (
     agg_wrapper_to_concept,
     window_item_to_concept,
@@ -739,8 +737,8 @@ class ParseToObjects(Transformer):
                 x = arbitrary_to_concept(
                     x,
                     namespace=namespace,
-                    name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(x))}",
                 )
+                self.environment.add_concept(x)
             return x
 
         return [
@@ -781,6 +779,11 @@ class ParseToObjects(Transformer):
     def rawsql_statement(self, meta: Meta, args) -> RawSQLStatement:
         return RawSQLStatement(meta=Metadata(line_number=meta.line), text=args[0])
 
+    def resolve_import_address(self, address) -> str:
+        with open(address, "r", encoding="utf-8") as f:
+            text = f.read()
+        return text
+
     def import_statement(self, args: list[str]) -> ImportStatement:
         alias = args[-1]
         path = args[0].split(".")
@@ -790,8 +793,7 @@ class ParseToObjects(Transformer):
             nparser = self.parsed[target]
         else:
             try:
-                with open(target, "r", encoding="utf-8") as f:
-                    text = f.read()
+                text = self.resolve_import_address(target)
                 nparser = ParseToObjects(
                     visit_tokens=True,
                     text=text,
@@ -1093,7 +1095,6 @@ class ParseToObjects(Transformer):
             left = arbitrary_to_concept(
                 args[0],
                 namespace=self.environment.namespace,
-                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(args[0]))}",
             )
             self.environment.add_concept(left)
         else:
@@ -1102,7 +1103,6 @@ class ParseToObjects(Transformer):
             right = arbitrary_to_concept(
                 args[2],
                 namespace=self.environment.namespace,
-                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(args[2]))}",
             )
             self.environment.add_concept(right)
         else:
@@ -1137,7 +1137,6 @@ class ParseToObjects(Transformer):
             right = arbitrary_to_concept(
                 right,
                 namespace=self.environment.namespace,
-                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(right))}",
             )
             self.environment.add_concept(right, meta=meta)
         return SubselectComparison(
@@ -1186,8 +1185,9 @@ class ParseToObjects(Transformer):
     def window_item_order(self, args):
         return WindowItemOrder(contents=args[0])
 
-    def window_item(self, args) -> WindowItem:
-        type = args[0]
+    @v_args(meta=True)
+    def window_item(self, meta, args) -> WindowItem:
+        type: WindowType = args[0]
         order_by = []
         over = []
         index = None
@@ -1203,6 +1203,14 @@ class ParseToObjects(Transformer):
                 concept = self.environment.concepts[item]
             elif isinstance(item, Concept):
                 concept = item
+            elif isinstance(item, WindowType):
+                type = item
+            else:
+                concept = arbitrary_to_concept(
+                    item,
+                    namespace=self.environment.namespace,
+                )
+                self.environment.add_concept(concept, meta=meta)
         assert concept
         return WindowItem(
             type=type, content=concept, over=over, order_by=order_by, index=index

trilogy/parsing/trilogy.lark

@@ -90,9 +90,9 @@
 
 
     // rank/lag/lead
-    WINDOW_TYPE: ("row_number"i|"rank"i|"lag"i|"lead"i | "sum"i)  /[\s]+/
+    WINDOW_TYPE: ("row_number"i|"rank"i|"lag"i|"lead"i | "sum"i | "avg"i | "max"i | "min"i )  /[\s]+/
     
-    window_item: WINDOW_TYPE int_lit? concept_lit  window_item_over? window_item_order?
+    window_item: WINDOW_TYPE int_lit? expr  window_item_over? window_item_order?
     
     window_item_over: ("OVER"i over_list)
     
@@ -296,7 +296,7 @@
 
     MODIFIER: "Optional"i | "Partial"i | "Nullable"i
 
-    SHORTHAND_MODIFIER: "~"
+    SHORTHAND_MODIFIER: "~" | "?"
 
     struct_type: "struct"i "<" ((data_type | IDENTIFIER) ",")* (data_type | IDENTIFIER) ","? ">"