pytrilogy 0.0.3.45__py3-none-any.whl → 0.0.3.47__py3-none-any.whl

{pytrilogy-0.0.3.45.dist-info → pytrilogy-0.0.3.47.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.45
+Version: 0.0.3.47
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.45.dist-info → pytrilogy-0.0.3.47.dist-info}/RECORD

@@ -1,5 +1,5 @@
-pytrilogy-0.0.3.45.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=CExOPBg5voukcSp-h8dQa4Sf1R0iZ0oQBnfggESqllo,303
+pytrilogy-0.0.3.47.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=SPFHWIHKOsEgt2qXg_bEpYrsQE1oX5pNjD1gj43FZW4,303
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=5eQxk1A0pv-TQk3CCvgZCFA9_K-6nxrOm7E5Lxd7KIY,1652
 trilogy/engine.py,sha256=OK2RuqCIUId6yZ5hfF8J1nxGP0AJqHRZiafcowmW0xc,1728
@@ -13,17 +13,17 @@ trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=7XaCpZn5mQmjTobbeBn56SzPWq9eMNDfzfsRU-fP0VE,171
 trilogy/core/enums.py,sha256=JwbWyAHOC2xRTZe2SeEvlIGPvmC1KjcJ4uh1Po5USzQ,7380
 trilogy/core/env_processor.py,sha256=pFsxnluKIusGKx1z7tTnfsd_xZcPy9pZDungkjkyvI0,3170
-trilogy/core/environment_helpers.py,sha256=UWtF5ZQqFyzHdrjUBEd7c2ZfASBhBWFoa9WkUHBbyHI,9700
+trilogy/core/environment_helpers.py,sha256=VvPIiFemqaLLpIpLIqprfu63K7muZ1YzNg7UZIUph8w,8267
 trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
 trilogy/core/exceptions.py,sha256=JPYyBcit3T_pRtlHdtKSeVJkIyWUTozW2aaut25A2xI,673
 trilogy/core/functions.py,sha256=4fEOGgXWDvgrJtCg_5m2Y9iWnHfLbvLQ82RkIMl_1K0,27722
 trilogy/core/graph_models.py,sha256=z17EoO8oky2QOuO6E2aMWoVNKEVJFhLdsQZOhC4fNLU,2079
 trilogy/core/internal.py,sha256=iicDBlC6nM8d7e7jqzf_ZOmpUsW8yrr2AA8AqEiLx-s,1577
 trilogy/core/optimization.py,sha256=aihzx4-2-mSjx5td1TDTYGvc7e9Zvy-_xEyhPqLS-Ig,8314
-trilogy/core/query_processor.py,sha256=Vl-u0F0rbqI2liv82yJgiZCB255Kx_KiuzZVHL6aeTM,19459
+trilogy/core/query_processor.py,sha256=FFZlTzF9DBDH7dvpTDgd5SxDkE4EkING-MVtUgqx9gQ,19459
 trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/models/author.py,sha256=b5GQc79w-gFQfZhgBdHeRCJAtCYr4j_da6k3Dkx4YAA,76863
-trilogy/core/models/build.py,sha256=EsI7BLmFXdxj1an3NnKR_Qm79tcjlFKjmLjmt3_v2eA,61829
+trilogy/core/models/author.py,sha256=NhTKuk1eYAuYBbpvaFUxr-LntIoVarFQlNuNJwZmMmw,76990
+trilogy/core/models/build.py,sha256=fFPGyHIPzQU8DNOxkGQGC3_sbZt-MHP0o5ftSA67LtU,61962
 trilogy/core/models/build_environment.py,sha256=s_C9xAHuD3yZ26T15pWVBvoqvlp2LdZ8yjsv2_HdXLk,5363
 trilogy/core/models/core.py,sha256=wx6hJcFECMG-Ij972ADNkr-3nFXkYESr82ObPiC46_U,10875
 trilogy/core/models/datasource.py,sha256=6RjJUd2u4nYmEwFBpJlM9LbHVYDv8iHJxqiBMZqUrwI,9422
@@ -41,7 +41,7 @@ trilogy/core/processing/utility.py,sha256=rfzdgl-vWkCyhLzXNNuWgPLK59eiYypQb6TdZK
 trilogy/core/processing/node_generators/__init__.py,sha256=o8rOFHPSo-s_59hREwXMW6gjUJCsiXumdbJNozHUf-Y,800
 trilogy/core/processing/node_generators/basic_node.py,sha256=UVsXMn6jTjm_ofVFt218jAS11s4RV4zD781vP4im-GI,3371
 trilogy/core/processing/node_generators/common.py,sha256=nVeH_AdO58ygtNSO0wNgMR7_h2D0dFSGM_rh1fJd4Yc,9468
-trilogy/core/processing/node_generators/filter_node.py,sha256=lT167yBgy3P9sDBM1Cjj0PKSXro8dvGtBmc8nwsUjig,8366
+trilogy/core/processing/node_generators/filter_node.py,sha256=JymSKzA-9oQAZ3ZtJRK9c3w5FXs8MjJBGWU9TYUqx4E,9099
 trilogy/core/processing/node_generators/group_node.py,sha256=kO-ersxIL04rZwX5-vFIFQQnp357PFo_7ZKXoGq3wyc,5989
 trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
@@ -55,7 +55,7 @@ trilogy/core/processing/node_generators/unnest_node.py,sha256=cOEKnMRzXUW3bwmiOl
 trilogy/core/processing/node_generators/window_node.py,sha256=RUHgpYovQObFod1xRIMWtDzMcxwlm4-1Fdrf_Cuw5W4,6346
 trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=GMW07bb6hXurhF0hZLYoMAKSIS65tat5hwBjvqqPeSA,6516
-trilogy/core/processing/nodes/__init__.py,sha256=MfookoG73Z9p7kYFDh033uZJpP4exqkwdKtheDRepAw,5893
+trilogy/core/processing/nodes/__init__.py,sha256=xPFF7x3TFs1Z4IcfthCykZgrksb-UhN-pc_oIigfFSo,6014
 trilogy/core/processing/nodes/base_node.py,sha256=FHrY8GsTKPuMJklOjILbhGqCt5s1nmlj62Z-molARDA,16835
 trilogy/core/processing/nodes/filter_node.py,sha256=5VtRfKbCORx0dV-vQfgy3gOEkmmscL9f31ExvlODwvY,2461
 trilogy/core/processing/nodes/group_node.py,sha256=MUvcOg9U5J6TnWBel8eht9PdI9BfAKjUxmfjP_ZXx9o,10484
@@ -65,12 +65,12 @@ trilogy/core/processing/nodes/union_node.py,sha256=fDFzLAUh5876X6_NM7nkhoMvHEdGJ
 trilogy/core/processing/nodes/unnest_node.py,sha256=oLKMMNMx6PLDPlt2V5neFMFrFWxET8r6XZElAhSNkO0,2181
 trilogy/core/processing/nodes/window_node.py,sha256=JXJ0iVRlSEM2IBr1TANym2RaUf_p5E_l2sNykRzXWDo,1710
 trilogy/core/statements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/statements/author.py,sha256=rYDf9rCQ4YKEO9br1OWmOZd-51AiaDkaWYegvteJa8M,14728
+trilogy/core/statements/author.py,sha256=2q0yvP_0AbExdXnASlLG7OaDcM7sBaRco6YALnrQwzg,15255
 trilogy/core/statements/build.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/statements/common.py,sha256=KxEmz2ySySyZ6CTPzn0fJl5NX2KOk1RPyuUSwWhnK1g,759
 trilogy/core/statements/execute.py,sha256=cSlvpHFOqpiZ89pPZ5GDp9Hu6j6uj-5_h21FWm_L-KM,1248
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=vp6_9fUkblAWVpCXGBIcoAx6N7vof9M7s9t6-b_waUY,41409
+trilogy/dialect/base.py,sha256=kIwkNWWhMEIWDJ-6nr5bb2CJgbpKvJ9CxbhH4tYOFEc,41482
 trilogy/dialect/bigquery.py,sha256=7LcgPLDkeNBk6YTfaE-RBBi7SjWFV-jjuvZM1VMIXqk,3350
 trilogy/dialect/common.py,sha256=XjHkP8Dqezjkd2JU5xoAlMRS_6HNyXQCF4CykLK3C8o,5011
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
@@ -87,13 +87,13 @@ trilogy/hooks/graph_hook.py,sha256=c-vC-IXoJ_jDmKQjxQyIxyXPOuUcLIURB573gCsAfzQ,2
 trilogy/hooks/query_debugger.py,sha256=1npRjww94sPV5RRBBlLqMJRaFkH9vhEY6o828MeoEcw,5583
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=U9RNi1GyPTQaitZGwXy1QftdC5PWYArP7V8t-v3H8Po,27157
+trilogy/parsing/common.py,sha256=CCCkoRkFG9nxBI6u0lWt81JZvtBvXbTdiblvexOaeSY,29250
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
-trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
+trilogy/parsing/exceptions.py,sha256=Xwwsv2C9kSNv2q-HrrKC1f60JNHShXcCMzstTSEbiCw,154
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=9SO2q8m5MlZo_Eho-_r6hmTSm5VH38k47C2iHTtYwjU,68224
+trilogy/parsing/parse_engine.py,sha256=m8t6De4q-PhqthMl7iIJsB7JdSQ-YYnBlLumTkLzw1Q,68652
 trilogy/parsing/render.py,sha256=hI4y-xjXrEXvHslY2l2TQ8ic0zAOpN41ADH37J2_FZY,19047
-trilogy/parsing/trilogy.lark,sha256=zbDAIG7gpsImxBtteD8E2pKwcJCGpM-rEQDRqpgzoSQ,13717
+trilogy/parsing/trilogy.lark,sha256=q15J3P71yA_4lsWjC1vb7eDTemkJGLPKYvf5Hn9IBIk,13584
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -102,8 +102,8 @@ trilogy/std/display.preql,sha256=2BbhvqR4rcltyAbOXAUo7SZ_yGFYZgFnurglHMbjW2g,40
 trilogy/std/geography.preql,sha256=-fqAGnBL6tR-UtT8DbSek3iMFg66ECR_B_41pODxv-k,504
 trilogy/std/money.preql,sha256=ZHW-csTX-kYbOLmKSO-TcGGgQ-_DMrUXy0BjfuJSFxM,80
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.45.dist-info/METADATA,sha256=7z2vOsBA8amZQl0YEkjnWFBqxh4amLBZznjHGwZJ6-E,9100
-pytrilogy-0.0.3.45.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-pytrilogy-0.0.3.45.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.45.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.45.dist-info/RECORD,,
+pytrilogy-0.0.3.47.dist-info/METADATA,sha256=pWnJM7LvUZHAV5x2DH41DNeZHCo3lqmuEwmSPeWCeJQ,9100
+pytrilogy-0.0.3.47.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+pytrilogy-0.0.3.47.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.47.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.47.dist-info/RECORD,,

{pytrilogy-0.0.3.45.dist-info → pytrilogy-0.0.3.47.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.4.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.45"
+__version__ = "0.0.3.47"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/environment_helpers.py

@@ -6,23 +6,8 @@ from trilogy.core.models.core import DataType, StructType, arg_to_datatype
 from trilogy.core.models.environment import Environment
 from trilogy.parsing.common import Meta
 
-FUNCTION_DESCRIPTION_MAPS = {
-    FunctionType.DATE: "The date part of a timestamp/date. Integer, 0-31 depending on month.",
-    FunctionType.MONTH: "The month part of a timestamp/date. Integer, 1-12.",
-    FunctionType.YEAR: "The year part of a timestamp/date. Integer.",
-    FunctionType.QUARTER: "The quarter part of a timestamp/date. Integer, 1-4.",
-    FunctionType.DAY_OF_WEEK: "The day of the week part of a timestamp/date. Integer, 0-6.",
-    FunctionType.HOUR: "The hour part of a timestamp. Integer, 0-23.",
-    FunctionType.MINUTE: "The minute part of a timestamp. Integer, 0-59.",
-    FunctionType.SECOND: "The second part of a timestamp. Integer, 0-59.",
-}
-
 
 def generate_date_concepts(concept: Concept, environment: Environment):
-    if concept.metadata and concept.metadata.description:
-        base_description = concept.metadata.description
-    else:
-        base_description = f"a {concept.address}"
     if concept.metadata and concept.metadata.line_number:
         base_line_number = concept.metadata.line_number
     else:
@@ -67,7 +52,6 @@ def generate_date_concepts(concept: Concept, environment: Environment):
                 [concept.address],
             ),
             metadata=Metadata(
-                description=f"Auto-derived from {base_description}. {FUNCTION_DESCRIPTION_MAPS.get(ftype, ftype.value)}",
                 line_number=base_line_number,
                 concept_source=ConceptSource.AUTO_DERIVED,
             ),
@@ -95,7 +79,7 @@ def generate_date_concepts(concept: Concept, environment: Environment):
                 [concept.address],
             ),
             metadata=Metadata(
-                description=f"Auto-derived from {base_description}. The date truncated to the {grain.value}.",
+                # description=f"Auto-derived from {base_description}. The date truncated to the {grain.value}.",
                 line_number=base_line_number,
                 concept_source=ConceptSource.AUTO_DERIVED,
             ),
@@ -105,10 +89,6 @@ def generate_date_concepts(concept: Concept, environment: Environment):
 
 
 def generate_datetime_concepts(concept: Concept, environment: Environment):
-    if concept.metadata and concept.metadata.description:
-        base_description = concept.metadata.description
-    else:
-        base_description = concept.address
     if concept.metadata and concept.metadata.line_number:
         base_line_number = concept.metadata.line_number
     else:
@@ -146,7 +126,6 @@ def generate_datetime_concepts(concept: Concept, environment: Environment):
                 [concept.address],
             ),
             metadata=Metadata(
-                description=f"Auto-derived from {base_description}. {FUNCTION_DESCRIPTION_MAPS.get(ftype, ftype.value)}",
                 line_number=base_line_number,
                 concept_source=ConceptSource.AUTO_DERIVED,
             ),
@@ -157,10 +136,6 @@ def generate_datetime_concepts(concept: Concept, environment: Environment):
 
 
 def generate_key_concepts(concept: Concept, environment: Environment):
-    if concept.metadata and concept.metadata.description:
-        base_description = concept.metadata.description
-    else:
-        base_description = f"a {concept.datatype.value}"
     if concept.metadata and concept.metadata.line_number:
         base_line_number = concept.metadata.line_number
     else:
@@ -186,7 +161,7 @@ def generate_key_concepts(concept: Concept, environment: Environment):
             namespace=concept.namespace,
             keys=set(),
             metadata=Metadata(
-                description=f"Auto-derived integer. The {ftype.value} of {concept.address}, {base_description}",
+                # description=f"Auto-derived integer. The {ftype.value} of {concept.address}, {base_description}",
                 line_number=base_line_number,
                 concept_source=ConceptSource.AUTO_DERIVED,
             ),

trilogy/core/models/author.py

@@ -445,13 +445,16 @@ class Grain(Namespaced, BaseModel):
         concepts: Iterable[Concept | ConceptRef | str],
         environment: Environment | None = None,
         where_clause: WhereClause | None = None,
+        local_concepts: dict[str, Concept] | None = None,
     ) -> Grain:
         from trilogy.parsing.common import concepts_to_grain_concepts
 
         x = Grain.model_construct(
             components={
                 c.address
-                for c in concepts_to_grain_concepts(concepts, environment=environment)
+                for c in concepts_to_grain_concepts(
+                    concepts, environment=environment, local_concepts=local_concepts
+                )
             },
             where_clause=where_clause,
         )

trilogy/core/models/build.py

@@ -126,9 +126,14 @@ def concept_is_relevant(
 
         return False
     if concept.purpose in (Purpose.PROPERTY, Purpose.METRIC) and concept.keys:
-        if any([c in others for c in concept.keys]):
-
+        if all([c in others for c in concept.keys]):
             return False
+    if (
+        concept.purpose == Purpose.KEY
+        and concept.keys
+        and all([c in others for c in concept.keys])
+    ):
+        return False
     if concept.purpose in (Purpose.METRIC,):
         if all([c in others for c in concept.grain.components]):
             return False
@@ -1458,11 +1463,35 @@ class Factory:
             {} if local_concepts is None else local_concepts
         )
 
+    def instantiate_concept(
+        self,
+        arg: (
+            AggregateWrapper
+            | FunctionCallWrapper
+            | WindowItem
+            | FilterItem
+            | Function
+            | ListWrapper[Any]
+            | MapWrapper[Any, Any]
+            | int
+            | float
+            | str
+        ),
+    ) -> tuple[Concept, BuildConcept]:
+        from trilogy.parsing.common import arbitrary_to_concept
+
+        new = arbitrary_to_concept(
+            arg,
+            environment=self.environment,
+        )
+        built = self.build(new)
+        self.local_concepts[new.address] = built
+        return new, built
+
     @singledispatchmethod
     def build(self, base):
         raise NotImplementedError("Cannot build {}".format(type(base)))
 
-    @build.register
     @build.register
     def _(
         self,
@@ -1496,31 +1525,6 @@ class Factory:
     ):
         return base
 
-    def instantiate_concept(
-        self,
-        arg: (
-            AggregateWrapper
-            | FunctionCallWrapper
-            | WindowItem
-            | FilterItem
-            | Function
-            | ListWrapper[Any]
-            | MapWrapper[Any, Any]
-            | int
-            | float
-            | str
-        ),
-    ) -> tuple[Concept, BuildConcept]:
-        from trilogy.parsing.common import arbitrary_to_concept
-
-        new = arbitrary_to_concept(
-            arg,
-            environment=self.environment,
-        )
-        built = self.build(new)
-        self.local_concepts[new.address] = built
-        return new, built
-
     @build.register
     def _(self, base: None) -> None:
         return base
@@ -1626,6 +1630,7 @@ class Factory:
             derivation, final_grain, build_lineage
         )
         is_aggregate = Concept.calculate_is_aggregate(build_lineage)
+
         rval = BuildConcept.model_construct(
             name=base.name,
             datatype=base.datatype,
@@ -1646,7 +1651,6 @@ class Factory:
 
     @build.register
     def _(self, base: AggregateWrapper) -> BuildAggregateWrapper:
-
         if not base.by:
             by = [
                 self.build(self.environment.concepts[c]) for c in self.grain.components

trilogy/core/processing/node_generators/filter_node.py

@@ -56,12 +56,40 @@ def gen_filter_node(
                 continue
         if conditions and conditions == where:
             optional_included.append(x)
+
+    # sometimes, it's okay to include other local optional above the filter
+    # in case it is, prep our list
+    extra_row_level_optional: list[BuildConcept] = []
+    for x in local_optional:
+        if x.address in optional_included:
+            continue
+        extra_row_level_optional.append(x)
+
+    # this flag controls whether we materialize the filter as a where on the prior CTE
+    # or do the filtering inline as a case statement
+    optimized_pushdown = False
+    if not is_scalar_condition(where.conditional):
+        optimized_pushdown = False
+    elif not local_optional:
+        optimized_pushdown = True
+    elif conditions and conditions == where:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} query conditions are the same as filter conditions, can optimize across all concepts"
+        )
+        optimized_pushdown = True
+    elif optional_included == local_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} all optional concepts are included in the filter, can optimize across all concepts"
+        )
+        optimized_pushdown = True
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} filter `{concept}` condition `{concept.lineage.where}` derived from {immediate_parent.address} row parents {[x.address for x in parent_row_concepts]} and {[[y.address] for x  in parent_existence_concepts for y  in x]} existence parents"
     )
     # we'll populate this with the row parent
     # and the existence parent(s)
     core_parents = []
+    if not optimized_pushdown:
+        parent_row_concepts += extra_row_level_optional
 
     row_parent: StrategyNode = source_concepts(
         mandatory_list=parent_row_concepts,
@@ -99,21 +127,6 @@ def gen_filter_node(
         )
         return None
 
-    optimized_pushdown = False
-    if not is_scalar_condition(where.conditional):
-        optimized_pushdown = False
-    elif not local_optional:
-        optimized_pushdown = True
-    elif conditions and conditions == where:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} query conditions are the same as filter conditions, can optimize across all concepts"
-        )
-        optimized_pushdown = True
-    elif optional_included == local_optional:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} all optional concepts are included in the filter, can optimize across all concepts"
-        )
-        optimized_pushdown = True
     if optimized_pushdown:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional}"
@@ -193,7 +206,9 @@ def gen_filter_node(
             + optional_outputs
         )
         return filter_node
-
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} need to enrich filter node with additional concepts {[x.address for x in local_optional if x.address not in filter_node.output_concepts]}"
+    )
     enrich_node: StrategyNode = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
         mandatory_list=[immediate_parent] + parent_row_concepts + local_optional,

trilogy/core/processing/nodes/__init__.py

@@ -1,6 +1,7 @@
 from pydantic import BaseModel, ConfigDict, Field
 
 from trilogy.core.exceptions import UnresolvableQueryException
+from trilogy.core.models.author import Concept
 from trilogy.core.models.build import BuildConcept, BuildWhereClause
 from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.models.environment import Environment
@@ -17,6 +18,7 @@ from .window_node import WindowNode
 
 class History(BaseModel):
     base_environment: Environment
+    local_base_concepts: dict[str, Concept] = Field(default_factory=dict)
     history: dict[str, StrategyNode | None] = Field(default_factory=dict)
     select_history: dict[str, StrategyNode | None] = Field(default_factory=dict)
     started: dict[str, int] = Field(default_factory=dict)

trilogy/core/query_processor.py

@@ -373,10 +373,9 @@ def get_query_node(
 ) -> StrategyNode:
     if not statement.output_components:
         raise ValueError(f"Statement has no output components {statement}")
-
     history = history or History(base_environment=environment)
     build_statement: BuildSelectLineage | BuildMultiSelectLineage = Factory(
-        environment=environment
+        environment=environment,
     ).build(statement)
 
     # build_statement = statement

trilogy/core/statements/author.py

@@ -7,6 +7,7 @@ from pydantic.functional_validators import PlainValidator
 
 from trilogy.constants import CONFIG
 from trilogy.core.enums import (
+    ConceptSource,
     FunctionClass,
     IOType,
     Modifier,
@@ -134,7 +135,7 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
             meta=meta or Metadata(),
         )
 
-        output.grain = output.calculate_grain(environment)
+        output.grain = output.calculate_grain(environment, output.local_concepts)
 
         for x in selection:
             if x.is_undefined and environment.concepts.fail_on_missing:
@@ -144,12 +145,13 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
             elif isinstance(x.content, ConceptTransform):
                 if isinstance(x.content.output, UndefinedConcept):
                     continue
-                if (
-                    CONFIG.parsing.select_as_definition
-                    and not environment.frozen
-                    and x.concept.address not in environment.concepts
-                ):
-                    environment.add_concept(x.content.output)
+                if CONFIG.parsing.select_as_definition and not environment.frozen:
+                    if x.concept.address not in environment.concepts:
+                        environment.add_concept(x.content.output)
+                    elif x.concept.address in environment.concepts:
+                        version = environment.concepts[x.concept.address]
+                        if version.metadata.concept_source == ConceptSource.SELECT:
+                            environment.add_concept(x.content.output, force=True)
                 x.content.output = x.content.output.set_select_grain(
                     output.grain, environment
                 )
@@ -160,16 +162,26 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
                 output.local_concepts[x.content.address] = environment.concepts[
                     x.content.address
                 ]
+
+        output.grain = output.calculate_grain(environment, output.local_concepts)
+
         output.validate_syntax(environment)
         return output
 
-    def calculate_grain(self, environment: Environment | None = None) -> Grain:
+    def calculate_grain(
+        self,
+        environment: Environment | None = None,
+        local_concepts: dict[str, Concept] | None = None,
+    ) -> Grain:
         targets = []
         for x in self.selection:
             targets.append(x.concept)
 
         result = Grain.from_concepts(
-            targets, where_clause=self.where_clause, environment=environment
+            targets,
+            where_clause=self.where_clause,
+            environment=environment,
+            local_concepts=local_concepts,
         )
         return result
 

trilogy/dialect/base.py

@@ -199,6 +199,7 @@ FUNCTION_MAP = {
     FunctionType.DATE_TRUNCATE: lambda x: f"date_trunc({x[0]},{x[1]})",
     FunctionType.DATE_PART: lambda x: f"date_part({x[0]},{x[1]})",
     FunctionType.DATE_ADD: lambda x: f"date_add({x[0]},{x[1]}, {x[2]})",
+    FunctionType.DATE_SUB: lambda x: f"date_sub({x[0]},{x[1]}, {x[2]})",
     FunctionType.DATE_DIFF: lambda x: f"date_diff({x[0]},{x[1]}, {x[2]})",
     FunctionType.DATE: lambda x: f"date({x[0]})",
     FunctionType.DATETIME: lambda x: f"datetime({x[0]})",

trilogy/parsing/common.py

@@ -24,6 +24,7 @@ from trilogy.core.models.author import (
     AlignClause,
     AlignItem,
     Concept,
+    ConceptArgs,
     ConceptRef,
     FilterItem,
     Function,
@@ -191,16 +192,53 @@ def constant_to_concept(
     )
 
 
+def atom_is_relevant(
+    atom,
+    others: list[Concept | ConceptRef],
+    environment: Environment | None = None,
+):
+    if isinstance(atom, (ConceptRef, Concept)):
+        # when we are looking at atoms, if there is a concept that is in others
+        # return directly
+        if atom.address in others:
+            return False
+        return concept_is_relevant(atom, others, environment)
+
+    if isinstance(atom, AggregateWrapper) and not atom.by:
+        return False
+    elif isinstance(atom, AggregateWrapper):
+        return any(atom_is_relevant(x, others, environment) for x in atom.by)
+
+    if isinstance(atom, Function):
+        relevant = False
+        print("atom args")
+        for arg in atom.arguments:
+            relevant = relevant or atom_is_relevant(arg, others, environment)
+        return relevant
+    elif isinstance(atom, FunctionCallWrapper):
+        return any(
+            [atom_is_relevant(atom.content, others, environment)]
+            + [atom_is_relevant(x, others, environment) for x in atom.args]
+        )
+    elif isinstance(atom, ConceptArgs):
+        # use atom is relevant here to trigger the early exit behavior for concpets in set
+        return any(
+            [atom_is_relevant(x, others, environment) for x in atom.concept_arguments]
+        )
+    return False
+
+
 def concept_is_relevant(
     concept: Concept | ConceptRef,
     others: list[Concept | ConceptRef],
     environment: Environment | None = None,
 ) -> bool:
-    if isinstance(concept, UndefinedConcept):
 
+    if isinstance(concept, UndefinedConcept):
         return False
     if concept.datatype == DataType.UNKNOWN:
         return False
+
     if isinstance(concept, ConceptRef):
         if environment:
             concept = environment.concepts[concept.address]
@@ -208,41 +246,56 @@ def concept_is_relevant(
             raise SyntaxError(
                 "Require environment to determine relevance of ConceptRef"
             )
-
+    if concept.derivation == Derivation.CONSTANT:
+        return False
     if concept.is_aggregate and not (
         isinstance(concept.lineage, AggregateWrapper) and concept.lineage.by
     ):
 
         return False
     if concept.purpose in (Purpose.PROPERTY, Purpose.METRIC) and concept.keys:
-        if any([c in others for c in concept.keys]):
-
+        if all([c in others for c in concept.keys]):
             return False
+    if (
+        concept.purpose == Purpose.KEY
+        and concept.keys
+        and all([c in others for c in concept.keys])
+    ):
+        return False
     if concept.purpose in (Purpose.METRIC,):
         if all([c in others for c in concept.grain.components]):
             return False
-    if concept.derivation in (Derivation.BASIC,):
-        return any(
-            concept_is_relevant(c, others, environment)
-            for c in concept.concept_arguments
-        )
+    if concept.derivation in (Derivation.BASIC,) and isinstance(
+        concept.lineage, Function
+    ):
+        relevant = False
+        for arg in concept.lineage.arguments:
+            relevant = atom_is_relevant(arg, others, environment) or relevant
+        return relevant
     if concept.granularity == Granularity.SINGLE_ROW:
         return False
     return True
 
 
 def concepts_to_grain_concepts(
-    concepts: Iterable[Concept | ConceptRef | str], environment: Environment | None
+    concepts: Iterable[Concept | ConceptRef | str],
+    environment: Environment | None,
+    local_concepts: dict[str, Concept] | None = None,
 ) -> list[Concept]:
     pconcepts: list[Concept] = []
     for c in concepts:
-
         if isinstance(c, Concept):
             pconcepts.append(c)
         elif isinstance(c, ConceptRef) and environment:
-            pconcepts.append(environment.concepts[c.address])
+            if local_concepts and c.address in local_concepts:
+                pconcepts.append(local_concepts[c.address])
+            else:
+                pconcepts.append(environment.concepts[c.address])
         elif isinstance(c, str) and environment:
-            pconcepts.append(environment.concepts[c])
+            if local_concepts and c in local_concepts:
+                pconcepts.append(local_concepts[c])
+            else:
+                pconcepts.append(environment.concepts[c])
         else:
             raise ValueError(
                 f"Unable to resolve input {c} without environment provided to concepts_to_grain call"
@@ -250,6 +303,7 @@ def concepts_to_grain_concepts(
 
     final: List[Concept] = []
     for sub in pconcepts:
+
         if not concept_is_relevant(sub, pconcepts, environment):  # type: ignore
             continue
         final.append(sub)
@@ -366,7 +420,12 @@ def function_to_concept(
     is_metric = False
     ref_args, is_metric = get_relevant_parent_concepts(parent)
     concrete_args = [environment.concepts[c.address] for c in ref_args]
-    pkeys += [x for x in concrete_args if not x.derivation == Derivation.CONSTANT]
+    pkeys += [
+        x
+        for x in concrete_args
+        if not x.derivation == Derivation.CONSTANT
+        and not (x.derivation == Derivation.AGGREGATE and not x.grain.components)
+    ]
     grain: Grain | None = Grain()
     for x in pkeys:
         grain += x.grain
@@ -376,7 +435,7 @@ def function_to_concept(
     modifiers = get_upstream_modifiers(pkeys, environment)
     key_grain: list[str] = []
     for x in pkeys:
-        # metrics will group to keys, so do no do key traversal
+        # metrics will group to keys, so do not do key traversal
         if is_metric:
             key_grain.append(x.address)
         # otherwse, for row ops, assume keys are transitive
@@ -419,7 +478,6 @@ def function_to_concept(
     else:
         derivation = Derivation.BASIC
         granularity = Granularity.MULTI_ROW
-
     if grain is not None:
         r = Concept(
             name=name,

trilogy/parsing/exceptions.py

@@ -1,2 +1,8 @@
 class ParseError(Exception):
     pass
+
+
+class NameShadowError(ParseError):
+    """
+    Raised when a name shadows another name in the same scope.
+    """

trilogy/parsing/parse_engine.py

@@ -117,7 +117,6 @@ from trilogy.core.statements.author import (
     CopyStatement,
     FunctionDeclaration,
     ImportStatement,
-    KeyMergeStatement,
     Limit,
     MergeStatementV2,
     MultiSelectStatement,
@@ -136,7 +135,7 @@ from trilogy.parsing.common import (
     process_function_args,
     rowset_to_concepts,
 )
-from trilogy.parsing.exceptions import ParseError
+from trilogy.parsing.exceptions import NameShadowError, ParseError
 
 perf_logger = getLogger("trilogy.parse.performance")
 
@@ -814,6 +813,24 @@ class ParseToObjects(Transformer):
                     )
         if self.parse_pass == ParsePass.VALIDATION:
             self.environment.add_datasource(datasource, meta=meta)
+            # if we have any foreign keys on the datasource, we can
+            # at this point optimize them to properties if they do not have other usage.
+            for column in columns:
+                # skip partial for now
+                if not grain:
+                    continue
+                if column.concept.address in grain.components:
+                    continue
+                target_c = self.environment.concepts[column.concept.address]
+                if target_c.purpose != Purpose.KEY:
+                    continue
+
+                key_inputs = grain.components
+                keys = [self.environment.concepts[grain] for grain in key_inputs]
+                # target_c.purpose = Purpose.PROPERTY
+                target_c.keys = set([x.address for x in keys])
+                # target_c.grain = Grain(components={x.address for x in keys})
+
         return datasource
 
     @v_args(meta=True)
@@ -903,29 +920,6 @@ class ParseToObjects(Transformer):
     def over_list(self, args):
         return [x for x in args]
 
-    @v_args(meta=True)
-    def key_merge_statement(self, meta: Meta, args) -> KeyMergeStatement | None:
-        key_inputs = args[:-1]
-        target = args[-1]
-        keys = [self.environment.concepts[grain] for grain in key_inputs]
-        target_c = self.environment.concepts[target]
-        new = KeyMergeStatement(
-            keys=set([x.address for x in keys]),
-            target=target_c.reference,
-        )
-        internal = Concept(
-            name="_" + target_c.address.replace(".", "_"),
-            namespace=self.environment.namespace,
-            purpose=Purpose.PROPERTY,
-            keys=set([x.address for x in keys]),
-            datatype=target_c.datatype,
-            grain=Grain(components={x.address for x in keys}),
-        )
-        self.environment.add_concept(internal)
-        # always a full merge
-        self.environment.merge_concept(target_c, internal, [])
-        return new
-
     @v_args(meta=True)
     def merge_statement(self, meta: Meta, args) -> MergeStatementV2 | None:
         modifiers = []
@@ -1253,9 +1247,18 @@ class ParseToObjects(Transformer):
         ):
             intersection = base.locally_derived.intersection(pre_keys)
             if intersection:
-                raise ParseError(
-                    f"Select statement {base} creates new derived concepts {list(intersection)} from transformations with identical name(s) to existing concept(s). Do you mean to drop the calculation and directly use the existing concept? If not, alias these concept(s) under new names."
-                )
+                for x in intersection:
+                    if (
+                        base.local_concepts[x].derivation
+                        == self.environment.concepts[x].derivation
+                    ):
+                        raise NameShadowError(
+                            f"Select statement {base} derives concept {x} with identical derivation as named concept. Use the named concept directly."
+                        )
+                else:
+                    raise NameShadowError(
+                        f"Select statement {base} creates new derived concepts {list(intersection)} with identical name(s) to existing concept(s). If these are identical, reference the concept directly. Otherwise alias your column as a new name."
+                    )
         return base
 
     @v_args(meta=True)
@@ -1854,6 +1857,8 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def fround(self, meta, args) -> Function:
+        if len(args) == 1:
+            args.append(0)
         return self.function_factory.create_function(args, FunctionType.ROUND, meta)
 
     @v_args(meta=True)

trilogy/parsing/trilogy.lark

@@ -10,7 +10,6 @@
     | rowset_derivation_statement
     | import_statement
     | copy_statement
-    | key_merge_statement
     | merge_statement
     | rawsql_statement
     
@@ -77,8 +76,6 @@
 
     align_clause: align_item ("AND"i align_item)*  "AND"i?
 
-    key_merge_statement: "merge"i "property"i "<" IDENTIFIER ("," IDENTIFIER )* ","? ">" "from"i IDENTIFIER 
-
     merge_statement: "merge"i  WILDCARD_IDENTIFIER "into"i SHORTHAND_MODIFIER? WILDCARD_IDENTIFIER
 
     // raw sql statement
@@ -223,7 +220,7 @@
     fdiv: ( "divide"i "(" expr "," expr ")") 
     fmod: ( "mod"i "(" expr "," (int_lit | concept_lit ) ")")  
     _ROUND.1: "round"i "("
-    fround: _ROUND expr "," expr ")"
+    fround: _ROUND expr ("," expr)? ")"
     fabs: "abs"i "(" expr ")"
     _SQRT.1: "sqrt("
     fsqrt:  _SQRT expr ")"