pytrilogy 0.0.3.32__py3-none-any.whl → 0.0.3.33__py3-none-any.whl

{pytrilogy-0.0.3.32.dist-info → pytrilogy-0.0.3.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.32
+Version: 0.0.3.33
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.32.dist-info → pytrilogy-0.0.3.33.dist-info}/RECORD

@@ -1,5 +1,5 @@
-pytrilogy-0.0.3.32.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=LEjwWz5ASSeELMnWJRDTgcuJHGstMRNS9gvzU8PvHfQ,303
+pytrilogy-0.0.3.33.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=d8QFTlSiA9H5vsTbRyWcF6hZW9Dih9-6ZorFRaWkb_Q,303
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=5eQxk1A0pv-TQk3CCvgZCFA9_K-6nxrOm7E5Lxd7KIY,1652
 trilogy/engine.py,sha256=OK2RuqCIUId6yZ5hfF8J1nxGP0AJqHRZiafcowmW0xc,1728
@@ -11,7 +11,7 @@ trilogy/utility.py,sha256=euQccZLKoYBz0LNg5tzLlvv2YHvXh9HArnYp1V3uXsM,763
 trilogy/authoring/__init__.py,sha256=ohkYA3_LGYZh3fwzEYKTN6ofACDI5GYl3VCbGxVvlzY,2233
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=7XaCpZn5mQmjTobbeBn56SzPWq9eMNDfzfsRU-fP0VE,171
-trilogy/core/enums.py,sha256=7SW4V0PpceeAUREyxDKPBe2JAtDMOM5IOFhn9fYKObM,7318
+trilogy/core/enums.py,sha256=fWexUZtssfvP5TiD7eQ66Q_tPUCNCCTGNSzLbVXrnqQ,7358
 trilogy/core/env_processor.py,sha256=pFsxnluKIusGKx1z7tTnfsd_xZcPy9pZDungkjkyvI0,3170
 trilogy/core/environment_helpers.py,sha256=5ayyhf4CGBlg_LssPu3DbS_H9H1Kq6Qog5TgR8pwyMk,8518
 trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
@@ -19,34 +19,34 @@ trilogy/core/exceptions.py,sha256=JPYyBcit3T_pRtlHdtKSeVJkIyWUTozW2aaut25A2xI,67
 trilogy/core/functions.py,sha256=z5uyC5sAS_vnFBJcky7TjA0QAy-xI4Z9uD_9vK_XP1s,27134
 trilogy/core/graph_models.py,sha256=z17EoO8oky2QOuO6E2aMWoVNKEVJFhLdsQZOhC4fNLU,2079
 trilogy/core/internal.py,sha256=iicDBlC6nM8d7e7jqzf_ZOmpUsW8yrr2AA8AqEiLx-s,1577
-trilogy/core/optimization.py,sha256=xGO8piVsLrpqrx-Aid_Y56_5slSv4eZmlP64hCHRiEc,7957
+trilogy/core/optimization.py,sha256=aihzx4-2-mSjx5td1TDTYGvc7e9Zvy-_xEyhPqLS-Ig,8314
 trilogy/core/query_processor.py,sha256=Do8YpdPBdsbKtl9n37hobzk8SORMGqH-e_zNNxd-BE4,19456
 trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/models/author.py,sha256=FtECoNRevZ8sD9JI-D8a9ngxxVoKNtfrdEjbnofYcg8,76131
-trilogy/core/models/build.py,sha256=3_n6qxoMuzX3Ahu9HLfoyPbtH0Wb--6crZNwCnZhu7k,58104
+trilogy/core/models/author.py,sha256=qEN8dJCnOMtB2Uu58GcbiYz6t5LiluCAdDUCUfl6BMk,76572
+trilogy/core/models/build.py,sha256=N3Zr47iN4y9gFWS72uYjecmiQfcm8Fl9wT3QS8xKBAA,58185
 trilogy/core/models/build_environment.py,sha256=8UggvlPU708GZWYPJMc_ou2r7M3TY2g69eqGvz03YX0,5528
 trilogy/core/models/core.py,sha256=wx6hJcFECMG-Ij972ADNkr-3nFXkYESr82ObPiC46_U,10875
 trilogy/core/models/datasource.py,sha256=6RjJUd2u4nYmEwFBpJlM9LbHVYDv8iHJxqiBMZqUrwI,9422
 trilogy/core/models/environment.py,sha256=yd19vDdC-IiKPV5YbNZcDUOEr1BC_ZxE_5fQDIipa3s,26910
-trilogy/core/models/execute.py,sha256=KZHiovlSr_3ZjyzKD1mdBlAqnPCqFCChQkO4_4WlGtg,34224
+trilogy/core/models/execute.py,sha256=cmDPkVng3LHw87Q4WiOxJvDQJCoPhEiVMeMqxSxRj7Q,34261
 trilogy/core/optimizations/__init__.py,sha256=EBanqTXEzf1ZEYjAneIWoIcxtMDite5-n2dQ5xcfUtg,356
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
 trilogy/core/optimizations/inline_constant.py,sha256=lvNTIXaLNkw3HseJyXyDNk5R52doLU9sIg3pmU2_S08,1332
 trilogy/core/optimizations/inline_datasource.py,sha256=AHuTGh2x0GQ8usOe0NiFncfTFQ_KogdgDl4uucmhIbI,4241
 trilogy/core/optimizations/predicate_pushdown.py,sha256=g4AYE8Aw_iMlAh68TjNXGP754NTurrDduFECkUjoBnc,9399
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=O-O08cGLl4XX3Faf59UlkFjsjzTU-QkzlDQ0mzY-LRE,40515
+trilogy/core/processing/concept_strategies_v3.py,sha256=MhSCS1M6etHaonfrPNwe6Gk-QuDTVGHkPjt1_9_gqko,42536
 trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
 trilogy/core/processing/utility.py,sha256=mnN_pewdpDgRou4QJ1JLcqYHyZdp8DrcsGsqW3QmA3o,20591
 trilogy/core/processing/node_generators/__init__.py,sha256=o8rOFHPSo-s_59hREwXMW6gjUJCsiXumdbJNozHUf-Y,800
 trilogy/core/processing/node_generators/basic_node.py,sha256=UVsXMn6jTjm_ofVFt218jAS11s4RV4zD781vP4im-GI,3371
 trilogy/core/processing/node_generators/common.py,sha256=ZsDzThjm_mAtdQpKAg8QIJiPVZ4KuUkKyilj4eOhSDs,9439
 trilogy/core/processing/node_generators/filter_node.py,sha256=rlY7TbgjJlGhahYgdCIJpJbaSREAGVJEsyUIGaA38O0,8271
-trilogy/core/processing/node_generators/group_node.py,sha256=iB5kq-EaueoaSQkgegULwo4g-fe8PKtRtSDiphNO5_k,5939
+trilogy/core/processing/node_generators/group_node.py,sha256=kO-ersxIL04rZwX5-vFIFQQnp357PFo_7ZKXoGq3wyc,5989
 trilogy/core/processing/node_generators/group_to_node.py,sha256=E5bEjovSx422d_MlAUCDFdY4P2WJVp61BmWwltkhzA8,3095
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=sv55oynfqgpHEpo1OEtVDri-5fywzPhDlR85qaWikvY,16195
-trilogy/core/processing/node_generators/rowset_node.py,sha256=tQog2YbG7h2TSVZGdXCFbwjqpq-CKqNewFOOfpZNqh4,6751
+trilogy/core/processing/node_generators/rowset_node.py,sha256=YmBs6ZQ7azLXRFEmeoecpGjK4pMHsUCovuBxfb3UKZI,6848
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=pIsHfXWA30RkKSMBDKPtDmCriJtHoNKRMJC0THSDxpI,19951
 trilogy/core/processing/node_generators/select_node.py,sha256=Y-zO0AFkTrpi2LyebjpyHU7WWANr7nKZSS9rY7DH4Wo,1888
 trilogy/core/processing/node_generators/synonym_node.py,sha256=9LHK2XHDjbyTLjmDQieskG8fqbiSpRnFOkfrutDnOTE,2258
@@ -58,8 +58,8 @@ trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,s
 trilogy/core/processing/nodes/__init__.py,sha256=DqPG3Y8vl5-UTeox6hn1EE6iwPIJpsM-XeZALHSgLZQ,5058
 trilogy/core/processing/nodes/base_node.py,sha256=FHrY8GsTKPuMJklOjILbhGqCt5s1nmlj62Z-molARDA,16835
 trilogy/core/processing/nodes/filter_node.py,sha256=5VtRfKbCORx0dV-vQfgy3gOEkmmscL9f31ExvlODwvY,2461
-trilogy/core/processing/nodes/group_node.py,sha256=1caU36nHknnXS-dM6xmL9Mc7kK0wCq-IS3kL7_XYNlA,8024
-trilogy/core/processing/nodes/merge_node.py,sha256=bEz1QU2o-fl_O-VotE5dN1GmlZPClufMvUOvL2-2Uo8,15262
+trilogy/core/processing/nodes/group_node.py,sha256=MUvcOg9U5J6TnWBel8eht9PdI9BfAKjUxmfjP_ZXx9o,10484
+trilogy/core/processing/nodes/merge_node.py,sha256=djFBJ8Sq1_crP0g7REGCbMFl4so7zPzabkPgAA1TUl4,15720
 trilogy/core/processing/nodes/select_node_v2.py,sha256=Xyfq8lU7rP7JTAd8VV0ATDNal64n4xIBgWQsOuMe_Ak,8824
 trilogy/core/processing/nodes/union_node.py,sha256=fDFzLAUh5876X6_NM7nkhoMvHEdGJ_LpvPokpZKOhx4,1425
 trilogy/core/processing/nodes/unnest_node.py,sha256=oLKMMNMx6PLDPlt2V5neFMFrFWxET8r6XZElAhSNkO0,2181
@@ -87,13 +87,13 @@ trilogy/hooks/graph_hook.py,sha256=c-vC-IXoJ_jDmKQjxQyIxyXPOuUcLIURB573gCsAfzQ,2
 trilogy/hooks/query_debugger.py,sha256=1npRjww94sPV5RRBBlLqMJRaFkH9vhEY6o828MeoEcw,5583
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=_61lUd6DONwteY_D6lanZZgi2NoEWivAxJtrOYrmw34,21915
+trilogy/parsing/common.py,sha256=bhZ_f520D2diRcXC3YeEToPSXBbUBBNq42zII_smdYc,23518
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=HBvJewa5zP4y8FasH6nYiqKP9WiIsdl7udfaY_GWxHU,62233
+trilogy/parsing/parse_engine.py,sha256=2BeNdsYoyn5UktTdafEKTZ8H7-ELEahDJeqcwMvPtO8,62567
 trilogy/parsing/render.py,sha256=ElpmCWUhGs8h489S7cdlbI8bilJlnBgHZ8KMR8y1FrM,18840
-trilogy/parsing/trilogy.lark,sha256=h7mJiad7GgTTXnrjntE6OF7xpND5TQlvHqZFfx0nkyk,12993
+trilogy/parsing/trilogy.lark,sha256=SeuOZrpIUSqQwCj5mTSmTSkeSLTPwBgtm2AdpcGAy9A,13023
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -102,8 +102,8 @@ trilogy/std/date.preql,sha256=0MHeGLp2mG4QBKtmozcBZ7qVjOAdWOtrliIKn6hz1Pc,95
 trilogy/std/display.preql,sha256=2BbhvqR4rcltyAbOXAUo7SZ_yGFYZgFnurglHMbjW2g,40
 trilogy/std/geography.preql,sha256=-fqAGnBL6tR-UtT8DbSek3iMFg66ECR_B_41pODxv-k,504
 trilogy/std/money.preql,sha256=ZHW-csTX-kYbOLmKSO-TcGGgQ-_DMrUXy0BjfuJSFxM,80
-pytrilogy-0.0.3.32.dist-info/METADATA,sha256=xnPeyXro94RAWPJBvFkrPPA219KCrQe0sAUhb0kcFgA,9100
-pytrilogy-0.0.3.32.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-pytrilogy-0.0.3.32.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.32.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.32.dist-info/RECORD,,
+pytrilogy-0.0.3.33.dist-info/METADATA,sha256=nCv6QIF6oA0nSwlsLa_znFnm6RetCf4zyfBdBszleNY,9100
+pytrilogy-0.0.3.33.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+pytrilogy-0.0.3.33.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.33.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.33.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.32"
+__version__ = "0.0.3.33"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/enums.py

@@ -27,6 +27,7 @@ class Purpose(Enum):
     CONSTANT = "const"
     KEY = "key"
     PROPERTY = "property"
+    UNIQUE_PROPERTY = "unique_property"
     METRIC = "metric"
     ROWSET = "rowset"
     AUTO = "auto"

trilogy/core/models/author.py

@@ -1326,7 +1326,7 @@ class OrderItem(Mergeable, ConceptArgs, Namespaced, BaseModel):
 
 class WindowItem(DataTyped, ConceptArgs, Mergeable, Namespaced, BaseModel):
     type: WindowType
-    content: ConceptRef
+    content: FuncArgs
     order_by: List["OrderItem"]
     over: List["ConceptRef"] = Field(default_factory=list)
     index: Optional[int] = None
@@ -1335,7 +1335,7 @@ class WindowItem(DataTyped, ConceptArgs, Mergeable, Namespaced, BaseModel):
         return self.__repr__()
 
     def __repr__(self):
-        return f"{self.type}({self.content} {self.index}, {self.over}, {self.order_by})"
+        return f"{self.type.value} {self.content} by {self.index} over {self.over} order {self.order_by}"
 
     @field_validator("content", mode="before")
     def enforce_concept_ref(cls, v):
@@ -1358,7 +1358,11 @@ class WindowItem(DataTyped, ConceptArgs, Mergeable, Namespaced, BaseModel):
     ) -> "WindowItem":
         output = WindowItem.model_construct(
             type=self.type,
-            content=self.content.with_merge(source, target, modifiers),
+            content=(
+                self.content.with_merge(source, target, modifiers)
+                if isinstance(self.content, Mergeable)
+                else self.content
+            ),
             over=[x.with_merge(source, target, modifiers) for x in self.over],
             order_by=[x.with_merge(source, target, modifiers) for x in self.order_by],
             index=self.index,
@@ -1379,7 +1383,11 @@ class WindowItem(DataTyped, ConceptArgs, Mergeable, Namespaced, BaseModel):
     def with_namespace(self, namespace: str) -> "WindowItem":
         return WindowItem.model_construct(
             type=self.type,
-            content=self.content.with_namespace(namespace),
+            content=(
+                self.content.with_namespace(namespace)
+                if isinstance(self.content, Namespaced)
+                else self.content
+            ),
             over=[x.with_namespace(namespace) for x in self.over],
             order_by=[x.with_namespace(namespace) for x in self.order_by],
             index=self.index,
@@ -1387,7 +1395,8 @@ class WindowItem(DataTyped, ConceptArgs, Mergeable, Namespaced, BaseModel):
 
     @property
     def concept_arguments(self) -> List[ConceptRef]:
-        output = [self.content]
+        output = []
+        output += get_concept_arguments(self.content)
         for order in self.order_by:
             output += get_concept_arguments(order)
         for item in self.over:
@@ -1787,6 +1796,10 @@ class FunctionCallWrapper(
             ],
         )
 
+    def with_reference_replacement(self, source, target):
+        raise NotImplementedError("Cannot reference replace")
+        return self
+
     def with_merge(
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ) -> "FunctionCallWrapper":

trilogy/core/models/build.py

@@ -1646,7 +1646,7 @@ class Factory:
 
     @build.register
     def _(self, base: WindowItem) -> BuildWindowItem:
-
+        # to do proper discovery, we need to inject virtual intermediate ocncepts
         return BuildWindowItem.model_construct(
             type=base.type,
             content=self.build(base.content),

trilogy/core/models/execute.py

@@ -637,7 +637,7 @@ class QueryDatasource(BaseModel):
                     and CONFIG.validate_missing
                 ):
                     raise SyntaxError(
-                        f"On query datasource missing source map for {concept.address} on {key}, have {v}"
+                        f"On query datasource missing source map for {concept.address} on {key} with pseudonyms {concept.pseudonyms}, have {v}"
                     )
         return v
 

trilogy/core/optimization.py

@@ -94,6 +94,11 @@ def filter_irrelevant_ctes(
     inverse_map = gen_inverse_map(input)
     recurse(root_cte, inverse_map)
     final = [cte for cte in input if cte.name in relevant_ctes]
+    filtered = [cte for cte in input if cte.name not in relevant_ctes]
+    if filtered:
+        logger.info(
+            f"[Optimization][Irrelevent CTE filtering] Removing redundant CTEs {[x.name for x in filtered]}"
+        )
     if len(final) == len(input):
         return input
     return filter_irrelevant_ctes(final, root_cte)
@@ -130,6 +135,9 @@ def is_direct_return_eligible(cte: CTE | UnionCTE) -> CTE | UnionCTE | None:
     if not output_addresses.issubset(parent_output_addresses):
         return None
     if not direct_parent.grain == cte.grain:
+        logger.info("grain mismatch, cannot early exit")
+        logger.info(direct_parent.grain)
+        logger.info(cte.grain)
         return None
 
     assert isinstance(cte, CTE)

trilogy/core/processing/concept_strategies_v3.py

@@ -625,6 +625,7 @@ def validate_stack(
         for concept in resolved.output_concepts:
             if concept.address in resolved.hidden_concepts:
                 continue
+
             validate_concept(
                 concept,
                 node,
@@ -773,6 +774,8 @@ def _search_concepts(
 ) -> StrategyNode | None:
     # these are the concepts we need in the output projection
     mandatory_list = unique(mandatory_list, "address")
+    # cache our values before an filter injection
+    original_mandatory = [*mandatory_list]
     for x in mandatory_list:
         if isinstance(x, UndefinedConcept):
             raise SyntaxError(f"Undefined concept {x.address}")
@@ -921,7 +924,7 @@ def _search_concepts(
         mandatory_completion = [c.address for c in completion_mandatory]
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} finished concept loop for {priority_concept} {priority_concept.derivation} condition {conditions} flag for accepting partial addresses is"
-            f" {accept_partial} (complete: {complete}), have {found} from {[n for n in stack]} (missing {missing} partial {partial} virtual {virtual}), attempted {attempted}, mandatory w/ filter {mandatory_completion}"
+            f" {accept_partial} (complete: {complete}), have {found} from {[n for n in stack]} (missing {missing} synonyms  partial {partial} virtual {virtual}), attempted {attempted}, mandatory w/ filter {mandatory_completion}"
         )
         if complete == ValidationResult.INCOMPLETE_CONDITION:
             cond_dict = {str(node): node.preexisting_conditions for node in stack}
@@ -948,6 +951,11 @@ def _search_concepts(
     if complete == ValidationResult.COMPLETE:
         condition_required = True
         non_virtual = [c for c in completion_mandatory if c.address not in virtual]
+        non_virtual_output = [c for c in original_mandatory if c.address not in virtual]
+        non_virtual_different = len(completion_mandatory) != len(original_mandatory)
+        non_virtual_difference_values = set(
+            [x.address for x in completion_mandatory]
+        ).difference(set([x.address for x in original_mandatory]))
         if not conditions:
             condition_required = False
             non_virtual = [c for c in mandatory_list if c.address not in virtual]
@@ -966,7 +974,19 @@ def _search_concepts(
             )
         if len(stack) == 1:
             output: StrategyNode = stack[0]
-            # _ = restrict_node_outputs_targets(output, mandatory_list, depth)
+            if non_virtual_different:
+                logger.info(
+                    f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found different non-virtual output concepts ({non_virtual_difference_values}), removing condition injected values"
+                )
+                output.set_output_concepts(
+                    [
+                        x
+                        for x in output.output_concepts
+                        if x.address in non_virtual_output
+                    ],
+                    rebuild=False,
+                )
+
             logger.info(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)}"
             )
@@ -995,6 +1015,30 @@ def _search_concepts(
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node partial {[c.address for c in output.partial_concepts]}"
         )
+        if condition_required and conditions and non_virtual_different:
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Conditions {conditions} were injected, checking if we need a group to restore grain"
+            )
+            result = GroupNode.check_if_required(
+                downstream_concepts=original_mandatory,
+                parents=[output.resolve()],
+                environment=environment,
+                depth=depth,
+            )
+            logger.info(f"gcheck result is {result}")
+            if result.required:
+                logger.info(
+                    f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Adding group node"
+                )
+                return GroupNode(
+                    output_concepts=original_mandatory,
+                    input_concepts=original_mandatory,
+                    environment=environment,
+                    parents=[output],
+                    partial_concepts=output.partial_concepts,
+                    preexisting_conditions=conditions.conditional,
+                    depth=depth,
+                )
         return output
 
     # if we can't find it after expanding to a merge, then

trilogy/core/processing/node_generators/group_node.py

@@ -149,7 +149,9 @@ def gen_group_node(
         g=g,
         depth=depth,
         source_concepts=source_concepts,
-        log_lambda=create_log_lambda(LOGGER_PREFIX, depth, logger),
+        log_lambda=create_log_lambda(
+            LOGGER_PREFIX + f" for {concept.address}", depth, logger
+        ),
         history=history,
         conditions=conditions,
     )

trilogy/core/processing/node_generators/rowset_node.py

@@ -117,7 +117,11 @@ def gen_rowset_node(
         f"{padding(depth)}{LOGGER_PREFIX} final output is {[x.address for x in node.output_concepts]}"
     )
     if not local_optional or all(
-        x.address in node.output_concepts and x.address not in node.partial_concepts
+        (
+            x.address in node.output_concepts
+            or (z in x.pseudonyms for z in node.output_concepts)
+        )
+        and x.address not in node.partial_concepts
         for x in local_optional
     ):
         logger.info(

trilogy/core/processing/nodes/group_node.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 from typing import List, Optional
 
 from trilogy.constants import logger
-from trilogy.core.enums import SourceType
+from trilogy.core.enums import Purpose, SourceType
 from trilogy.core.models.build import (
     BuildComparison,
     BuildConcept,
@@ -78,7 +78,9 @@ class GroupNode(StrategyNode):
         downstream_concepts: List[BuildConcept],
         parents: list[QueryDatasource | BuildDatasource],
         environment: BuildEnvironment,
+        depth: int = 0,
     ) -> GroupRequiredResponse:
+        padding = "\t" * depth
         target_grain = BuildGrain.from_concepts(
             downstream_concepts,
             environment=environment,
@@ -95,12 +97,67 @@ class GroupNode(StrategyNode):
         lookups: list[BuildConcept | str] = [
             concept_map[x] if x in concept_map else x for x in comp_grain.components
         ]
+
         comp_grain = BuildGrain.from_concepts(lookups, environment=environment)
+
         # dynamically select if we need to group
         # because sometimes, we are already at required grain
         if comp_grain.issubset(target_grain):
+
+            logger.info(
+                f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, is subset, no grain required"
+            )
             return GroupRequiredResponse(target_grain, comp_grain, False)
+        # find out what extra is in the comp grain vs target grain
+        difference = [
+            environment.concepts[c] for c in (comp_grain - target_grain).components
+        ]
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, difference {difference}"
+        )
+
+        # if the difference is all unique properties whose keys are in the source grain
+        # we can also suppress the group
+        if all(
+            [
+                x.purpose == Purpose.UNIQUE_PROPERTY
+                and x.keys
+                and all(z in comp_grain.components for z in x.keys)
+                for x in difference
+            ]
+        ):
+            logger.info(
+                f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
+            )
+            return GroupRequiredResponse(target_grain, comp_grain, False)
+        if all([x.purpose == Purpose.KEY for x in difference]):
+            logger.info(
+                f"{padding}{LOGGER_PREFIX} checking if downstream is unique properties of key"
+            )
+            replaced_grain_raw: list[set[str]] = [
+                (
+                    x.keys or set()
+                    if x.purpose == Purpose.UNIQUE_PROPERTY
+                    else set([x.address])
+                )
+                for x in downstream_concepts
+                if x.address in target_grain.components
+            ]
+            # flatten the list of lists
+            replaced_grain = [
+                item for sublist in replaced_grain_raw for item in sublist
+            ]
+            # if the replaced grain is a subset of the comp grain, we can skip the group
+            unique_grain_comp = BuildGrain.from_concepts(
+                replaced_grain, environment=environment
+            )
+            if comp_grain.issubset(unique_grain_comp):
+                logger.info(
+                    f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
+                )
+                return GroupRequiredResponse(target_grain, comp_grain, False)
 
+        logger.info(f"{padding}{LOGGER_PREFIX} Group requirement check: group required")
         return GroupRequiredResponse(target_grain, comp_grain, True)
 
     def _resolve(self) -> QueryDatasource:

trilogy/core/processing/nodes/merge_node.py

@@ -296,9 +296,21 @@ class MergeNode(StrategyNode):
                 return dataset
 
         pregrain = BuildGrain()
+
         for source in final_datasets:
+            if all(
+                [x.address in self.existence_concepts for x in source.output_concepts]
+            ):
+                logger.info(
+                    f"{self.logging_prefix}{LOGGER_PREFIX} skipping existence only source with {source.output_concepts} from grain accumulation"
+                )
+                continue
             pregrain += source.grain
 
+        pregrain = BuildGrain.from_concepts(
+            pregrain.components, environment=self.environment
+        )
+
         grain = self.grain if self.grain else pregrain
         logger.info(
             f"{self.logging_prefix}{LOGGER_PREFIX} has pre grain {pregrain} and final merge node grain {grain}"
@@ -310,6 +322,7 @@ class MergeNode(StrategyNode):
             )
         else:
             joins = []
+
         logger.info(
             f"{self.logging_prefix}{LOGGER_PREFIX} Final join count for CTE parent count {len(join_candidates)} is {len(joins)}"
         )
@@ -343,7 +356,6 @@ class MergeNode(StrategyNode):
         nullable_concepts = find_nullable_concepts(
             source_map=source_map, joins=joins, datasources=final_datasets
         )
-
         qds = QueryDatasource(
             input_concepts=unique(self.input_concepts, "address"),
             output_concepts=unique(self.output_concepts, "address"),

trilogy/parsing/common.py

@@ -40,6 +40,7 @@ from trilogy.core.models.author import (
     UndefinedConcept,
     WhereClause,
     WindowItem,
+    address_with_namespace,
 )
 from trilogy.core.models.core import DataType, arg_to_datatype
 from trilogy.core.models.environment import Environment
@@ -255,7 +256,7 @@ def concepts_to_grain_concepts(
     return v2
 
 
-def get_relevant_parent_concepts(arg) -> tuple[list[ConceptRef], bool]:
+def _get_relevant_parent_concepts(arg) -> tuple[list[ConceptRef], bool]:
     from trilogy.core.models.author import get_concept_arguments
 
     is_metric = False
@@ -270,9 +271,16 @@ def get_relevant_parent_concepts(arg) -> tuple[list[ConceptRef], bool]:
         return [], True
     elif isinstance(arg, AggregateWrapper) and arg.by:
         return arg.by, True
+    elif isinstance(arg, FunctionCallWrapper):
+        return get_relevant_parent_concepts(arg.content)
     return get_concept_arguments(arg), False
 
 
+def get_relevant_parent_concepts(arg):
+    results = _get_relevant_parent_concepts(arg)
+    return results
+
+
 def function_to_concept(
     parent: Function,
     name: str,
@@ -415,10 +423,31 @@ def window_item_to_concept(
     metadata: Metadata | None = None,
 ) -> Concept:
     fmetadata = metadata or Metadata()
+    # if isinstance(
+    #     parent.content,
+    #     (
+    #         AggregateWrapper
+    #         | FunctionCallWrapper
+    #         | WindowItem
+    #         | FilterItem
+    #         | Function
+    #         | ListWrapper
+    #         | MapWrapper
+    #     ),
+    # ):
+    #     new_parent = arbitrary_to_concept(
+    #         parent.content, environment=environment, namespace=namespace
+    #     )
+    #     environment.add_concept(new_parent)
+    #     parent = parent.model_copy(update={"content": new_parent.reference})
+
+    if not isinstance(parent.content, ConceptRef):
+        raise NotImplementedError(
+            f"Window function wiht non ref content {parent.content} not yet supported"
+        )
     bcontent = environment.concepts[parent.content.address]
     if isinstance(bcontent, UndefinedConcept):
         return UndefinedConcept(address=f"{namespace}.{name}", metadata=fmetadata)
-
     if bcontent.purpose == Purpose.METRIC:
         local_purpose, keys = get_purpose_and_keys(None, (bcontent,), environment)
     else:
@@ -534,38 +563,58 @@ def align_item_to_concept(
     return new
 
 
+def rowset_concept(
+    orig_address: ConceptRef,
+    environment: Environment,
+    rowset: RowsetDerivationStatement,
+    pre_output: list[Concept],
+    orig: dict[str, Concept],
+    orig_map: dict[str, Concept],
+):
+    orig_concept = environment.concepts[orig_address.address]
+    name = orig_concept.name
+    if isinstance(orig_concept.lineage, FilterItem):
+        if orig_concept.lineage.where == rowset.select.where_clause:
+            name = environment.concepts[orig_concept.lineage.content.address].name
+    base_namespace = (
+        f"{rowset.name}.{orig_concept.namespace}"
+        if orig_concept.namespace != rowset.namespace
+        else rowset.name
+    )
+
+    new_concept = Concept(
+        name=name,
+        datatype=orig_concept.datatype,
+        purpose=orig_concept.purpose,
+        lineage=None,
+        grain=orig_concept.grain,
+        metadata=Metadata(concept_source=ConceptSource.CTE),
+        namespace=base_namespace,
+        keys=orig_concept.keys,
+        derivation=Derivation.ROWSET,
+        granularity=orig_concept.granularity,
+        pseudonyms={
+            address_with_namespace(x, rowset.name) for x in orig_concept.pseudonyms
+        },
+    )
+    for x in orig_concept.pseudonyms:
+        new_address = address_with_namespace(x, rowset.name)
+        origa = environment.alias_origin_lookup[x]
+        environment.concepts[new_address] = new_concept
+        environment.alias_origin_lookup[new_address] = origa.model_copy(
+            update={"namespace": f"{rowset.name}.{origa.namespace}"}
+        )
+    orig[orig_concept.address] = new_concept
+    orig_map[new_concept.address] = orig_concept
+    pre_output.append(new_concept)
+
+
 def rowset_to_concepts(rowset: RowsetDerivationStatement, environment: Environment):
     pre_output: list[Concept] = []
     orig: dict[str, Concept] = {}
     orig_map: dict[str, Concept] = {}
     for orig_address in rowset.select.output_components:
-        orig_concept = environment.concepts[orig_address.address]
-        name = orig_concept.name
-        if isinstance(orig_concept.lineage, FilterItem):
-            if orig_concept.lineage.where == rowset.select.where_clause:
-                name = environment.concepts[orig_concept.lineage.content.address].name
-        base_namespace = (
-            f"{rowset.name}.{orig_concept.namespace}"
-            if orig_concept.namespace != rowset.namespace
-            else rowset.name
-        )
-
-        new_concept = Concept(
-            name=name,
-            datatype=orig_concept.datatype,
-            purpose=orig_concept.purpose,
-            lineage=None,
-            grain=orig_concept.grain,
-            # TODO: add proper metadata
-            metadata=Metadata(concept_source=ConceptSource.CTE),
-            namespace=base_namespace,
-            keys=orig_concept.keys,
-            derivation=Derivation.ROWSET,
-            granularity=orig_concept.granularity,
-        )
-        orig[orig_concept.address] = new_concept
-        orig_map[new_concept.address] = orig_concept
-        pre_output.append(new_concept)
+        rowset_concept(orig_address, environment, rowset, pre_output, orig, orig_map)
     select_lineage = rowset.select.as_lineage(environment)
     for x in pre_output:
         x.lineage = RowsetItem(

trilogy/parsing/parse_engine.py

@@ -252,6 +252,8 @@ def rehydrate_lineage(
         )
         return lineage
     elif isinstance(lineage, WindowItem):
+        # this is temporarily guaranteed until we do some upstream work
+        assert isinstance(lineage.content, ConceptRef)
         lineage.content.datatype = environment.concepts[
             lineage.content.address
         ].datatype
@@ -542,6 +544,10 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def concept_property_declaration(self, meta: Meta, args) -> Concept:
+        unique = False
+        if not args[0] == Purpose.PROPERTY:
+            unique = True
+            args = args[1:]
         metadata = Metadata()
         modifiers = []
         for arg in args:
@@ -569,7 +575,7 @@ class ParseToObjects(Transformer):
         concept = Concept(
             name=name,
             datatype=args[2],
-            purpose=args[0],
+            purpose=Purpose.PROPERTY if not unique else Purpose.UNIQUE_PROPERTY,
             metadata=metadata,
             grain=Grain(components={x.address for x in parents}),
             namespace=namespace,
@@ -633,7 +639,8 @@ class ParseToObjects(Transformer):
             source_value = source_value.content
 
         if isinstance(
-            source_value, (FilterItem, WindowItem, AggregateWrapper, Function)
+            source_value,
+            (FilterItem, WindowItem, AggregateWrapper, Function, FunctionCallWrapper),
         ):
             concept = arbitrary_to_concept(
                 source_value,

trilogy/parsing/trilogy.lark

@@ -21,7 +21,8 @@
     concept_declaration: PURPOSE IDENTIFIER data_type concept_nullable_modifier? metadata?
     //customer_id.property first_name STRING;
     //<customer_id,country>.property local_alias STRING
-    concept_property_declaration: PROPERTY (prop_ident | IDENTIFIER) data_type concept_nullable_modifier? metadata?
+    UNIQUE: "UNIQUE"i
+    concept_property_declaration: UNIQUE? PROPERTY (prop_ident | IDENTIFIER) data_type concept_nullable_modifier? metadata?
     //metric post_length <- len(post_text);
     concept_derivation:  (PURPOSE | AUTO | PROPERTY ) (prop_ident | IDENTIFIER) "<-" expr