pytrilogy 0.0.3.99__py3-none-any.whl → 0.0.3.100__py3-none-any.whl

{pytrilogy-0.0.3.99.dist-info → pytrilogy-0.0.3.100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.99
+Version: 0.0.3.100
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -123,15 +123,12 @@ Versus SQL, Trilogy aims to:
 **Improve:**
 - Simplicity
 - Refactoring/maintainability
-- Reusability
+- Reusability/composability
+- Expressivness
 
 **Maintain:**
 - Acceptable performance
 
-**Remove:**
-- Lower-level procedural features
-- Transactional optimizations/non-analytics features
-
 ## Backend Support
 
 | Backend | Status | Notes |
@@ -161,6 +158,7 @@ property sentence_id.word_three string::word; # a description to it
 # comments in other places are just comments
 
 # define our datasource to bind the model to data
+# for most work, you can import something already defined
 # testing using query fixtures is a common pattern
 datasource word_one(
     sentence: sentence_id,
@@ -323,7 +321,7 @@ from pytrilogy import Executor, Dialect
 
 ### Authoring Imports
 
-Are also stable, and should be used for cases which programatically generate Trilogy statements without a base text format 
+Are also stable, and should be used for cases which programatically generate Trilogy statements without text inputs
 or need to process/transform parsed code in more complicated ways.
 
 ```python
@@ -391,6 +389,16 @@ datasource <name>(
 )
 grain(<concept>, <concept>)
 address <table>;
+
+datasource orders(
+    order_id,
+    order_date,
+    total_rev: point_of_sale_rev,
+    customomer_id: customer.id
+)
+grain orders
+address orders;
+
 ```
 
 ### Queries

{pytrilogy-0.0.3.99.dist-info → pytrilogy-0.0.3.100.dist-info}/RECORD

@@ -1,6 +1,6 @@
-pytrilogy-0.0.3.99.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=_mu-EP3oMYeyuDb0ESiaTOvq4vYDFOgPukUj0Nk4pwI,303
-trilogy/constants.py,sha256=SSsRMg9HTou259nMKAw-rJNBgzkWjQ3QIQXcrq9i5Kk,1717
+pytrilogy-0.0.3.100.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=g9zY8214LmrIQyjxs_q4DVvbSXccQFQwFe0o0_PZcD8,304
+trilogy/constants.py,sha256=ohmro6so7PPNp2ruWQKVc0ijjXYPOyRrxB9LI8dr3TU,1746
 trilogy/engine.py,sha256=3MiADf5MKcmxqiHBuRqiYdsXiLj7oitDfVvXvHrfjkA,2178
 trilogy/executor.py,sha256=KgCAQhHPT-j0rPkBbALX0f84W9-Q-bkjHayGuavg99w,16490
 trilogy/parser.py,sha256=o4cfk3j3yhUFoiDKq9ZX_GjBF3dKhDjXEwb63rcBkBM,293
@@ -28,45 +28,44 @@ trilogy/core/models/build_environment.py,sha256=mpx7MKGc60fnZLVdeLi2YSREy7eQbQYy
 trilogy/core/models/core.py,sha256=EofJ8-kltNr_7oFhyCPqauVX1bSJzJI5xOp0eMP_vlA,12892
 trilogy/core/models/datasource.py,sha256=wogTevZ-9CyUW2a8gjzqMCieircxi-J5lkI7EOAZnck,9596
 trilogy/core/models/environment.py,sha256=hwTIRnJgaHUdCYof7U5A9NPitGZ2s9yxqiW5O2SaJ9Y,28759
-trilogy/core/models/execute.py,sha256=lQTpiuNhBT4In-oQ76ImgIoTdUbs4mmyd0J0iTOZOdw,42105
+trilogy/core/models/execute.py,sha256=lsNzNjS3nZvoW5CHjYwxDTwBe502NZyytpK1eq8CwW4,42357
 trilogy/core/optimizations/__init__.py,sha256=YH2-mGXZnVDnBcWVi8vTbrdw7Qs5TivG4h38rH3js_I,290
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
 trilogy/core/optimizations/inline_datasource.py,sha256=2sWNRpoRInnTgo9wExVT_r9RfLAQHI57reEV5cGHUcg,4329
 trilogy/core/optimizations/predicate_pushdown.py,sha256=g4AYE8Aw_iMlAh68TjNXGP754NTurrDduFECkUjoBnc,9399
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=Izo7yfR6sGHTaD17lN7ZzGYCtXA5AkXAmIp_OBjHH58,23161
-trilogy/core/processing/discovery_loop.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/core/processing/concept_strategies_v3.py,sha256=tvSN_aiqb1H7LkTl96vj7YK_DKcq_1nDdRJ69wZCLc8,22158
 trilogy/core/processing/discovery_node_factory.py,sha256=5QVYUsci_h6iYWhS0GCoDow2tSAipiBW1OyTRX-g_L8,15581
-trilogy/core/processing/discovery_utility.py,sha256=eY4n7_r6_R-cx_Sm8FiouMXh78v2iO2SGhi0aI5jvDg,4549
+trilogy/core/processing/discovery_utility.py,sha256=Xntgug6VnEF96uw5Zwen1qMEUwKjqrm_ZDUr4i4tc1U,5595
 trilogy/core/processing/discovery_validation.py,sha256=eZ4HfHMpqZLI8MGG2jez8arS8THs6ceuVrQFIY6gXrU,5364
 trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
 trilogy/core/processing/utility.py,sha256=PGQKZgX58kI3gG4nprY8HRGayc2D8fF5RmbvBhQj8ok,23319
 trilogy/core/processing/node_generators/__init__.py,sha256=iVJ-crowPxYeut-hFjyEjfibKIDq7PfB4LEuDAUCjGY,943
-trilogy/core/processing/node_generators/basic_node.py,sha256=TLZCv4WS196a-0g5xgKuJGthnGP8Ugm46iz85_3NIY4,5626
+trilogy/core/processing/node_generators/basic_node.py,sha256=0Uhnf07056SBbRkt-wYLw4DZqsFR6jztGLUaE9ebPZs,5577
 trilogy/core/processing/node_generators/common.py,sha256=PdysdroW9DUADP7f5Wv_GKPUyCTROZV1g3L45fawxi8,9443
 trilogy/core/processing/node_generators/constant_node.py,sha256=LfpDq2WrBRZ3tGsLxw77LuigKfhbteWWh9L8BGdMGwk,1146
 trilogy/core/processing/node_generators/filter_node.py,sha256=ArBsQJl-4fWBJWCE28CRQ7UT7ErnFfbcseoQQZrBodY,11220
-trilogy/core/processing/node_generators/group_node.py,sha256=8HJ1lkOvIXfX3xoS2IMbM_wCu_mT0J_hQ7xnTaxsVlo,6611
+trilogy/core/processing/node_generators/group_node.py,sha256=pq8aqKe4hCtkzFtpHvE15BJoYvpveoe50_2IM1pqjIQ,6732
 trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=DfivU4xAo3iOFvuZxmztlD5AlusM_Nq0IOOgvPKMH_M,22964
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=531ptEAZIczc7PR-kfuNe_KBaDToyIMUMKq4bkoZkgw,23561
 trilogy/core/processing/node_generators/recursive_node.py,sha256=l5zdh0dURKwmAy8kK4OpMtZfyUEQRk6N-PwSWIyBpSM,2468
 trilogy/core/processing/node_generators/rowset_node.py,sha256=5L5u6xz1In8EaHQdcYgR2si-tz9WB9YLXURo4AkUT9A,6630
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=KQvGoNT5ZBWQ_caEomRTtG1PKZC7OPT4PKfY0QmwMGE,22270
 trilogy/core/processing/node_generators/select_node.py,sha256=Ta1G39V94gjX_AgyZDz9OqnwLz4BjY3D6Drx9YpziMQ,3555
 trilogy/core/processing/node_generators/synonym_node.py,sha256=AnAsa_Wj50NJ_IK0HSgab_7klYmKVrv0WI1uUe-GvEY,3766
 trilogy/core/processing/node_generators/union_node.py,sha256=VNo6Oey4p8etU9xrOh2oTT2lIOTvY6PULUPRvVa2uxU,2877
-trilogy/core/processing/node_generators/unnest_node.py,sha256=ueOQtoTf2iJHO09RzWHDFQ5iKZq2fVhGf2KAF2U2kU8,2677
+trilogy/core/processing/node_generators/unnest_node.py,sha256=w9vhPzASz53QPASLqFcLDdR9eY132tgVUcp3QolD5Jw,3726
 trilogy/core/processing/node_generators/window_node.py,sha256=A90linr4pkZtTNfn9k2YNLqrJ_SFII3lbHxB-BC6mI8,6688
 trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=m2YQ4OmG0N2O61a7NEq1ZzbTa7JsCC00lxB2ymjcYRI,8224
 trilogy/core/processing/nodes/__init__.py,sha256=zTge1EzwzEydlcMliIFO_TT7h7lS8l37lyZuQDir1h0,5487
 trilogy/core/processing/nodes/base_node.py,sha256=C_CjlOzlGMXckyV0b_PJZerpopNesRCKfambMq7Asvc,18221
 trilogy/core/processing/nodes/filter_node.py,sha256=5VtRfKbCORx0dV-vQfgy3gOEkmmscL9f31ExvlODwvY,2461
-trilogy/core/processing/nodes/group_node.py,sha256=ZJ9LRF1sDOyel5v0MHXHORn6DYdn771nX0-KdHdt3-4,10517
-trilogy/core/processing/nodes/merge_node.py,sha256=02oWRca0ba41U6PSAB14jwnWWxoyrvxRPLwkli259SY,15865
+trilogy/core/processing/nodes/group_node.py,sha256=njz-5T7OJ3-kaBC7EhdtPra3G77HnI7apjUwMGhUeXo,10569
+trilogy/core/processing/nodes/merge_node.py,sha256=daJywBxh44Gqk-7eTiXbYtY7xo6O6fNvqX-DagTOTmE,16231
 trilogy/core/processing/nodes/recursive_node.py,sha256=k0rizxR8KE64ievfHx_GPfQmU8QAP118Laeyq5BLUOk,1526
-trilogy/core/processing/nodes/select_node_v2.py,sha256=Xyfq8lU7rP7JTAd8VV0ATDNal64n4xIBgWQsOuMe_Ak,8824
+trilogy/core/processing/nodes/select_node_v2.py,sha256=IWyKyNgFlV8A2S1FUTPdIaogg6PzaHh-HmQo6v24sbg,8862
 trilogy/core/processing/nodes/union_node.py,sha256=hLAXXVWqEgMWi7dlgSHfCF59fon64av14-uPgJzoKzM,1870
 trilogy/core/processing/nodes/unnest_node.py,sha256=oLKMMNMx6PLDPlt2V5neFMFrFWxET8r6XZElAhSNkO0,2181
 trilogy/core/processing/nodes/window_node.py,sha256=JXJ0iVRlSEM2IBr1TANym2RaUf_p5E_l2sNykRzXWDo,1710
@@ -118,8 +117,8 @@ trilogy/std/money.preql,sha256=XWwvAV3WxBsHX9zfptoYRnBigcfYwrYtBHXTME0xJuQ,2082
 trilogy/std/net.preql,sha256=WZCuvH87_rZntZiuGJMmBDMVKkdhTtxeHOkrXNwJ1EE,416
 trilogy/std/ranking.preql,sha256=LDoZrYyz4g3xsII9XwXfmstZD-_92i1Eox1UqkBIfi8,83
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.99.dist-info/METADATA,sha256=Symb6C1gs-2By5qBIt51posuNDI3_en-b84YtlWKTiU,11683
-pytrilogy-0.0.3.99.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pytrilogy-0.0.3.99.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.99.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.99.dist-info/RECORD,,
+pytrilogy-0.0.3.100.dist-info/METADATA,sha256=T9p4b_yjL4_HtEwChltpxh5mlP8pCoRQPn28Ucu_1gI,11811
+pytrilogy-0.0.3.100.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytrilogy-0.0.3.100.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.100.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.100.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.99"
+__version__ = "0.0.3.100"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -40,6 +40,7 @@ class Comments:
     joins: bool = True
     nullable: bool = True
     partial: bool = True
+    source_map: bool = False
 
 
 @dataclass

trilogy/core/models/execute.py

@@ -118,19 +118,20 @@ class CTE(BaseModel):
         base += f" Source: {self.source.source_type}."
         if self.parent_ctes:
             base += f" References: {', '.join([x.name for x in self.parent_ctes])}."
-        if self.joins:
+        if self.joins and CONFIG.comments.joins:
             base += f"\n-- Joins: {', '.join([str(x) for x in self.joins])}."
-        if self.partial_concepts:
+        if self.partial_concepts and CONFIG.comments.partial:
             base += (
                 f"\n-- Partials: {', '.join([str(x) for x in self.partial_concepts])}."
             )
-        base += f"\n-- Source Map: {self.source_map}."
+        if CONFIG.comments.source_map:
+            base += f"\n-- Source Map: {self.source_map}."
         base += f"\n-- Output: {', '.join([str(x) for x in self.output_columns])}."
         if self.source.input_concepts:
             base += f"\n-- Inputs: {', '.join([str(x) for x in self.source.input_concepts])}."
         if self.hidden_concepts:
             base += f"\n-- Hidden: {', '.join([str(x) for x in self.hidden_concepts])}."
-        if self.nullable_concepts:
+        if self.nullable_concepts and CONFIG.comments.nullable:
             base += (
                 f"\n-- Nullable: {', '.join([str(x) for x in self.nullable_concepts])}."
             )
@@ -368,6 +369,7 @@ class CTE(BaseModel):
     @property
     def group_concepts(self) -> List[BuildConcept]:
         def check_is_not_in_group(c: BuildConcept):
+
             if len(self.source_map.get(c.address, [])) > 0:
                 return False
             if c.derivation == Derivation.ROWSET:
@@ -381,8 +383,6 @@ class CTE(BaseModel):
                 and c.lineage.operator in FunctionClass.AGGREGATE_FUNCTIONS.value
             ):
                 return True
-            if c.purpose == Purpose.METRIC:
-                return True
 
             if c.derivation == Derivation.BASIC and c.lineage:
                 if all([check_is_not_in_group(x) for x in c.lineage.concept_arguments]):
@@ -392,6 +392,10 @@ class CTE(BaseModel):
                     and c.lineage.operator == FunctionType.GROUP
                 ):
                     return check_is_not_in_group(c.lineage.concept_arguments[0])
+                return False
+            if c.purpose == Purpose.METRIC:
+                return True
+
             return False
 
         return (
@@ -707,6 +711,8 @@ class QueryDatasource(BaseModel):
             f" {[c.address for c in self.output_concepts]} concepts and"
             f" {other.name} with {[c.address for c in other.output_concepts]} concepts"
         )
+        logger.info(self.source_map)
+        logger.info(other.source_map)
 
         merged_datasources: dict[str, Union[BuildDatasource, "QueryDatasource"]] = {}
 
@@ -770,6 +776,7 @@ class QueryDatasource(BaseModel):
         logger.debug(
             f"[Query Datasource] merged with {[c.address for c in qds.output_concepts]} concepts"
         )
+        logger.debug(qds.source_map)
         return qds
 
     @property
@@ -777,7 +784,7 @@ class QueryDatasource(BaseModel):
         filters = abs(hash(str(self.condition))) if self.condition else ""
         grain = "_".join([str(c).replace(".", "_") for c in self.grain.components])
         group = ""
-        if self.source_type == SourceType.GROUP:
+        if self.group_required:
             keys = [
                 x.address for x in self.output_concepts if x.purpose != Purpose.METRIC
             ]

trilogy/core/processing/concept_strategies_v3.py

@@ -19,13 +19,13 @@ from trilogy.core.processing.discovery_utility import (
     LOGGER_PREFIX,
     depth_to_prefix,
     get_priority_concept,
+    group_if_required,
 )
 from trilogy.core.processing.discovery_validation import (
     ValidationResult,
     validate_stack,
 )
 from trilogy.core.processing.nodes import (
-    GroupNode,
     History,
     MergeNode,
     StrategyNode,
@@ -218,6 +218,7 @@ def initialize_loop_context(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} derived condition row inputs {[x.address for x in required_filters]} present in mandatory list, forcing condition evaluation at this level. "
             )
             mandatory_list = completion_mandatory
+            all_mandatory = set(c.address for c in completion_mandatory)
             must_evaluate_condition_on_this_level_not_push_down = True
         else:
             logger.info(
@@ -263,7 +264,7 @@ def evaluate_loop_conditions(
             ]
         ) and not any(
             [
-                x.derivation not in ROOT_DERIVATIONS
+                x.derivation not in ROOT_DERIVATIONS + [Derivation.BASIC]
                 for x in context.mandatory_list
                 if x.address not in context.conditions.row_arguments
             ]
@@ -282,7 +283,7 @@ def evaluate_loop_conditions(
     # to ensure filtering happens before something like a SUM
     if (
         context.conditions
-        and priority_concept.derivation not in ROOT_DERIVATIONS
+        and priority_concept.derivation not in ROOT_DERIVATIONS + [Derivation.BASIC]
         and priority_concept.address not in context.conditions.row_arguments
     ):
         logger.info(
@@ -419,26 +420,9 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
         logger.info(
             f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Conditions {context.conditions} were injected, checking if we need a group to restore grain"
         )
-
-        result = GroupNode.check_if_required(
-            downstream_concepts=output.usable_outputs,
-            parents=[output.resolve()],
-            environment=context.environment,
-            depth=context.depth,
+        return group_if_required(
+            output, context.original_mandatory, context.environment
         )
-        if result.required:
-            logger.info(
-                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Adding group node with outputs {[x.address for x in context.original_mandatory]}"
-            )
-            return GroupNode(
-                output_concepts=context.original_mandatory,
-                input_concepts=output.usable_outputs,
-                environment=context.environment,
-                parents=[output],
-                partial_concepts=output.partial_concepts,
-                preexisting_conditions=context.conditions.conditional,
-                depth=context.depth,
-            )
     return output
 
 
@@ -604,18 +588,4 @@ def source_query_concepts(
     logger.info(
         f"{depth_to_prefix(0)}{LOGGER_PREFIX} final concepts are {[x.address for x in final]}"
     )
-    if GroupNode.check_if_required(
-        downstream_concepts=final,
-        parents=[root.resolve()],
-        environment=environment,
-    ).required:
-        candidate: StrategyNode = GroupNode(
-            output_concepts=final,
-            input_concepts=final,
-            environment=environment,
-            parents=[root],
-            partial_concepts=root.partial_concepts,
-        )
-    else:
-        candidate = root
-    return candidate
+    return group_if_required(root, output_concepts, environment)

trilogy/core/processing/discovery_utility.py

@@ -6,6 +6,8 @@ from trilogy.core.models.build import (
     BuildConcept,
     BuildRowsetItem,
 )
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.nodes import GroupNode, MergeNode, StrategyNode
 
 
 def depth_to_prefix(depth: int) -> str:
@@ -15,6 +17,34 @@ def depth_to_prefix(depth: int) -> str:
 LOGGER_PREFIX = "[DISCOVERY LOOP]"
 
 
+def group_if_required(
+    root: StrategyNode, final: List[BuildConcept], environment: BuildEnvironment
+):
+    if isinstance(root, MergeNode) and root.force_group is True:
+        return root
+    elif isinstance(root, GroupNode):
+        return root
+    elif GroupNode.check_if_required(
+        downstream_concepts=final,
+        parents=[root.resolve()],
+        environment=environment,
+    ).required:
+        if isinstance(root, MergeNode):
+            root.force_group = True
+            root.set_output_concepts(final, rebuild=False)
+            root.rebuild_cache()
+            return root
+        return GroupNode(
+            output_concepts=final,
+            input_concepts=final,
+            environment=environment,
+            parents=[root],
+            partial_concepts=root.partial_concepts,
+            preexisting_conditions=root.preexisting_conditions,
+        )
+    return root
+
+
 def get_upstream_concepts(base: BuildConcept, nested: bool = False) -> set[str]:
     upstream = set()
     if nested:

trilogy/core/processing/node_generators/basic_node.py

@@ -120,8 +120,8 @@ def gen_basic_node(
             f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"
         )
         return None
-
-    parent_node.source_type = SourceType.BASIC
+    if parent_node.source_type != SourceType.CONSTANT:
+        parent_node.source_type = SourceType.BASIC
     parent_node.add_output_concept(concept)
     for x in equivalent_optional:
         parent_node.add_output_concept(x)
@@ -129,24 +129,18 @@ def gen_basic_node(
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: output {[x.address for x in parent_node.output_concepts]}"
     )
+    # if it's a constant, don't prune outputs
+    if parent_node.source_type == SourceType.CONSTANT:
+        return parent_node
     targets = [concept] + local_optional + equivalent_optional
-    should_hide = [
-        x
-        for x in parent_node.output_concepts
-        if (
-            x.address not in targets
-            and not any(x.address in y.pseudonyms for y in targets)
-        )
-    ]
-    parent_node.hide_output_concepts(should_hide)
-    should_not_hide = [
-        x
-        for x in parent_node.output_concepts
-        if x.address in targets or any(x.address in y.pseudonyms for y in targets)
-    ]
-    parent_node.unhide_output_concepts(should_not_hide)
+    targets = [
+        s
+        for s in parent_node.output_concepts
+        if any(s.address in y.pseudonyms for y in targets)
+    ] + targets
+    parent_node.set_output_concepts(targets)
 
     logger.info(
-        f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: output {[x.address for x in parent_node.output_concepts]} hidden {[x for x in parent_node.hidden_concepts]}"
+        f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: input: {[x.address for x in parent_node.input_concepts]} output {[x.address for x in parent_node.output_concepts]} hidden {[x for x in parent_node.hidden_concepts]}"
     )
     return parent_node

trilogy/core/processing/node_generators/group_node.py

@@ -108,13 +108,14 @@ def gen_group_node(
                         f"{padding(depth)}{LOGGER_PREFIX} cannot include optional agg {possible_agg.address}; it has mismatched parent grain {comp_grain } vs local parent {build_grain_parents}"
                     )
     if parent_concepts:
+        target_grain = BuildGrain.from_concepts(parent_concepts)
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} fetching group node parents {LooseBuildConceptList(concepts=parent_concepts)}"
+            f"{padding(depth)}{LOGGER_PREFIX} fetching group node parents {LooseBuildConceptList(concepts=parent_concepts)} with expected grain {target_grain}"
         )
         parent_concepts = unique(
             [x for x in parent_concepts if not x.name == ALL_ROWS_CONCEPT], "address"
         )
-        parent = source_concepts(
+        parent: StrategyNode | None = source_concepts(
             mandatory_list=parent_concepts,
             environment=environment,
             g=g,

trilogy/core/processing/node_generators/node_merge_node.py

@@ -20,6 +20,9 @@ from trilogy.core.models.build import (
     BuildWhereClause,
 )
 from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.discovery_utility import (
+    group_if_required,
+)
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
 from trilogy.core.processing.utility import padding
 from trilogy.utility import unique
@@ -510,6 +513,7 @@ def subgraphs_to_merge_node(
     search_conditions: BuildWhereClause | None = None,
     enable_early_exit: bool = True,
 ):
+    target_grain = BuildGrain.from_concepts(output_concepts, environment=environment)
     parents: List[StrategyNode] = []
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} fetching subgraphs {[[c.address for c in subgraph] for subgraph in concept_subgraphs]}"
@@ -549,16 +553,20 @@ def subgraphs_to_merge_node(
                 output_c.append(y)
 
     if len(parents) == 1 and enable_early_exit:
+
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} only one parent node, exiting early w/ {[c.address for c in parents[0].output_concepts]}"
         )
-        return parents[0]
+        parent = parents[0]
+        return group_if_required(parent, output_concepts, environment)
+
     rval = MergeNode(
         input_concepts=unique(input_c, "address"),
-        output_concepts=output_c,
+        output_concepts=output_concepts,
         environment=environment,
         parents=parents,
         depth=depth,
+        grain=target_grain,
         # hidden_concepts=[]
         # conditions=conditions,
         # conditions=search_conditions.conditional,
@@ -579,6 +587,12 @@ def gen_merge_node(
     conditions: BuildConditional | None = None,
     search_conditions: BuildWhereClause | None = None,
 ) -> Optional[MergeNode]:
+
+    # we do not actually APPLY these conditions anywhere
+    # though we could look at doing that as an optimization
+    # it's important to include them so the base discovery loop that was generating
+    # the merge node can then add them automatically
+    # so we should not return a node with preexisting conditions
     if search_conditions:
         all_search_concepts = unique(
             all_concepts + list(search_conditions.row_arguments), "address"

trilogy/core/processing/node_generators/unnest_node.py

@@ -33,6 +33,8 @@ def gen_unnest_node(
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} with lineage {concept.lineage} has parents + optional {all_parents} and equivalent optional {equivalent_optional}"
     )
+    local_conditions = False
+    expected_outputs = [concept] + local_optional
     if arguments or local_optional:
         parent = source_concepts(
             mandatory_list=all_parents,
@@ -47,24 +49,49 @@ def gen_unnest_node(
                 f"{padding(depth)}{LOGGER_PREFIX} could not find unnest node parents"
             )
             return None
+    elif conditions:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} unnest node has no parents but conditions inputs {conditions.row_arguments} vs expected output {expected_outputs}"
+        )
+        if all([x.address in expected_outputs for x in conditions.row_arguments]):
+            local_conditions = True
+        else:
+            parent = source_concepts(
+                mandatory_list=conditions.conditional.row_arguments,
+                environment=environment,
+                g=g,
+                depth=depth + 1,
+                history=history,
+                conditions=conditions,
+            )
+            if not parent:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} could not find unnest node condition inputs with no parents"
+                )
+                return None
+    else:
+        parent = None
 
     base = UnnestNode(
         unnest_concepts=[concept] + equivalent_optional,
         input_concepts=arguments + non_equivalent_optional,
         output_concepts=[concept] + local_optional,
         environment=environment,
-        parents=([parent] if (arguments or local_optional) else []),
+        parents=([parent] if parent else []),
     )
     # we need to sometimes nest an unnest node,
     # as unnest operations are not valid in all situations
     # TODO: inline this node when we can detect it's safe
+    conditional = conditions.conditional if conditions else None
     new = StrategyNode(
         input_concepts=base.output_concepts,
         output_concepts=base.output_concepts,
         environment=environment,
         parents=[base],
-        # conditions=conditions.conditional if conditions else None,
-        preexisting_conditions=conditions.conditional if conditions else None,
+        conditions=conditional if local_conditions is True else None,
+        preexisting_conditions=(
+            conditional if conditional and local_conditions is False else None
+        ),
     )
     qds = new.resolve()
     assert qds.source_map[concept.address] == {base.resolve()}

trilogy/core/processing/nodes/group_node.py

@@ -113,16 +113,18 @@ class GroupNode(StrategyNode):
             environment.concepts[c] for c in (comp_grain - target_grain).components
         ]
         logger.info(
-            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, difference {difference}"
+            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, difference {[x.address for x in difference]}"
         )
 
         # if the difference is all unique properties whose keys are in the source grain
         # we can also suppress the group
         if all(
             [
-                x.purpose == Purpose.UNIQUE_PROPERTY
-                and x.keys
-                and all(z in comp_grain.components for z in x.keys)
+                x.keys
+                and all(
+                    environment.concepts[z].address in comp_grain.components
+                    for z in x.keys
+                )
                 for x in difference
             ]
         ):

trilogy/core/processing/nodes/merge_node.py

@@ -308,7 +308,13 @@ class MergeNode(StrategyNode):
                     f"{self.logging_prefix}{LOGGER_PREFIX} skipping existence only source with {source.output_concepts} from grain accumulation"
                 )
                 continue
+            logger.info(
+                f"{self.logging_prefix}{LOGGER_PREFIX} adding source grain {source.grain} from source {source.identifier} to pregrain"
+            )
             pregrain += source.grain
+            logger.info(
+                f"{self.logging_prefix}{LOGGER_PREFIX} pregrain is now {pregrain}"
+            )
 
         pregrain = BuildGrain.from_concepts(
             pregrain.components, environment=self.environment
@@ -334,7 +340,9 @@ class MergeNode(StrategyNode):
             if isinstance(join, BaseJoin) and join.join_type == JoinType.FULL:
                 full_join_concepts += join.input_concepts
 
-        if self.whole_grain:
+        if self.force_group is True:
+            force_group = True
+        elif self.whole_grain:
             force_group = False
         elif self.force_group is False:
             force_group = False

trilogy/core/processing/nodes/select_node_v2.py

@@ -221,6 +221,7 @@ class SelectNode(StrategyNode):
 
 
 class ConstantNode(SelectNode):
+    source_type = SourceType.CONSTANT
     """Represents a constant value."""
 
     def copy(self) -> "ConstantNode":