pytrilogy 0.0.3.102__py3-none-any.whl → 0.0.3.104__py3-none-any.whl

trilogy/core/processing/node_generators/filter_node.py

@@ -246,13 +246,6 @@ def gen_filter_node(
         filter_node = parent
     else:
         core_parent_nodes.append(row_parent)
-        filters = [concept] + same_filter_optional
-        parents_for_grain = [
-            x.lineage.content
-            for x in filters
-            if isinstance(x.lineage, BuildFilterItem)
-            and isinstance(x.lineage.content, BuildConcept)
-        ]
         filter_node = FilterNode(
             input_concepts=unique(
                 parent_row_concepts + flattened_existence,
@@ -261,9 +254,6 @@ def gen_filter_node(
             output_concepts=[concept] + same_filter_optional + parent_row_concepts,
             environment=environment,
             parents=core_parent_nodes,
-            grain=BuildGrain.from_concepts(
-                parents_for_grain + parent_row_concepts, environment=environment
-            ),
             preexisting_conditions=conditions.conditional if conditions else None,
         )
 

trilogy/core/processing/node_generators/group_node.py

@@ -108,6 +108,42 @@ def gen_group_node(
                     logger.info(
                         f"{padding(depth)}{LOGGER_PREFIX} cannot include optional agg {possible_agg.address}; it has mismatched parent grain {comp_grain } vs local parent {build_grain_parents}"
                     )
+    elif concept.grain.abstract:
+        for possible_agg in local_optional:
+            if not isinstance(
+                possible_agg.lineage,
+                (BuildAggregateWrapper, BuildFunction),
+            ):
+
+                continue
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} considering optional agg {possible_agg.address} for {concept.address}"
+            )
+            agg_parents = resolve_function_parent_concepts(
+                possible_agg,
+                environment=environment,
+            )
+            comp_grain = get_aggregate_grain(possible_agg, environment)
+            if not possible_agg.grain.abstract:
+                continue
+            if set([x.address for x in agg_parents]).issubset(
+                set([x.address for x in parent_concepts])
+            ):
+                output_concepts.append(possible_agg)
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                )
+            elif comp_grain == get_aggregate_grain(concept, environment):
+                extra = [x for x in agg_parents if x.address not in parent_concepts]
+                parent_concepts += extra
+                output_concepts.append(possible_agg)
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                )
+            else:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} cannot include optional agg {possible_agg.address}; it has mismatched parent grain {comp_grain } vs local parent {get_aggregate_grain(concept, environment)}"
+                )
     if parent_concepts:
         target_grain = BuildGrain.from_concepts(parent_concepts)
         logger.info(

trilogy/core/processing/node_generators/multiselect_node.py

@@ -156,7 +156,7 @@ def gen_multiselect_node(
     possible_joins = concept_to_relevant_joins(additional_relevant)
     if not local_optional:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no enriched required for rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node; exiting early"
         )
         return node
     if not possible_joins:

trilogy/core/processing/node_generators/node_merge_node.py

@@ -20,9 +20,6 @@ from trilogy.core.models.build import (
     BuildWhereClause,
 )
 from trilogy.core.models.build_environment import BuildEnvironment
-from trilogy.core.processing.discovery_utility import (
-    group_if_required,
-)
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
 from trilogy.core.processing.utility import padding
 from trilogy.utility import unique
@@ -510,7 +507,7 @@ def subgraphs_to_merge_node(
     search_conditions: BuildWhereClause | None = None,
     enable_early_exit: bool = True,
 ):
-    target_grain = BuildGrain.from_concepts(output_concepts, environment=environment)
+
     parents: List[StrategyNode] = []
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} fetching subgraphs {[[c.address for c in subgraph] for subgraph in concept_subgraphs]}"
@@ -555,7 +552,7 @@ def subgraphs_to_merge_node(
             f"{padding(depth)}{LOGGER_PREFIX} only one parent node, exiting early w/ {[c.address for c in parents[0].output_concepts]}"
         )
         parent = parents[0]
-        return group_if_required(parent, output_concepts, environment)
+        return parent
 
     rval = MergeNode(
         input_concepts=unique(input_c, "address"),
@@ -563,7 +560,6 @@ def subgraphs_to_merge_node(
         environment=environment,
         parents=parents,
         depth=depth,
-        grain=target_grain,
         # hidden_concepts=[]
         # conditions=conditions,
         # conditions=search_conditions.conditional,

trilogy/core/processing/node_generators/rowset_node.py

@@ -64,14 +64,6 @@ def gen_rowset_node(
         v for v in concept_pool if v.address in rowset_outputs
     ]
 
-    select_hidden = node.hidden_concepts
-    rowset_hidden = [
-        x
-        for x in rowset_relevant
-        if x.address in lineage.rowset.derived_concepts
-        and isinstance(x.lineage, BuildRowsetItem)
-        and x.lineage.content.address in select_hidden
-    ]
     additional_relevant = [
         factory.build(x) for x in select.output_components if x.address in enrichment
     ]
@@ -84,18 +76,6 @@ def gen_rowset_node(
             )
             node.partial_concepts.append(item)
 
-    final_hidden = rowset_hidden + [
-        x
-        for x in node.output_concepts
-        if x.address not in local_optional + [concept]
-        and x.derivation != Derivation.ROWSET
-        and not any(z in lineage.rowset.derived_concepts for z in x.pseudonyms)
-    ]
-    logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} hiding {final_hidden} local optional {local_optional}"
-    )
-    node.hide_output_concepts(final_hidden)
-
     node.grain = BuildGrain.from_concepts(
         [
             x
@@ -112,7 +92,7 @@ def gen_rowset_node(
 
     node.rebuild_cache()
     logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} final output is {[x.address for x in node.output_concepts]}"
+        f"{padding(depth)}{LOGGER_PREFIX} final output is {[x.address for x in node.output_concepts]} with grain {node.grain}"
     )
     if not local_optional or all(
         (

trilogy/core/processing/node_generators/union_node.py

@@ -74,7 +74,7 @@ def gen_union_node(
             history=history,
             conditions=conditions,
         )
-        parent.hide_output_concepts(parent.output_concepts)
+
         parent.add_output_concepts(resolved)
         parent_nodes.append(parent)
         if not parent:

trilogy/core/processing/node_generators/unnest_node.py

@@ -1,9 +1,19 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildFunction,
+    BuildGrain,
+    BuildWhereClause,
+)
 from trilogy.core.models.build_environment import BuildEnvironment
-from trilogy.core.processing.nodes import History, StrategyNode, UnnestNode
+from trilogy.core.processing.nodes import (
+    History,
+    StrategyNode,
+    UnnestNode,
+    WhereSafetyNode,
+)
 from trilogy.core.processing.utility import padding
 
 LOGGER_PREFIX = "[GEN_UNNEST_NODE]"
@@ -71,7 +81,9 @@ def gen_unnest_node(
                 return None
     else:
         parent = None
-
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} got parent {parent}"
+    )
     base = UnnestNode(
         unnest_concepts=[concept] + equivalent_optional,
         input_concepts=arguments + non_equivalent_optional,
@@ -83,7 +95,7 @@ def gen_unnest_node(
     # as unnest operations are not valid in all situations
     # TODO: inline this node when we can detect it's safe
     conditional = conditions.conditional if conditions else None
-    new = StrategyNode(
+    new = WhereSafetyNode(
         input_concepts=base.output_concepts,
         output_concepts=base.output_concepts,
         environment=environment,
@@ -92,9 +104,13 @@ def gen_unnest_node(
         preexisting_conditions=(
             conditional if conditional and local_conditions is False else None
         ),
+        grain=BuildGrain.from_concepts(
+            concepts=base.output_concepts,
+            environment=environment,
+        ),
     )
-    qds = new.resolve()
-    assert qds.source_map[concept.address] == {base.resolve()}
-    for x in equivalent_optional:
-        assert qds.source_map[x.address] == {base.resolve()}
+    # qds = new.resolve()
+    # assert qds.source_map[concept.address] == {base.resolve()}
+    # for x in equivalent_optional:
+    #     assert qds.source_map[x.address] == {base.resolve()}
     return new

trilogy/core/processing/nodes/base_node.py

@@ -285,12 +285,23 @@ class StrategyNode:
             self.rebuild_cache()
         return self
 
-    def set_output_concepts(self, concepts: List[BuildConcept], rebuild: bool = True):
+    def set_visible_concepts(self, concepts: List[BuildConcept]):
+        for x in self.output_concepts:
+            if x.address not in [c.address for c in concepts]:
+                self.hidden_concepts.add(x.address)
+        return self
+
+    def set_output_concepts(
+        self,
+        concepts: List[BuildConcept],
+        rebuild: bool = True,
+        change_visibility: bool = True,
+    ):
         # exit if no changes
         if self.output_concepts == concepts:
             return self
         self.output_concepts = concepts
-        if self.hidden_concepts:
+        if self.hidden_concepts and change_visibility:
             self.hidden_concepts = set(
                 x for x in self.hidden_concepts if x not in concepts
             )
@@ -489,7 +500,6 @@ class WhereSafetyNode(StrategyNode):
             parent = parent.copy()
             # avoid performance hit by not rebuilding until end
             parent.set_output_concepts(self.output_concepts, rebuild=False)
-            parent.hide_output_concepts(self.hidden_concepts, rebuild=False)
 
             # these conditions
             if self.preexisting_conditions:

trilogy/core/processing/nodes/group_node.py

@@ -1,14 +1,12 @@
-from dataclasses import dataclass
 from typing import List, Optional
 
 from trilogy.constants import logger
-from trilogy.core.enums import Purpose, SourceType
+from trilogy.core.enums import SourceType
 from trilogy.core.models.build import (
     BuildComparison,
     BuildConcept,
     BuildConditional,
     BuildDatasource,
-    BuildGrain,
     BuildOrderBy,
     BuildParenthetical,
 )
@@ -18,19 +16,16 @@ from trilogy.core.processing.nodes.base_node import (
     StrategyNode,
     resolve_concept_map,
 )
-from trilogy.core.processing.utility import find_nullable_concepts, is_scalar_condition
+from trilogy.core.processing.utility import (
+    GroupRequiredResponse,
+    find_nullable_concepts,
+    is_scalar_condition,
+)
 from trilogy.utility import unique
 
 LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
 
 
-@dataclass
-class GroupRequiredResponse:
-    target: BuildGrain
-    upstream: BuildGrain
-    required: bool
-
-
 class GroupNode(StrategyNode):
     source_type = SourceType.GROUP
 
@@ -80,92 +75,15 @@ class GroupNode(StrategyNode):
         environment: BuildEnvironment,
         depth: int = 0,
     ) -> GroupRequiredResponse:
-        padding = "\t" * depth
-        target_grain = BuildGrain.from_concepts(
-            downstream_concepts,
-            environment=environment,
-        )
-
-        # the concepts of the souce grain might not exist in the output environment
-        # so we need to construct a new
-        concept_map: dict[str, BuildConcept] = {}
-        comp_grain = BuildGrain()
-        for source in parents:
-            comp_grain += source.grain
-            for x in source.output_concepts:
-                concept_map[x.address] = x
-        lookups: list[BuildConcept | str] = [
-            concept_map[x] if x in concept_map else x for x in comp_grain.components
-        ]
-
-        comp_grain = BuildGrain.from_concepts(lookups, environment=environment)
-
-        # dynamically select if we need to group
-        # because sometimes, we are already at required grain
-        if comp_grain.issubset(target_grain):
-
-            logger.info(
-                f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, grain is subset of target, no group node required"
-            )
-            return GroupRequiredResponse(target_grain, comp_grain, False)
-        # find out what extra is in the comp grain vs target grain
-        difference = [
-            environment.concepts[c] for c in (comp_grain - target_grain).components
-        ]
-        logger.info(
-            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, difference {[x.address for x in difference]}"
-        )
+        from trilogy.core.processing.discovery_utility import check_if_group_required
 
-        # if the difference is all unique properties whose keys are in the source grain
-        # we can also suppress the group
-        if all(
-            [
-                x.keys
-                and all(
-                    environment.concepts[z].address in comp_grain.components
-                    for z in x.keys
-                )
-                for x in difference
-            ]
-        ):
-            logger.info(
-                f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
-            )
-            return GroupRequiredResponse(target_grain, comp_grain, False)
-        if all([x.purpose == Purpose.KEY for x in difference]):
-            logger.info(
-                f"{padding}{LOGGER_PREFIX} checking if downstream is unique properties of key"
-            )
-            replaced_grain_raw: list[set[str]] = [
-                (
-                    x.keys or set()
-                    if x.purpose == Purpose.UNIQUE_PROPERTY
-                    else set([x.address])
-                )
-                for x in downstream_concepts
-                if x.address in target_grain.components
-            ]
-            # flatten the list of lists
-            replaced_grain = [
-                item for sublist in replaced_grain_raw for item in sublist
-            ]
-            # if the replaced grain is a subset of the comp grain, we can skip the group
-            unique_grain_comp = BuildGrain.from_concepts(
-                replaced_grain, environment=environment
-            )
-            if comp_grain.issubset(unique_grain_comp):
-                logger.info(
-                    f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
-                )
-                return GroupRequiredResponse(target_grain, comp_grain, False)
-
-        logger.info(f"{padding}{LOGGER_PREFIX} Group requirement check: group required")
-        return GroupRequiredResponse(target_grain, comp_grain, True)
+        return check_if_group_required(downstream_concepts, parents, environment, depth)
 
     def _resolve(self) -> QueryDatasource:
         parent_sources: List[QueryDatasource | BuildDatasource] = [
             p.resolve() for p in self.parents
         ]
+
         grains = self.check_if_required(
             self.output_concepts, parent_sources, self.environment, self.depth
         )

trilogy/core/processing/nodes/merge_node.py

@@ -341,6 +341,7 @@ class MergeNode(StrategyNode):
                 full_join_concepts += join.input_concepts
 
         if self.force_group is True:
+
             force_group = True
         elif self.whole_grain:
             force_group = False
@@ -367,6 +368,14 @@ class MergeNode(StrategyNode):
         nullable_concepts = find_nullable_concepts(
             source_map=source_map, joins=joins, datasources=final_datasets
         )
+        if force_group:
+
+            grain = BuildGrain.from_concepts(
+                self.output_concepts, environment=self.environment
+            )
+            logger.info(
+                f"{self.logging_prefix}{LOGGER_PREFIX} forcing group by to achieve grain {grain}"
+            )
         qds = QueryDatasource(
             input_concepts=unique(self.input_concepts, "address"),
             output_concepts=unique(self.output_concepts, "address"),

trilogy/core/processing/utility.py

@@ -27,6 +27,7 @@ from trilogy.core.models.build import (
     BuildDatasource,
     BuildFilterItem,
     BuildFunction,
+    BuildGrain,
     BuildParenthetical,
     BuildSubselectComparison,
     BuildWindowItem,
@@ -82,6 +83,13 @@ class JoinOrderOutput:
         return set(self.keys.keys())
 
 
+@dataclass
+class GroupRequiredResponse:
+    target: BuildGrain
+    upstream: BuildGrain
+    required: bool
+
+
 def resolve_join_order_v2(
     g: nx.Graph, partials: dict[str, list[str]], nullables: dict[str, list[str]]
 ) -> list[JoinOrderOutput]:

trilogy/dialect/base.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from datetime import date, datetime
 from typing import Any, Callable, Dict, List, Optional, Sequence, Union
 
@@ -311,6 +312,7 @@ def safe_get_cte_value(
     c: BuildConcept,
     quote_char: str,
     render_expr: Callable,
+    use_map: dict[str, set[str]],
 ) -> Optional[str]:
     address = c.address
     raw = cte.source_map.get(address, None)
@@ -319,13 +321,17 @@ def safe_get_cte_value(
         return None
     if isinstance(raw, str):
         rendered = cte.get_alias(c, raw)
+        use_map[raw].add(c.address)
         return f"{quote_char}{raw}{quote_char}.{safe_quote(rendered, quote_char)}"
     if isinstance(raw, list) and len(raw) == 1:
         rendered = cte.get_alias(c, raw[0])
         if isinstance(rendered, FUNCTION_ITEMS):
             # if it's a function, we need to render it as a function
             return f"{render_expr(rendered, cte=cte, raise_invalid=True)}"
+        use_map[raw[0]].add(c.address)
         return f"{quote_char}{raw[0]}{quote_char}.{safe_quote(rendered, quote_char)}"
+    for x in raw:
+        use_map[x].add(c.address)
     return coalesce(
         sorted(
             [
@@ -350,13 +356,12 @@ class BaseDialect:
 
     def __init__(self, rendering: Rendering | None = None):
         self.rendering = rendering or CONFIG.rendering
+        self.used_map: dict[str, set[str]] = defaultdict(set)
 
     def render_order_item(
         self,
         order_item: BuildOrderItem,
         cte: CTE | UnionCTE,
-        final: bool = False,
-        alias: bool = True,
     ) -> str:
         # if final:
         #     if not alias:
@@ -527,6 +532,9 @@ class BaseDialect:
             )
 
             raw_content = cte.get_alias(c)
+            parent = cte.source_map.get(c.address, None)
+            if parent:
+                self.used_map[parent[0]].add(c.address)
             if isinstance(raw_content, RawColumnExpr):
                 rval = raw_content.text
             elif isinstance(raw_content, FUNCTION_ITEMS):
@@ -540,6 +548,7 @@ class BaseDialect:
                     c,
                     self.QUOTE_CHARACTER,
                     self.render_expr,
+                    self.used_map,
                 )
                 if not rval:
                     # unions won't have a specific source mapped; just use a generic column reference
@@ -615,6 +624,7 @@ class BaseDialect:
                 lookup_cte = cte
                 if cte_map and not lookup_cte:
                     lookup_cte = cte_map.get(e.right.address)
+
                 assert lookup_cte, "Subselects must be rendered with a CTE in context"
                 if e.right.address not in lookup_cte.existence_source_map:
                     lookup = lookup_cte.source_map.get(
@@ -634,6 +644,7 @@ class BaseDialect:
                         f"Missing source CTE for {e.right.address}"
                     )
                 assert cte, "CTE must be provided for inlined CTEs"
+                self.used_map[target].add(e.right.address)
                 if target in cte.inlined_ctes:
                     info = cte.inlined_ctes[target]
                     return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} (select {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {info.new_base} as {target} where {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
@@ -738,6 +749,7 @@ class BaseDialect:
                     raise_invalid=raise_invalid,
                 )
             elif cte_map:
+                self.used_map[cte_map[e.address].name].add(e.address)
                 return f"{cte_map[e.address].name}.{self.QUOTE_CHARACTER}{e.safe_address}{self.QUOTE_CHARACTER}"
             return f"{self.QUOTE_CHARACTER}{e.safe_address}{self.QUOTE_CHARACTER}"
         elif isinstance(e, bool):
@@ -774,6 +786,8 @@ class BaseDialect:
             return str(e.value)
         elif isinstance(e, ArrayType):
             return f"{self.COMPLEX_DATATYPE_MAP[DataType.ARRAY](self.render_expr(e.value_data_type, cte=cte, cte_map=cte_map))}"
+        elif isinstance(e, list):
+            return f"{self.FUNCTION_MAP[FunctionType.ARRAY]([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}"
         elif isinstance(e, BuildParamaterizedConceptReference):
             if self.rendering.parameters:
                 if e.concept.namespace == DEFAULT_NAMESPACE:
@@ -782,6 +796,7 @@ class BaseDialect:
             elif e.concept.lineage:
                 return self.render_expr(e.concept.lineage, cte=cte, cte_map=cte_map)
             return f"{self.QUOTE_CHARACTER}{e.concept.address}{self.QUOTE_CHARACTER}"
+
         else:
             raise ValueError(f"Unable to render type {type(e)} {e}")
 
@@ -822,10 +837,7 @@ class BaseDialect:
             )
             if cte.order_by:
 
-                ordering = [
-                    self.render_order_item(i, cte, final=True, alias=False)
-                    for i in cte.order_by.items
-                ]
+                ordering = [self.render_order_item(i, cte) for i in cte.order_by.items]
                 base_statement += "\nORDER BY " + ",".join(ordering)
             return CompiledCTE(name=cte.name, statement=base_statement)
         elif isinstance(cte, RecursiveCTE):
@@ -950,7 +962,8 @@ class BaseDialect:
                             self.QUOTE_CHARACTER,
                             self.render_expr,
                             cte,
-                            self.UNNEST_MODE,
+                            use_map=self.used_map,
+                            unnest_mode=self.UNNEST_MODE,
                         )
                         for join in final_joins
                     ]

trilogy/dialect/common.py

@@ -62,10 +62,12 @@ def render_join_concept(
     concept: BuildConcept,
     render_expr,
     inlined_ctes: set[str],
+    use_map: dict[str, set[str]],
 ):
     if cte.name in inlined_ctes:
         base = render_expr(concept, cte)
         return base
+    use_map[name].add(concept.address)
     return f"{quote_character}{name}{quote_character}.{quote_character}{concept.safe_address}{quote_character}"
 
 
@@ -85,6 +87,7 @@ def render_join(
         str,
     ],
     cte: CTE,
+    use_map: dict[str, set[str]],
     unnest_mode: UnnestMode = UnnestMode.CROSS_APPLY,
 ) -> str | None:
     # {% for key in join.joinkeys %}{{ key.inner }} = {{ key.outer}}{% endfor %}
@@ -121,6 +124,7 @@ def render_join(
                         pair.left,
                         render_expr_func,
                         join.inlined_ctes,
+                        use_map=use_map,
                     ),
                     render_join_concept(
                         right_name,
@@ -129,6 +133,7 @@ def render_join(
                         pair.right,
                         render_expr_func,
                         join.inlined_ctes,
+                        use_map=use_map,
                     ),
                     modifiers=pair.modifiers
                     + (pair.left.modifiers or [])

trilogy/std/color.preql

@@ -0,0 +1,3 @@
+
+
+type hex string; # Hex color value, e.g. #FF5733

trilogy/std/display.preql

@@ -3,7 +3,7 @@
 type percent float; # Percentage value
 
 def calc_percent(a, b, digits=-1) -> case when digits =-1 then 
-case when b = 0 then 0.0::float::percent else
-(a/b)::float::percent end
+case when b = 0 then 0.0::numeric::percent else
+(a/b)::numeric::percent end
 else round((case when b = 0 then 0.0::float::percent else
-(a/b)::float::percent end):: float::percent, digits) end;
+(a/b)::float::percent end):: numeric::percent, digits) end;