pytrilogy 0.0.2.57__py3-none-any.whl → 0.0.3.0__py3-none-any.whl

trilogy/core/processing/graph_utils.py

@@ -1,7 +1,7 @@
 from collections import defaultdict
 from typing import Dict, List
 
-from trilogy.core.models import Concept
+from trilogy.core.models.author import Concept
 from trilogy.utility import unique
 
 

trilogy/core/processing/node_generators/basic_node.py

@@ -1,14 +1,9 @@
-# directly select out a basic derivation
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.enums import SourceType
-from trilogy.core.models import (
-    Concept,
-    Function,
-    FunctionClass,
-    WhereClause,
-)
+from trilogy.core.enums import FunctionClass, SourceType
+from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
+from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.node_generators.common import (
     resolve_function_parent_concepts,
 )
@@ -19,10 +14,12 @@ LOGGER_PREFIX = "[GEN_BASIC_NODE]"
 
 
 def is_equivalent_basic_function_lineage(
-    x: Concept,
-    y: Concept,
+    x: BuildConcept,
+    y: BuildConcept,
 ):
-    if not isinstance(x.lineage, Function) or not isinstance(y.lineage, Function):
+    if not isinstance(x.lineage, BuildFunction) or not isinstance(
+        y.lineage, BuildFunction
+    ):
         return False
     if x.lineage.operator == y.lineage.operator:
         return True
@@ -35,20 +32,20 @@ def is_equivalent_basic_function_lineage(
 
 
 def gen_basic_node(
-    concept: Concept,
-    local_optional: List[Concept],
-    environment,
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    environment: BuildEnvironment,
     g,
     depth: int,
     source_concepts,
     history: History | None = None,
-    conditions: WhereClause | None = None,
+    conditions: BuildWhereClause | None = None,
 ):
     depth_prefix = "\t" * depth
     parent_concepts = resolve_function_parent_concepts(concept, environment=environment)
 
     logger.info(
-        f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has parents {[x.address for x in parent_concepts]}"
+        f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} with lineage {concept.lineage} has parents {[x for x in parent_concepts]}"
     )
 
     equivalent_optional = [
@@ -57,6 +54,7 @@ def gen_basic_node(
         if is_equivalent_basic_function_lineage(concept, x)
         and x.address != concept.address
     ]
+
     if equivalent_optional:
         logger.info(
             f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has equivalent optional {[x.address for x in equivalent_optional]}"
@@ -64,9 +62,12 @@ def gen_basic_node(
     for eo in equivalent_optional:
         parent_concepts += resolve_function_parent_concepts(eo, environment=environment)
     non_equivalent_optional = [
-        x for x in local_optional if x not in equivalent_optional
+        x
+        for x in local_optional
+        if x not in equivalent_optional
+        and not any(x.address in y.pseudonyms for y in equivalent_optional)
     ]
-    all_parents: list[Concept] = unique(
+    all_parents: list[BuildConcept] = unique(
         parent_concepts + non_equivalent_optional, "address"
     )
     logger.info(

trilogy/core/processing/node_generators/common.py

@@ -1,16 +1,16 @@
 from collections import defaultdict
 from typing import Callable, List, Tuple
 
-from trilogy.core.enums import Purpose, PurposeLineage
-from trilogy.core.models import (
-    AggregateWrapper,
-    Concept,
-    Environment,
-    FilterItem,
-    Function,
-    LooseConceptList,
-    WhereClause,
+from trilogy.core.enums import Derivation, Purpose
+from trilogy.core.models.build import (
+    BuildAggregateWrapper,
+    BuildConcept,
+    BuildFilterItem,
+    BuildFunction,
+    BuildWhereClause,
+    LooseBuildConceptList,
 )
+from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import (
     History,
     NodeJoin,
@@ -19,14 +19,19 @@ from trilogy.core.processing.nodes.base_node import StrategyNode
 from trilogy.core.processing.nodes.merge_node import MergeNode
 from trilogy.utility import unique
 
+AGGREGATE_TYPES = (BuildAggregateWrapper,)
+FUNCTION_TYPES = (BuildFunction,)
+
 
 def resolve_function_parent_concepts(
-    concept: Concept, environment: Environment
-) -> List[Concept]:
-    if not isinstance(concept.lineage, (Function, AggregateWrapper)):
-        raise ValueError(f"Concept {concept} lineage is not function or aggregate")
-    if concept.derivation == PurposeLineage.AGGREGATE:
-        base: list[Concept] = []
+    concept: BuildConcept, environment: BuildEnvironment
+) -> List[BuildConcept]:
+    if not isinstance(concept.lineage, (*FUNCTION_TYPES, *AGGREGATE_TYPES)):
+        raise ValueError(
+            f"Concept {concept} lineage is not function or aggregate, is {type(concept.lineage)}"
+        )
+    if concept.derivation == Derivation.AGGREGATE:
+        base: list[BuildConcept] = []
         if not concept.grain.abstract:
             base = concept.lineage.concept_arguments + [
                 environment.concepts[c] for c in concept.grain.components
@@ -35,13 +40,13 @@ def resolve_function_parent_concepts(
             # keep the key as a parent
         else:
             base = concept.lineage.concept_arguments
-        if isinstance(concept.lineage, AggregateWrapper):
+        if isinstance(concept.lineage, AGGREGATE_TYPES):
             # for aggregate wrapper, don't include the by
             extra_property_grain = concept.lineage.function.concept_arguments
         else:
             extra_property_grain = concept.lineage.concept_arguments
         for x in extra_property_grain:
-            if isinstance(x, Concept) and x.purpose == Purpose.PROPERTY and x.keys:
+            if isinstance(x, BuildConcept) and x.purpose == Purpose.PROPERTY and x.keys:
                 base += [environment.concepts[c] for c in x.keys]
         return unique(base, "address")
     # TODO: handle basic lineage chains?
@@ -49,10 +54,10 @@ def resolve_function_parent_concepts(
 
 
 def resolve_condition_parent_concepts(
-    condition: WhereClause,
-) -> Tuple[List[Concept], List[Tuple[Concept, ...]]]:
+    condition: BuildWhereClause,
+) -> Tuple[List[BuildConcept], List[Tuple[BuildConcept, ...]]]:
     base_existence = []
-    base_rows = []
+    base_rows: list[BuildConcept] = []
     base_rows += condition.row_arguments
     for ctuple in condition.existence_arguments:
         base_existence.append(ctuple)
@@ -60,10 +65,10 @@ def resolve_condition_parent_concepts(
 
 
 def resolve_filter_parent_concepts(
-    concept: Concept,
-    environment: Environment,
-) -> Tuple[Concept, List[Concept], List[Tuple[Concept, ...]]]:
-    if not isinstance(concept.lineage, FilterItem):
+    concept: BuildConcept,
+    environment: BuildEnvironment,
+) -> Tuple[BuildConcept, List[BuildConcept], List[Tuple[BuildConcept, ...]]]:
+    if not isinstance(concept.lineage, (BuildFilterItem,)):
         raise ValueError(
             f"Concept {concept} lineage is not filter item, is {type(concept.lineage)}"
         )
@@ -75,14 +80,14 @@ def resolve_filter_parent_concepts(
     )
     base_rows += condition_rows
     base_existence += condition_existence
-    if direct_parent.grain:
-        base_rows += [environment.concepts[c] for c in direct_parent.grain.components]
+    # this is required so that
     if (
-        isinstance(direct_parent, Concept)
-        and direct_parent.purpose == Purpose.PROPERTY
+        isinstance(direct_parent, BuildConcept)
+        and direct_parent.purpose in (Purpose.PROPERTY, Purpose.METRIC)
         and direct_parent.keys
     ):
         base_rows += [environment.concepts[c] for c in direct_parent.keys]
+
     if concept.lineage.where.existence_arguments:
         return (
             concept.lineage.content,
@@ -94,14 +99,14 @@ def resolve_filter_parent_concepts(
 
 def gen_property_enrichment_node(
     base_node: StrategyNode,
-    extra_properties: list[Concept],
-    environment: Environment,
+    extra_properties: list[BuildConcept],
+    history: History,
+    environment: BuildEnvironment,
     g,
     depth: int,
     source_concepts,
     log_lambda: Callable,
-    history: History | None = None,
-    conditions: WhereClause | None = None,
+    conditions: BuildWhereClause | None = None,
 ):
     required_keys: dict[str, set[str]] = defaultdict(set)
     for x in extra_properties:
@@ -146,17 +151,17 @@ def gen_property_enrichment_node(
 
 def gen_enrichment_node(
     base_node: StrategyNode,
-    join_keys: List[Concept],
-    local_optional: list[Concept],
-    environment: Environment,
+    join_keys: List[BuildConcept],
+    local_optional: list[BuildConcept],
+    environment: BuildEnvironment,
     g,
     depth: int,
     source_concepts,
     log_lambda,
-    history: History | None = None,
-    conditions: WhereClause | None = None,
+    history: History,
+    conditions: BuildWhereClause | None = None,
 ):
-    local_opts = LooseConceptList(concepts=local_optional)
+    local_opts = LooseBuildConceptList(concepts=local_optional)
 
     extra_required = [
         x
@@ -165,7 +170,8 @@ def gen_enrichment_node(
     ]
 
     # property lookup optimization
-    # this helps when evaluating a normalized star schema as you only want to lookup the missing properties based on the relevant keys
+    # this helps create ergonomic merge nodes when evaluating a normalized star schema
+    # as we only want to lookup the missing properties based on the relevant keys
     if all([x.purpose == Purpose.PROPERTY for x in extra_required]):
         if all(
             x.keys and all([key in base_node.output_lcl for key in x.keys])
@@ -177,16 +183,16 @@ def gen_enrichment_node(
             return gen_property_enrichment_node(
                 base_node,
                 extra_required,
-                environment,
-                g,
-                depth,
-                source_concepts,
+                environment=environment,
+                g=g,
+                depth=depth,
+                source_concepts=source_concepts,
                 history=history,
                 conditions=conditions,
                 log_lambda=log_lambda,
             )
     log_lambda(
-        f"{str(type(base_node).__name__)} searching for join keys {LooseConceptList(concepts=join_keys)} and extra required {local_opts}"
+        f"{str(type(base_node).__name__)} searching for join keys {LooseBuildConceptList(concepts=join_keys)} and extra required {local_opts}"
     )
     enrich_node: StrategyNode = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
@@ -208,7 +214,7 @@ def gen_enrichment_node(
     non_hidden = [
         x
         for x in base_node.output_concepts
-        if x.address not in [y.address for y in base_node.hidden_concepts]
+        if x.address not in base_node.hidden_concepts
     ]
     return MergeNode(
         input_concepts=unique(join_keys + extra_required + non_hidden, "address"),

trilogy/core/processing/node_generators/filter_node.py

@@ -1,7 +1,13 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.models import Concept, Environment, FilterItem, Grain, WhereClause
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildFilterItem,
+    BuildGrain,
+    BuildWhereClause,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.node_generators.common import (
     resolve_filter_parent_concepts,
 )
@@ -16,28 +22,30 @@ from trilogy.core.processing.utility import is_scalar_condition, padding, unique
 
 LOGGER_PREFIX = "[GEN_FILTER_NODE]"
 
+FILTER_TYPES = (BuildFilterItem,)
+
 
 def gen_filter_node(
-    concept: Concept,
-    local_optional: List[Concept],
-    environment: Environment,
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    environment: BuildEnvironment,
     g,
     depth: int,
     source_concepts,
     history: History | None = None,
-    conditions: WhereClause | None = None,
+    conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
     immediate_parent, parent_row_concepts, parent_existence_concepts = (
         resolve_filter_parent_concepts(concept, environment)
     )
-    if not isinstance(concept.lineage, FilterItem):
-        raise SyntaxError('Filter node must have a lineage of type "FilterItem"')
+    if not isinstance(concept.lineage, FILTER_TYPES):
+        raise SyntaxError('Filter node must have a filter type lineage"')
     where = concept.lineage.where
 
-    optional_included: list[Concept] = []
+    optional_included: list[BuildConcept] = []
 
     for x in local_optional:
-        if isinstance(x.lineage, FilterItem):
+        if isinstance(x.lineage, FILTER_TYPES):
             if concept.lineage.where == where:
                 logger.info(
                     f"{padding(depth)}{LOGGER_PREFIX} fetching {x.lineage.content.address} as optional parent with same filter conditions "
@@ -48,7 +56,7 @@ def gen_filter_node(
         if conditions and conditions == where:
             optional_included.append(x)
     logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} filter {concept.address} derived from {immediate_parent.address} row parents {[x.address for x in parent_row_concepts]} and {[[y.address] for x  in parent_existence_concepts for y  in x]} existence parents"
+        f"{padding(depth)}{LOGGER_PREFIX} filter `{concept}` condition `{concept.lineage.where}` derived from {immediate_parent.address} row parents {[x.address for x in parent_row_concepts]} and {[[y.address] for x  in parent_existence_concepts for y  in x]} existence parents"
     )
     # we'll populate this with the row parent
     # and the existence parent(s)
@@ -136,7 +144,7 @@ def gen_filter_node(
         parent.add_existence_concepts(flattened_existence, False).set_output_concepts(
             expected_output, False
         )
-        parent.grain = Grain.from_concepts(
+        parent.grain = BuildGrain.from_concepts(
             (
                 [environment.concepts[k] for k in immediate_parent.keys]
                 if immediate_parent.keys
@@ -162,7 +170,7 @@ def gen_filter_node(
             output_concepts=[concept, immediate_parent] + parent_row_concepts,
             environment=environment,
             parents=core_parents,
-            grain=Grain.from_concepts(
+            grain=BuildGrain.from_concepts(
                 [immediate_parent] + parent_row_concepts,
             ),
             preexisting_conditions=conditions.conditional if conditions else None,
@@ -185,7 +193,7 @@ def gen_filter_node(
         )
         return filter_node
 
-    enrich_node = source_concepts(  # this fetches the parent + join keys
+    enrich_node: StrategyNode = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
         mandatory_list=[immediate_parent] + parent_row_concepts + local_optional,
         environment=environment,
@@ -194,6 +202,11 @@ def gen_filter_node(
         history=history,
         conditions=conditions,
     )
+    if not enrich_node:
+        logger.error(
+            f"{padding(depth)}{LOGGER_PREFIX} filter node enrichment node could not be found"
+        )
+        return filter_node
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} returning filter node and enrich node with {enrich_node.output_concepts} and {enrich_node.input_concepts}"
     )

trilogy/core/processing/node_generators/group_node.py

@@ -1,15 +1,15 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.models import (
-    AggregateWrapper,
-    Concept,
-    Environment,
-    Function,
-    Grain,
-    LooseConceptList,
-    WhereClause,
+from trilogy.core.models.build import (
+    BuildAggregateWrapper,
+    BuildConcept,
+    BuildFunction,
+    BuildGrain,
+    BuildWhereClause,
+    LooseBuildConceptList,
 )
+from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.node_generators.common import (
     gen_enrichment_node,
     resolve_function_parent_concepts,
@@ -22,22 +22,22 @@ LOGGER_PREFIX = "[GEN_GROUP_NODE]"
 
 
 def gen_group_node(
-    concept: Concept,
-    local_optional: List[Concept],
-    environment: Environment,
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    environment: BuildEnvironment,
     g,
     depth: int,
     source_concepts,
-    history: History | None = None,
-    conditions: WhereClause | None = None,
+    history: History,
+    conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
     # aggregates MUST always group to the proper grain
     # except when the
-    parent_concepts: List[Concept] = unique(
+    parent_concepts: List[BuildConcept] = unique(
         resolve_function_parent_concepts(concept, environment=environment), "address"
     )
     logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} parent concepts are {[x.address for x in parent_concepts]} from group grain {concept.grain}"
+        f"{padding(depth)}{LOGGER_PREFIX} parent concepts for {concept} are {[x.address for x in parent_concepts]} from group grain {concept.grain}"
     )
 
     # if the aggregation has a grain, we need to ensure these are the ONLY optional in the output of the select
@@ -53,16 +53,18 @@ def gen_group_node(
         output_concepts += grain_components
         for possible_agg in local_optional:
 
-            if not isinstance(possible_agg.lineage, (AggregateWrapper, Function)):
+            if not isinstance(
+                possible_agg.lineage,
+                (BuildAggregateWrapper, BuildFunction),
+            ):
                 continue
-            logger.info(possible_agg)
             if possible_agg.grain and possible_agg.grain != concept.grain:
                 logger.info(
                     f"{padding(depth)}{LOGGER_PREFIX} mismatched equivalent group by with grain {possible_agg.grain} for {concept.address}"
                 )
 
             if possible_agg.grain and possible_agg.grain == concept.grain:
-                agg_parents: List[Concept] = resolve_function_parent_concepts(
+                agg_parents: List[BuildConcept] = resolve_function_parent_concepts(
                     possible_agg,
                     environment=environment,
                 )
@@ -73,7 +75,7 @@ def gen_group_node(
                     logger.info(
                         f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
                     )
-                elif Grain.from_concepts(agg_parents) == Grain.from_concepts(
+                elif BuildGrain.from_concepts(agg_parents) == BuildGrain.from_concepts(
                     parent_concepts
                 ):
                     extra = [x for x in agg_parents if x.address not in parent_concepts]
@@ -84,11 +86,11 @@ def gen_group_node(
                     )
                 else:
                     logger.info(
-                        f"{padding(depth)}{LOGGER_PREFIX} mismatched grain {Grain.from_concepts(agg_parents)} vs {Grain.from_concepts(parent_concepts)}"
+                        f"{padding(depth)}{LOGGER_PREFIX} cannot include optional agg; mismatched grain {BuildGrain.from_concepts(agg_parents)} vs {BuildGrain.from_concepts(parent_concepts)}"
                     )
     if parent_concepts:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} fetching group node parents {LooseConceptList(concepts=parent_concepts)}"
+            f"{padding(depth)}{LOGGER_PREFIX} fetching group node parents {LooseBuildConceptList(concepts=parent_concepts)}"
         )
         parent_concepts = unique(parent_concepts, "address")
         parent = source_concepts(
@@ -124,6 +126,9 @@ def gen_group_node(
     # early exit if no optional
 
     if not local_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no optional concepts, returning group node"
+        )
         return group_node
     missing_optional = [
         x.address for x in local_optional if x.address not in group_node.output_concepts

trilogy/core/processing/node_generators/group_to_node.py

@@ -1,7 +1,8 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.models import Concept, Environment, Function, WhereClause
+from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
+from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import (
     GroupNode,
     History,
@@ -14,20 +15,22 @@ LOGGER_PREFIX = "[GEN_GROUP_TO_NODE]"
 
 
 def gen_group_to_node(
-    concept: Concept,
+    concept: BuildConcept,
     local_optional,
-    environment: Environment,
+    environment: BuildEnvironment,
     g,
     depth: int,
     source_concepts,
     history: History | None = None,
-    conditions: WhereClause | None = None,
+    conditions: BuildWhereClause | None = None,
 ) -> GroupNode | MergeNode:
     # aggregates MUST always group to the proper grain
-    if not isinstance(concept.lineage, Function):
-        raise SyntaxError("Group to should have function lineage")
+    if not isinstance(concept.lineage, BuildFunction):
+        raise SyntaxError(
+            f"Group to should have function lineage, is {type(concept.lineage)}"
+        )
     group_arg = concept.lineage.arguments[0]
-    parent_concepts: List[Concept] = concept.lineage.concept_arguments
+    parent_concepts: List[BuildConcept] = concept.lineage.concept_arguments
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} group by node has required parents {[x.address for x in parent_concepts]}"
     )
@@ -49,9 +52,9 @@ def gen_group_to_node(
         parents=parents,
         depth=depth,
         preexisting_conditions=conditions.conditional if conditions else None,
-        hidden_concepts=(
-            [group_arg]
-            if isinstance(group_arg, Concept)
+        hidden_concepts=set(
+            [group_arg.address]
+            if isinstance(group_arg, BuildConcept)
             and group_arg.address not in local_optional
             else []
         ),