pytrilogy 0.0.1.118__py3-none-any.whl → 0.0.2.2__py3-none-any.whl

trilogy/core/processing/node_generators/select_node.py

@@ -1,5 +1,4 @@
-from itertools import combinations
-from typing import List, Optional
+from typing import List, Optional, Callable
 
 from trilogy.core.enums import PurposeLineage
 from trilogy.core.models import (
@@ -43,9 +42,11 @@ def dm_to_strategy_node(
     environment: Environment,
     g: nx.DiGraph,
     depth: int,
+    source_concepts: Callable,
     accept_partial: bool = False,
 ) -> StrategyNode:
     datasource = dm.datasource
+
     if target_grain and target_grain.issubset(datasource.grain):
         if all([x in dm.matched for x in target_grain.components]):
             force_group = False
@@ -66,6 +67,17 @@ def dm_to_strategy_node(
             f"{padding(depth)}{LOGGER_PREFIX} target grain is not subset of datasource grain {datasource.grain}, required to group"
         )
         force_group = True
+    # if isinstance(datasource, MergeDatasource):
+    #     # if we're within a namespace, don't find merge nodes
+    #     bcandidate: StrategyNode = gen_environment_merge_node(
+    #         all_concepts=dm.matched.concepts,
+    #         environment=environment,
+    #         g=g,
+    #         depth=depth,
+    #         datasource=datasource,
+    #         source_concepts=source_concepts,
+    #     )
+    # else:
     bcandidate: StrategyNode = SelectNode(
         input_concepts=[c.concept for c in datasource.columns],
         output_concepts=dm.matched.concepts,
@@ -77,6 +89,7 @@ def dm_to_strategy_node(
         accept_partial=accept_partial,
         datasource=datasource,
         grain=datasource.grain,
+        conditions=datasource.where.conditional if datasource.where else None,
     )
     # we need to nest the group node one further
     if force_group is True:
@@ -101,6 +114,7 @@ def gen_select_nodes_from_tables_v2(
     environment: Environment,
     depth: int,
     target_grain: Grain,
+    source_concepts: Callable,
     accept_partial: bool = False,
 ) -> tuple[bool, list[Concept], list[StrategyNode]]:
     # if we have only constants
@@ -168,6 +182,7 @@ def gen_select_nodes_from_tables_v2(
                 if ds_valid and address_valid:
                     matched_paths.append(path)
                     matched.append(all_lcl.concepts[idx])
+
             except nx.NodeNotFound:
                 continue
             except nx.exception.NetworkXNoPath:
@@ -206,6 +221,7 @@ def gen_select_nodes_from_tables_v2(
             - 0.1 * len(matches[x].partial.addresses),
         )
         final: DatasourceMatch = matches[final_key]
+
         candidate = dm_to_strategy_node(
             final,
             target_grain=Grain(
@@ -217,6 +233,7 @@ def gen_select_nodes_from_tables_v2(
             g=g,
             depth=depth,
             accept_partial=accept_partial,
+            source_concepts=source_concepts,
         )
         to_return.append(candidate)
         del matches[final_key]
@@ -233,6 +250,7 @@ def gen_select_node_from_table(
     environment: Environment,
     depth: int,
     target_grain: Grain,
+    source_concepts,
     accept_partial: bool = False,
 ) -> Optional[StrategyNode]:
     # if we have only constants
@@ -277,9 +295,7 @@ def gen_select_node_from_table(
                     try:
                         g.nodes[ncandidate]
                     except KeyError:
-                        raise SyntaxError(
-                            "Could not find node for {}".format(ncandidate)
-                        )
+                        raise nx.exception.NetworkXNoPath
                 raise e
             except nx.exception.NetworkXNoPath:
                 all_found = False
@@ -354,6 +370,7 @@ def gen_select_node_from_table(
             accept_partial=accept_partial,
             datasource=datasource,
             grain=Grain(components=all_concepts),
+            conditions=datasource.where.conditional if datasource.where else None,
         )
         # we need to nest the group node one further
         if force_group is True:
@@ -379,81 +396,13 @@ def gen_select_node_from_table(
     return candidates[final]
 
 
-def gen_select_nodes_from_tables(
-    local_optional: List[Concept],
-    depth: int,
-    concept: Concept,
-    environment: Environment,
-    g: nx.DiGraph,
-    accept_partial: bool,
-    all_concepts: List[Concept],
-) -> tuple[bool, list[Concept], list[StrategyNode]]:
-    parents: List[StrategyNode] = []
-    found: List[Concept] = []
-    logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} looking for multiple sources that can satisfy"
-    )
-    all_found = False
-    unreachable: list[str] = []
-    # first pass
-    for opt_con in local_optional:
-        ds = gen_select_node_from_table(
-            concept,
-            [concept, opt_con],
-            g=g,
-            environment=environment,
-            depth=depth + 1,
-            accept_partial=accept_partial,
-            target_grain=Grain(components=all_concepts),
-        )
-        if not ds:
-            unreachable.append(opt_con.address)
-    all_found = False
-    for x in reversed(range(1, len(local_optional) + 1)):
-        if all_found:
-            break
-        for combo in combinations(local_optional, x):
-            if all_found:
-                break
-            # filter to just the original ones we need to get
-            local_combo = [
-                x for x in combo if x not in found and x.address not in unreachable
-            ]
-            # skip if nothing new in this combo
-            if not local_combo:
-                continue
-            # include core concept as join
-            all_concepts = [concept, *local_combo]
-
-            ds = gen_select_node_from_table(
-                concept,
-                all_concepts,
-                g=g,
-                environment=environment,
-                depth=depth + 1,
-                accept_partial=accept_partial,
-                target_grain=Grain(components=all_concepts),
-            )
-            if ds:
-                logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} found a source with {[x.address for x in all_concepts]}"
-                )
-                parents.append(ds)
-                found += [x for x in ds.output_concepts if x != concept]
-                if {x.address for x in found} == {c.address for c in local_optional}:
-                    logger.info(
-                        f"{padding(depth)}{LOGGER_PREFIX} found all optional {[c.address for c in local_optional]}"
-                    )
-                    all_found = True
-    return all_found, found, parents
-
-
 def gen_select_node(
     concept: Concept,
     local_optional: List[Concept],
     environment: Environment,
     g,
     depth: int,
+    source_concepts,
     accept_partial: bool = False,
     fail_if_not_found: bool = True,
     accept_partial_optional: bool = True,
@@ -477,8 +426,8 @@ def gen_select_node(
         target_grain = Grain(components=all_concepts)
     if materialized_lcl != all_lcl:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concept.address} "
-            f" as it + optional (looking for all {all_lcl}) includes non-materialized concepts {all_lcl.difference(materialized_lcl)} vs materialized: {materialized_lcl}"
+            f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concept.address}"
+            f" as it + optional (looking for all {all_lcl}) includes non-materialized concepts"
         )
         if fail_if_not_found:
             raise NoDatasourceException(f"No datasource exists for {concept}")
@@ -495,6 +444,7 @@ def gen_select_node(
         depth=depth,
         accept_partial=accept_partial,
         target_grain=target_grain,
+        source_concepts=source_concepts,
     )
     if ds:
         logger.info(
@@ -503,17 +453,16 @@ def gen_select_node(
         return ds
     # if we cannot find a match
     all_found, found, parents = gen_select_nodes_from_tables_v2(
-        concept, all_concepts, g, environment, depth, target_grain, accept_partial
+        concept,
+        all_concepts,
+        g,
+        environment,
+        depth=depth,
+        target_grain=target_grain,
+        accept_partial=accept_partial,
+        source_concepts=source_concepts,
     )
     if parents and (all_found or accept_partial_optional):
-        if all_found:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} found all optional {[c.address for c in local_optional]} via joins"
-            )
-        else:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} found some optional, returning"
-            )
         all_partial = [
             c
             for c in all_concepts
@@ -521,6 +470,16 @@ def gen_select_node(
                 [c.address in [x.address for x in p.partial_concepts] for p in parents]
             )
         ]
+
+        if all_found:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} found all optional {[c.address for c in local_optional]} via joins"
+            )
+        else:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} found some optional {[x.address for x in found]}, and partial return allowed: returning"
+            )
+
         force_group = None
         inferred_grain = sum([x.grain for x in parents if x.grain], Grain())
         for candidate in parents:
@@ -571,6 +530,7 @@ def gen_select_node(
         depth=depth,
         accept_partial=accept_partial,
         target_grain=Grain(components=[concept]),
+        source_concepts=source_concepts,
     )
 
     if not ds and fail_if_not_found:

trilogy/core/processing/nodes/__init__.py

@@ -93,6 +93,7 @@ class History(BaseModel):
         environment: Environment,
         g,
         depth: int,
+        source_concepts,
         fail_if_not_found: bool = False,
         accept_partial: bool = False,
         accept_partial_optional: bool = False,
@@ -117,6 +118,7 @@ class History(BaseModel):
             fail_if_not_found=fail_if_not_found,
             accept_partial=accept_partial,
             accept_partial_optional=accept_partial_optional,
+            source_concepts=source_concepts,
         )
         self.select_history[fingerprint] = gen
         return gen

trilogy/core/processing/nodes/base_node.py

@@ -17,7 +17,6 @@ from trilogy.core.models import (
 from trilogy.core.enums import Purpose, JoinType, PurposeLineage, Granularity
 from trilogy.utility import unique
 from dataclasses import dataclass
-from trilogy.constants import logger
 
 
 def concept_list_to_grain(
@@ -59,7 +58,6 @@ def resolve_concept_map(
     inherited = set([t.address for t in inherited_inputs])
     for input in inputs:
         for concept in input.output_concepts:
-            logger.info(concept.address)
             if concept.address not in input.non_partial_concept_addresses:
                 continue
             if concept.address not in inherited:
@@ -152,11 +150,26 @@ class StrategyNode:
             if not parent:
                 raise SyntaxError("Unresolvable parent")
 
-    def add_output_concept(self, concept: Concept):
-        self.output_concepts.append(concept)
+    def add_output_concepts(self, concepts: List[Concept]):
+        for concept in concepts:
+            self.output_concepts.append(concept)
         self.output_lcl = LooseConceptList(concepts=self.output_concepts)
         self.rebuild_cache()
 
+    def add_output_concept(self, concept: Concept):
+        self.add_output_concepts([concept])
+
+    def hide_output_concepts(self, concepts: List[Concept]):
+        for x in concepts:
+            self.hidden_concepts.append(x)
+        self.rebuild_cache()
+
+    def remove_output_concepts(self, concepts: List[Concept]):
+        for x in concepts:
+            self.hidden_concepts.append(x)
+        self.output_concepts = [x for x in self.output_concepts if x not in concepts]
+        self.rebuild_cache()
+
     @property
     def logging_prefix(self) -> str:
         return "\t" * self.depth
@@ -167,7 +180,7 @@ class StrategyNode:
 
     @property
     def all_used_concepts(self) -> list[Concept]:
-        return [*self.input_concepts]
+        return [*self.input_concepts, *self.existence_concepts]
 
     def __repr__(self):
         concepts = self.all_concepts
@@ -196,6 +209,7 @@ class StrategyNode:
             condition=self.conditions,
             partial_concepts=self.partial_concepts,
             force_group=self.force_group,
+            hidden_concepts=self.hidden_concepts,
         )
 
     def rebuild_cache(self) -> QueryDatasource:
@@ -239,8 +253,11 @@ class NodeJoin:
     concepts: List[Concept]
     join_type: JoinType
     filter_to_mutual: bool = False
+    concept_pairs: list[tuple[Concept, Concept]] | None = None
 
     def __post_init__(self):
+        if self.concept_pairs:
+            return
         final_concepts = []
         for concept in self.concepts:
             include = True

trilogy/core/processing/nodes/filter_node.py

@@ -7,6 +7,7 @@ from trilogy.core.models import (
     Conditional,
     Comparison,
     Parenthetical,
+    Grain,
 )
 from trilogy.core.processing.nodes.base_node import StrategyNode
 
@@ -34,6 +35,7 @@ class FilterNode(StrategyNode):
         conditions: Conditional | Comparison | Parenthetical | None = None,
         partial_concepts: List[Concept] | None = None,
         force_group: bool | None = False,
+        grain: Grain | None = None,
     ):
         super().__init__(
             output_concepts=output_concepts,
@@ -46,6 +48,7 @@ class FilterNode(StrategyNode):
             conditions=conditions,
             partial_concepts=partial_concepts,
             force_group=force_group,
+            grain=grain,
         )
 
     def copy(self) -> "FilterNode":

trilogy/core/processing/nodes/group_node.py

@@ -9,12 +9,16 @@ from trilogy.core.models import (
     Concept,
     Environment,
     LooseConceptList,
+    Conditional,
+    Comparison,
+    Parenthetical,
 )
 from trilogy.core.processing.nodes.base_node import (
     StrategyNode,
     resolve_concept_map,
     concept_list_to_grain,
 )
+from trilogy.utility import unique
 
 
 LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
@@ -34,6 +38,7 @@ class GroupNode(StrategyNode):
         depth: int = 0,
         partial_concepts: Optional[List[Concept]] = None,
         force_group: bool | None = None,
+        conditions: Conditional | Comparison | Parenthetical | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -45,6 +50,7 @@ class GroupNode(StrategyNode):
             depth=depth,
             partial_concepts=partial_concepts,
             force_group=force_group,
+            conditions=conditions,
         )
 
     def _resolve(self) -> QueryDatasource:
@@ -79,7 +85,10 @@ class GroupNode(StrategyNode):
                 logger.info(
                     f"{self.logging_prefix}{LOGGER_PREFIX} No group by required, returning parent node"
                 )
-                return parent_sources[0]
+                will_return: QueryDatasource = parent_sources[0]
+                if self.conditions:
+                    will_return.condition = self.conditions + will_return.condition
+                return will_return
             # otherwise if no group by, just treat it as a select
             source_type = SourceType.SELECT
         else:
@@ -109,7 +118,15 @@ class GroupNode(StrategyNode):
             source_type=source_type,
             source_map=resolve_concept_map(
                 parent_sources,
-                targets=self.output_concepts,
+                # targets = self.output_concepts,
+                targets=(
+                    unique(
+                        self.output_concepts + self.conditions.concept_arguments,
+                        "address",
+                    )
+                    if self.conditions
+                    else self.output_concepts
+                ),
                 inherited_inputs=self.input_concepts,
             ),
             joins=[],
@@ -129,4 +146,5 @@ class GroupNode(StrategyNode):
             depth=self.depth,
             partial_concepts=list(self.partial_concepts),
             force_group=self.force_group,
+            conditions=self.conditions,
         )

trilogy/core/processing/nodes/merge_node.py

@@ -14,6 +14,7 @@ from trilogy.core.models import (
     Conditional,
     Comparison,
     Parenthetical,
+    Environment,
 )
 from trilogy.utility import unique
 from trilogy.core.processing.nodes.base_node import (
@@ -108,6 +109,7 @@ class MergeNode(StrategyNode):
         conditions: Conditional | Comparison | Parenthetical | None = None,
         hidden_concepts: List[Concept] | None = None,
         virtual_output_concepts: List[Concept] | None = None,
+        existence_concepts: List[Concept] | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -123,18 +125,19 @@ class MergeNode(StrategyNode):
             conditions=conditions,
             hidden_concepts=hidden_concepts,
             virtual_output_concepts=virtual_output_concepts,
+            existence_concepts=existence_concepts,
         )
         self.join_concepts = join_concepts
         self.force_join_type = force_join_type
-        self.node_joins = node_joins
+        self.node_joins: List[NodeJoin] | None = node_joins
 
-        final_joins = []
-        if self.node_joins:
+        final_joins: List[NodeJoin] = []
+        if self.node_joins is not None:
             for join in self.node_joins:
                 if join.left_node.resolve().name == join.right_node.resolve().name:
                     continue
                 final_joins.append(join)
-        self.node_joins = final_joins
+            self.node_joins = final_joins
 
     def translate_node_joins(self, node_joins: List[NodeJoin]) -> List[BaseJoin]:
         joins = []
@@ -149,6 +152,7 @@ class MergeNode(StrategyNode):
                     right_datasource=right,
                     join_type=join.join_type,
                     concepts=join.concepts,
+                    concept_pairs=join.concept_pairs,
                 )
             )
         return joins
@@ -175,7 +179,12 @@ class MergeNode(StrategyNode):
         return joins
 
     def generate_joins(
-        self, final_datasets, final_joins, pregrain: Grain, grain: Grain
+        self,
+        final_datasets,
+        final_joins: List[NodeJoin] | None,
+        pregrain: Grain,
+        grain: Grain,
+        environment: Environment,
     ) -> List[BaseJoin]:
         # only finally, join between them for unique values
         dataset_list: List[QueryDatasource] = sorted(
@@ -186,15 +195,14 @@ class MergeNode(StrategyNode):
             f"{self.logging_prefix}{LOGGER_PREFIX} Merge node has {len(dataset_list)} parents, starting merge"
         )
         for item in dataset_list:
-            logger.info(f"{self.logging_prefix}{LOGGER_PREFIX} for {item.full_name}")
             logger.info(
-                f"{self.logging_prefix}{LOGGER_PREFIX} partial concepts {[x.address for x in item.partial_concepts]}"
+                f"{self.logging_prefix}{LOGGER_PREFIX} for {item.full_name} partial concepts {[x.address for x in item.partial_concepts]}"
             )
             logger.info(
                 f"{self.logging_prefix}{LOGGER_PREFIX} potential merge keys {[x.address+str(x.purpose) for x in item.output_concepts]} partial {[x.address for x in item.partial_concepts]}"
             )
 
-        if not final_joins:
+        if final_joins is None:
             if not pregrain.components:
                 logger.info(
                     f"{self.logging_prefix}{LOGGER_PREFIX} no grain components, doing full join"
@@ -204,7 +212,7 @@ class MergeNode(StrategyNode):
                 logger.info(
                     f"{self.logging_prefix}{LOGGER_PREFIX} inferring node joins to target grain {str(grain)}"
                 )
-                joins = get_node_joins(dataset_list, grain.components)
+                joins = get_node_joins(dataset_list, grain.components, environment)
         elif final_joins:
             logger.info(
                 f"{self.logging_prefix}{LOGGER_PREFIX} translating provided node joins {len(final_joins)}"
@@ -224,7 +232,7 @@ class MergeNode(StrategyNode):
             p.resolve() for p in self.parents
         ]
         merged: dict[str, QueryDatasource | Datasource] = {}
-        final_joins = self.node_joins
+        final_joins: List[NodeJoin] | None = self.node_joins
         for source in parent_sources:
             if source.full_name in merged:
                 logger.info(
@@ -280,12 +288,16 @@ class MergeNode(StrategyNode):
         for source in final_datasets:
             pregrain += source.grain
 
-        grain = Grain(
-            components=[
-                c
-                for c in pregrain.components
-                if c.address in [x.address for x in self.output_concepts]
-            ]
+        grain = (
+            self.grain
+            if self.grain
+            else Grain(
+                components=[
+                    c
+                    for c in pregrain.components
+                    if c.address in [x.address for x in self.output_concepts]
+                ]
+            )
         )
 
         logger.info(
@@ -293,7 +305,9 @@ class MergeNode(StrategyNode):
         )
 
         if len(final_datasets) > 1:
-            joins = self.generate_joins(final_datasets, final_joins, pregrain, grain)
+            joins = self.generate_joins(
+                final_datasets, final_joins, pregrain, grain, self.environment
+            )
         else:
             joins = []
 
@@ -318,7 +332,7 @@ class MergeNode(StrategyNode):
 
         qd_joins: List[BaseJoin | UnnestJoin] = [*joins]
         source_map = resolve_concept_map(
-            parent_sources,
+            list(merged.values()),
             targets=self.output_concepts,
             inherited_inputs=self.input_concepts + self.existence_concepts,
             full_joins=full_join_concepts,

trilogy/core/processing/nodes/select_node_v2.py

@@ -13,6 +13,9 @@ from trilogy.core.models import (
     Environment,
     UnnestJoin,
     Datasource,
+    Conditional,
+    Comparison,
+    Parenthetical,
 )
 from trilogy.utility import unique
 from trilogy.core.processing.nodes.base_node import StrategyNode
@@ -43,6 +46,8 @@ class SelectNode(StrategyNode):
         accept_partial: bool = False,
         grain: Optional[Grain] = None,
         force_group: bool | None = False,
+        conditions: Conditional | Comparison | Parenthetical | None = None,
+        hidden_concepts: List[Concept] | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -55,6 +60,8 @@ class SelectNode(StrategyNode):
             partial_concepts=partial_concepts,
             force_group=force_group,
             grain=grain,
+            conditions=conditions,
+            hidden_concepts=hidden_concepts,
         )
         self.accept_partial = accept_partial
         self.datasource = datasource
@@ -82,11 +89,12 @@ class SelectNode(StrategyNode):
             for x in c.alias.concept_arguments:
                 source_map[x.address] = {datasource}
         for x in all_concepts_final:
-            # add in any derived concepts to support a merge node
             if x.address not in source_map and x.derivation in (
                 PurposeLineage.MULTISELECT,
-                PurposeLineage.MERGE,
                 PurposeLineage.FILTER,
+                PurposeLineage.BASIC,
+                PurposeLineage.ROWSET,
+                PurposeLineage.BASIC,
             ):
                 source_map[x.address] = set()
 
@@ -111,6 +119,7 @@ class SelectNode(StrategyNode):
             condition=self.conditions,
             # select nodes should never group
             force_group=self.force_group,
+            hidden_concepts=self.hidden_concepts,
         )
 
     def resolve_from_constant_datasources(self) -> QueryDatasource:
@@ -127,6 +136,7 @@ class SelectNode(StrategyNode):
             joins=[],
             partial_concepts=[],
             source_type=SourceType.CONSTANT,
+            hidden_concepts=self.hidden_concepts,
         )
 
     def _resolve(self) -> QueryDatasource:
@@ -158,7 +168,7 @@ class SelectNode(StrategyNode):
                 return resolution
         required = [c.address for c in self.all_concepts]
         raise NoDatasourceException(
-            f"Could not find any way to associate required concepts {required}"
+            f"Could not find any way to resolve datasources for required concepts {required} with derivation {[x.derivation for x in self.all_concepts]}"
         )
 
     def copy(self) -> "SelectNode":
@@ -175,6 +185,8 @@ class SelectNode(StrategyNode):
             accept_partial=self.accept_partial,
             grain=self.grain,
             force_group=self.force_group,
+            conditions=self.conditions,
+            hidden_concepts=self.hidden_concepts,
         )
 
 
@@ -190,4 +202,6 @@ class ConstantNode(SelectNode):
             datasource=self.datasource,
             depth=self.depth,
             partial_concepts=list(self.partial_concepts),
+            conditions=self.conditions,
+            hidden_concepts=self.hidden_concepts,
         )