pytrilogy 0.0.2.17__py3-none-any.whl → 0.0.2.19__py3-none-any.whl

trilogy/core/processing/node_generators/select_node.py

@@ -1,424 +1,31 @@
-from typing import List, Optional, Callable
-
 from trilogy.core.enums import PurposeLineage
 from trilogy.core.models import (
     Concept,
     Environment,
-    Grain,
     LooseConceptList,
-    Datasource,
     WhereClause,
 )
 from trilogy.core.processing.nodes import (
     StrategyNode,
-    SelectNode,
-    MergeNode,
-    GroupNode,
-    ConstantNode,
 )
 from trilogy.core.exceptions import NoDatasourceException
-import networkx as nx
-from trilogy.core.graph_models import concept_to_node, datasource_to_node
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding
-from dataclasses import dataclass
+from trilogy.core.processing.node_generators.select_merge_node import (
+    gen_select_merge_node,
+)
 
 LOGGER_PREFIX = "[GEN_SELECT_NODE]"
 
 
-@dataclass
-class DatasourceMatch:
-    key: str
-    datasource: Datasource
-    matched: LooseConceptList
-    partial: LooseConceptList
-    nullable: LooseConceptList
-
-    def __repr__(self):
-        return f"DatasourceMatch({self.key}, {self.datasource.identifier}, {str(self.matched)}, {str(self.partial)})"
-
-
-def dm_to_strategy_node(
-    dm: DatasourceMatch,
-    target_grain: Grain,
-    environment: Environment,
-    g: nx.DiGraph,
-    depth: int,
-    source_concepts: Callable,
-    accept_partial: bool = False,
-) -> StrategyNode:
-    datasource = dm.datasource
-
-    if target_grain and target_grain.issubset(datasource.grain):
-        if all([x in dm.matched for x in target_grain.components]):
-            force_group = False
-        # if we are not returning the grain
-        # we have to group
-        else:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} not all grain components {target_grain} are in output {str(dm.matched)}, group to actual grain"
-            )
-            force_group = True
-    elif all([x in dm.matched for x in datasource.grain.components]):
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} query output includes all grain components, no reason to group further"
-        )
-        force_group = False
-    else:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} target grain is not subset of datasource grain {datasource.grain}, required to group"
-        )
-        force_group = True
-    bcandidate: StrategyNode = SelectNode(
-        input_concepts=[c.concept for c in datasource.columns],
-        output_concepts=dm.matched.concepts,
-        environment=environment,
-        g=g,
-        parents=[],
-        depth=depth,
-        partial_concepts=dm.partial.concepts,
-        nullable_concepts=dm.nullable.concepts,
-        accept_partial=accept_partial,
-        datasource=datasource,
-        grain=datasource.grain,
-        conditions=datasource.where.conditional if datasource.where else None,
-    )
-    # we need to nest the group node one further
-    if force_group is True:
-        candidate: StrategyNode = GroupNode(
-            output_concepts=dm.matched.concepts,
-            input_concepts=dm.matched.concepts,
-            environment=environment,
-            g=g,
-            parents=[bcandidate],
-            depth=depth,
-            partial_concepts=bcandidate.partial_concepts,
-        )
-    else:
-        candidate = bcandidate
-    return candidate
-
-
-def gen_select_nodes_from_tables_v2(
-    mandatory_concept: Concept,
-    all_concepts: List[Concept],
-    g: nx.DiGraph,
-    environment: Environment,
-    depth: int,
-    target_grain: Grain,
-    source_concepts: Callable,
-    accept_partial: bool = False,
-    conditions: WhereClause | None = None,
-) -> tuple[bool, list[Concept], list[StrategyNode]]:
-    # if we have only constants
-    # we don't need a table
-    # so verify nothing, select node will render
-    all_lcl = LooseConceptList(concepts=all_concepts)
-    if all([c.derivation == PurposeLineage.CONSTANT for c in all_lcl]):
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} All concepts {[x.address for x in all_lcl]} are constants, returning constant node"
-        )
-        return (
-            True,
-            all_lcl.concepts,
-            [
-                ConstantNode(
-                    output_concepts=all_lcl.concepts,
-                    input_concepts=[],
-                    environment=environment,
-                    g=g,
-                    parents=[],
-                    depth=depth,
-                    # no partial for constants
-                    partial_concepts=[],
-                    force_group=False,
-                )
-            ],
-        )
-    # otherwise, we need to look for a table
-    nodes_to_find = [concept_to_node(x.with_default_grain()) for x in all_lcl.concepts]
-    matches: dict[str, DatasourceMatch] = {}
-    for k, datasource in environment.datasources.items():
-        matched = []
-        matched_paths = []
-        for idx, req_concept in enumerate(nodes_to_find):
-            try:
-                path = nx.shortest_path(
-                    g,
-                    source=datasource_to_node(datasource),
-                    target=req_concept,
-                )
-                ds_valid = (
-                    sum(
-                        [
-                            1 if g.nodes[node]["type"] == "datasource" else 0
-                            for node in path
-                        ]
-                    )
-                    == 1
-                )
-                address_valid = (
-                    sum(
-                        [
-                            (
-                                1
-                                if g.nodes[node]["type"] == "concept"
-                                and g.nodes[node]["concept"].address
-                                != all_lcl.concepts[idx].address
-                                else 0
-                            )
-                            for node in path
-                        ]
-                    )
-                    == 0
-                )
-                if ds_valid and address_valid:
-                    matched_paths.append(path)
-                    matched.append(all_lcl.concepts[idx])
-
-            except nx.NodeNotFound:
-                continue
-            except nx.exception.NetworkXNoPath:
-                continue
-        dm = DatasourceMatch(
-            key=k,
-            datasource=datasource,
-            matched=LooseConceptList(concepts=matched),
-            partial=LooseConceptList(
-                concepts=[
-                    c.concept
-                    for c in datasource.columns
-                    if not c.is_complete and c.concept.address in all_lcl
-                ]
-            ),
-            nullable=LooseConceptList(
-                concepts=[
-                    c.concept
-                    for c in datasource.columns
-                    if c.is_nullable and c.concept in all_lcl
-                ]
-            ),
-        )
-        if not matched:
-            continue
-        if mandatory_concept.address not in dm.matched:
-            continue
-        if not accept_partial and dm.partial.addresses:
-            continue
-        matches[k] = dm
-    found: set[str] = set()
-    all_found = False
-    all_checked = False
-    to_return: list[StrategyNode] = []
-    if not matches:
-        return False, [], []
-    while not all_found and not all_checked:
-        final_key: str = max(
-            matches,
-            key=lambda x: len(
-                [m for m in matches[x].matched.addresses if m not in found]
-            )
-            - 0.1 * len(matches[x].partial.addresses),
-        )
-        final: DatasourceMatch = matches[final_key]
-
-        candidate = dm_to_strategy_node(
-            final,
-            target_grain=Grain(
-                components=[
-                    x for x in target_grain.components if x.address in final.matched
-                ]
-            ),
-            environment=environment,
-            g=g,
-            depth=depth,
-            accept_partial=accept_partial,
-            source_concepts=source_concepts,
-        )
-        to_return.append(candidate)
-        del matches[final_key]
-        found = found.union(final.matched.addresses)
-        all_found = all_lcl.addresses.issubset(found)
-        all_checked = len(matches) == 0
-    return all_found, [x for x in all_concepts if x.address in found], to_return
-
-
-def gen_select_node_from_table(
-    target_concept: Concept,
-    all_concepts: List[Concept],
-    g: nx.DiGraph,
-    environment: Environment,
-    depth: int,
-    target_grain: Grain,
-    source_concepts,
-    accept_partial: bool = False,
-    conditions: WhereClause | None = None,
-) -> Optional[StrategyNode]:
-    # if we have only constants
-    # we don't need a table
-    # so verify nothing, select node will render
-    all_lcl = LooseConceptList(concepts=all_concepts)
-    if all([c.derivation == PurposeLineage.CONSTANT for c in all_concepts]):
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} All concepts {[x.address for x in all_concepts]} are constants, returning constant node"
-        )
-        return ConstantNode(
-            output_concepts=all_concepts,
-            input_concepts=[],
-            environment=environment,
-            g=g,
-            parents=[],
-            depth=depth,
-            # no partial for constants
-            partial_concepts=[],
-            force_group=False,
-        )
-    candidates: dict[str, StrategyNode] = {}
-    scores: dict[str, int] = {}
-    # otherwise, we need to look for a table
-    nodes_to_find = [concept_to_node(x.with_default_grain()) for x in all_concepts]
-    for datasource in environment.datasources.values():
-        all_found = True
-        for idx, req_concept in enumerate(nodes_to_find):
-            try:
-                path = nx.shortest_path(
-                    g,
-                    source=datasource_to_node(datasource),
-                    target=req_concept,
-                )
-            except nx.NodeNotFound as e:
-                # just to provide better error
-                ncandidates = [
-                    datasource_to_node(datasource),
-                    req_concept,
-                ]
-                for ncandidate in ncandidates:
-                    try:
-                        g.nodes[ncandidate]
-                    except KeyError:
-                        raise nx.exception.NetworkXNoPath
-                raise e
-            except nx.exception.NetworkXNoPath:
-                all_found = False
-                break
-            # 2023-10-18 - more strict condition then below
-            # 2023-10-20 - we need this to get through abstract concepts
-            # but we may want to add a filter to avoid using this for anything with lineage
-            # if len(path) != 2:
-            #     all_found = False
-            #     break
-            if len([p for p in path if g.nodes[p]["type"] == "datasource"]) != 1:
-                all_found = False
-                break
-            for node in path:
-                if g.nodes[node]["type"] == "datasource":
-                    continue
-                if g.nodes[node]["concept"].address == all_concepts[idx].address:
-                    continue
-                all_found = False
-                break
-
-        if not all_found:
-            # skip to next node
-            continue
-        partial_concepts = [
-            c.concept
-            for c in datasource.columns
-            if not c.is_complete and c.concept in all_lcl
-        ]
-        partial_lcl = LooseConceptList(concepts=partial_concepts)
-        nullable_concepts = [
-            c.concept
-            for c in datasource.columns
-            if c.is_nullable and c.concept in all_lcl
-        ]
-        nullable_lcl = LooseConceptList(concepts=nullable_concepts)
-        if not accept_partial and target_concept in partial_lcl:
-            continue
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} target grain is {str(target_grain)}"
-        )
-        if target_grain and target_grain.issubset(datasource.grain):
-
-            if (
-                all([x in all_lcl for x in target_grain.components])
-                and target_grain == datasource.grain
-            ):
-                logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} target grain components match all lcl, group to false"
-                )
-                force_group = False
-            # if we are not returning the grain
-            # we have to group
-            else:
-                logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} not all grain components {target_grain} are in output {str(all_lcl)}, group to actual grain"
-                )
-                force_group = True
-        elif all([x in all_lcl for x in datasource.grain.components]):
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} query output includes all grain components, no reason to group further"
-            )
-            force_group = False
-        else:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} target grain is not subset of datasource grain {datasource.grain}, required to group"
-            )
-            force_group = True
-
-        bcandidate: StrategyNode = SelectNode(
-            input_concepts=[c.concept for c in datasource.columns],
-            output_concepts=all_concepts,
-            environment=environment,
-            g=g,
-            parents=[],
-            depth=depth,
-            partial_concepts=[c for c in all_concepts if c in partial_lcl],
-            nullable_concepts=[c for c in all_concepts if c in nullable_lcl],
-            accept_partial=accept_partial,
-            datasource=datasource,
-            grain=Grain(components=all_concepts),
-            conditions=datasource.where.conditional if datasource.where else None,
-        )
-        # if conditions:
-        #     for component in conditions.components:
-        #         if
-        # we need to nest the group node one further
-        if force_group is True:
-            candidate: StrategyNode = GroupNode(
-                output_concepts=all_concepts,
-                input_concepts=all_concepts,
-                environment=environment,
-                g=g,
-                parents=[bcandidate],
-                depth=depth,
-                partial_concepts=bcandidate.partial_concepts,
-                nullable_concepts=bcandidate.nullable_concepts,
-            )
-        else:
-            candidate = bcandidate
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} found select node with {datasource.identifier}, force group is {force_group}, returning {candidate.output_lcl}"
-        )
-        candidates[datasource.identifier] = candidate
-        scores[datasource.identifier] = -len(partial_concepts)
-    if not candidates:
-        return None
-    final = max(candidates, key=lambda x: scores[x])
-    return candidates[final]
-
-
 def gen_select_node(
     concept: Concept,
-    local_optional: List[Concept],
+    local_optional: list[Concept],
     environment: Environment,
     g,
     depth: int,
-    source_concepts,
     accept_partial: bool = False,
     fail_if_not_found: bool = True,
-    accept_partial_optional: bool = True,
-    target_grain: Grain | None = None,
     conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
     all_concepts = [concept] + local_optional
@@ -427,135 +34,24 @@ def gen_select_node(
         concepts=[
             x
             for x in all_concepts
-            if x.address in [z.address for z in environment.materialized_concepts]
+            if x.address in [z for z in environment.materialized_concepts]
             or x.derivation == PurposeLineage.CONSTANT
         ]
     )
-    if not target_grain:
-        target_grain = Grain()
-        for ac in all_concepts:
-            target_grain += ac.grain
-    if target_grain.abstract:
-        target_grain = Grain(components=all_concepts)
     if materialized_lcl != all_lcl:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concept.address}"
-            f" as it + optional (looking for all {all_lcl}) includes non-materialized concepts"
+            f" as it + optional includes non-materialized concepts (looking for all {all_lcl}) "
         )
         if fail_if_not_found:
             raise NoDatasourceException(f"No datasource exists for {concept}")
         return None
 
-    ds: StrategyNode | None = None
-
-    # attempt to select all concepts from table
-    ds = gen_select_node_from_table(
-        concept,
+    return gen_select_merge_node(
         [concept] + local_optional,
         g=g,
         environment=environment,
         depth=depth,
         accept_partial=accept_partial,
-        target_grain=target_grain,
-        source_concepts=source_concepts,
         conditions=conditions,
     )
-    if ds:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Found select node with all target concepts, force group is {ds.force_group}, target grain {target_grain}"
-        )
-        return ds
-    # if we cannot find a match
-    all_found, found, parents = gen_select_nodes_from_tables_v2(
-        concept,
-        all_concepts,
-        g,
-        environment,
-        depth=depth,
-        target_grain=target_grain,
-        accept_partial=accept_partial,
-        source_concepts=source_concepts,
-        conditions=conditions,
-    )
-    if parents and (all_found or accept_partial_optional):
-        all_partial = [
-            c
-            for c in all_concepts
-            if all(
-                [c.address in [x.address for x in p.partial_concepts] for p in parents]
-            )
-        ]
-
-        all_nullable = [
-            c
-            for c in all_concepts
-            if any(
-                [c.address in [x.address for x in p.nullable_concepts] for p in parents]
-            )
-        ]
-
-        if all_found:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} found all optional {[c.address for c in local_optional]} via joins"
-            )
-        else:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} found some optional {[x.address for x in found]}, and partial return allowed: returning"
-            )
-
-        force_group = None
-        inferred_grain = sum([x.grain for x in parents if x.grain], Grain())
-        for candidate in parents:
-            if candidate.grain and not candidate.grain.issubset(target_grain):
-                force_group = True
-        if len(parents) == 1:
-            candidate = parents[0]
-        else:
-            candidate = MergeNode(
-                output_concepts=[concept] + found,
-                input_concepts=[concept] + found,
-                environment=environment,
-                g=g,
-                parents=parents,
-                depth=depth,
-                partial_concepts=all_partial,
-                nullable_concepts=all_nullable,
-                grain=inferred_grain,
-            )
-
-        candidate.depth += 1
-        if force_group:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} datasource grain {inferred_grain} does not match target grain {target_grain} for select, adding group node"
-            )
-            return GroupNode(
-                output_concepts=candidate.output_concepts,
-                input_concepts=candidate.output_concepts,
-                environment=environment,
-                g=g,
-                parents=[candidate],
-                depth=depth,
-                partial_concepts=candidate.partial_concepts,
-            )
-        else:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} datasource grain {inferred_grain} matches target grain {target_grain} for select, returning without group"
-            )
-        return candidate
-
-    if not accept_partial_optional:
-        return None
-    ds = gen_select_node_from_table(
-        concept,
-        [concept],
-        g=g,
-        environment=environment,
-        depth=depth,
-        accept_partial=accept_partial,
-        target_grain=Grain(components=[concept]),
-        source_concepts=source_concepts,
-    )
-
-    if not ds and fail_if_not_found:
-        raise NoDatasourceException(f"No datasource exists for {concept}")
-    return ds

trilogy/core/processing/node_generators/unnest_node.py

@@ -6,7 +6,7 @@ from trilogy.core.processing.nodes import UnnestNode, History, StrategyNode
 from trilogy.core.processing.utility import padding
 from trilogy.constants import logger
 
-LOGGER_PREFIX = "[GEN_ROWSET_NODE]"
+LOGGER_PREFIX = "[GEN_UNNEST_NODE]"
 
 
 def gen_unnest_node(
@@ -54,11 +54,12 @@ def gen_unnest_node(
     # as unnest operations are not valid in all situations
     # TODO: inline this node when we can detect it's safe
     new = StrategyNode(
-        input_concepts=[concept] + local_optional,
-        output_concepts=[concept] + local_optional,
+        input_concepts=base.output_concepts,
+        output_concepts=base.output_concepts,
         environment=environment,
         g=g,
         parents=[base],
+        preexisting_conditions=conditions.conditional if conditions else None,
     )
     qds = new.resolve()
     assert qds.source_map[concept.address] == {base.resolve()}

trilogy/core/processing/node_generators/window_node.py

@@ -3,17 +3,9 @@ from typing import List
 
 from trilogy.core.models import Concept, WindowItem, Environment, WhereClause
 from trilogy.utility import unique
-from trilogy.core.processing.nodes import (
-    WindowNode,
-)
-from trilogy.core.processing.nodes import MergeNode, History
-
+from trilogy.core.processing.nodes import WindowNode, StrategyNode, History
 from trilogy.constants import logger
-from trilogy.core.processing.utility import padding, create_log_lambda
-from trilogy.core.processing.node_generators.common import (
-    gen_enrichment_node,
-)
-from trilogy.core.processing.utility import concept_to_relevant_joins
+from trilogy.core.processing.utility import padding
 
 LOGGER_PREFIX = "[GEN_WINDOW_NODE]"
 
@@ -39,10 +31,10 @@ def gen_window_node(
     source_concepts,
     history: History | None = None,
     conditions: WhereClause | None = None,
-) -> WindowNode | MergeNode | None:
+) -> StrategyNode | None:
     parent_concepts = resolve_window_parent_concepts(concept)
     parent_node = source_concepts(
-        mandatory_list=parent_concepts,
+        mandatory_list=parent_concepts + local_optional,
         environment=environment,
         g=g,
         depth=depth + 1,
@@ -69,8 +61,8 @@ def gen_window_node(
         )
         raise SyntaxError
     _window_node = WindowNode(
-        input_concepts=parent_concepts,
-        output_concepts=[concept] + parent_concepts,
+        input_concepts=parent_concepts + local_optional,
+        output_concepts=[concept] + parent_concepts + local_optional,
         environment=environment,
         g=g,
         parents=[
@@ -80,29 +72,12 @@ def gen_window_node(
     )
     _window_node.rebuild_cache()
     _window_node.resolve()
-    window_node = MergeNode(
-        parents=[_window_node],
-        environment=environment,
-        g=g,
-        input_concepts=[concept] + _window_node.input_concepts,
-        output_concepts=_window_node.output_concepts,
-        grain=_window_node.grain,
-        force_group=False,
-        depth=depth,
-    )
-    window_node.resolve()
-    if not local_optional:
-        return window_node
-    logger.info(f"{padding(depth)}{LOGGER_PREFIX} window node requires enrichment")
-    return gen_enrichment_node(
-        window_node,
-        join_keys=concept_to_relevant_joins(parent_concepts),
-        local_optional=local_optional,
+    window_node = StrategyNode(
+        input_concepts=[concept] + local_optional,
+        output_concepts=[concept] + local_optional,
         environment=environment,
         g=g,
-        depth=depth,
-        source_concepts=source_concepts,
-        log_lambda=create_log_lambda(LOGGER_PREFIX, depth, logger),
-        history=history,
-        conditions=conditions,
+        parents=[_window_node],
+        preexisting_conditions=conditions.conditional if conditions else None,
     )
+    return window_node

trilogy/core/processing/nodes/__init__.py

@@ -144,8 +144,6 @@ class History(BaseModel):
             depth + 1,
             fail_if_not_found=fail_if_not_found,
             accept_partial=accept_partial,
-            accept_partial_optional=accept_partial_optional,
-            source_concepts=source_concepts,
             conditions=conditions,
         )
         self.select_history[fingerprint] = gen