pytrilogy 0.3.142__cp312-cp312-win_amd64.whl

trilogy/core/processing/node_generators/recursive_node.py

@@ -0,0 +1,88 @@
+from typing import List
+
+from trilogy.constants import DEFAULT_NAMESPACE, RECURSIVE_GATING_CONCEPT, logger
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildConcept,
+    BuildFunction,
+    BuildGrain,
+    BuildWhereClause,
+    ComparisonOperator,
+    DataType,
+    Derivation,
+    Purpose,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.nodes import History, RecursiveNode, StrategyNode
+from trilogy.core.processing.utility import padding
+
+LOGGER_PREFIX = "[GEN_RECURSIVE_NODE]"
+
+GATING_CONCEPT = BuildConcept(
+    name=RECURSIVE_GATING_CONCEPT,
+    canonical_name=RECURSIVE_GATING_CONCEPT,
+    namespace=DEFAULT_NAMESPACE,
+    grain=BuildGrain(),
+    build_is_aggregate=False,
+    datatype=DataType.BOOL,
+    purpose=Purpose.KEY,
+    derivation=Derivation.BASIC,
+)
+
+
+def gen_recursive_node(
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    history: History,
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    source_concepts,
+    conditions: BuildWhereClause | None = None,
+) -> StrategyNode | None:
+    arguments = []
+    if isinstance(concept.lineage, BuildFunction):
+        arguments = concept.lineage.concept_arguments
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} Fetching recursive node for {concept.name} with arguments {arguments} and conditions {conditions}"
+    )
+    parent = source_concepts(
+        mandatory_list=arguments,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+        # conditions=conditions,
+    )
+    if not parent:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} could not find recursive node parents"
+        )
+        return None
+    outputs = (
+        [concept]
+        + arguments
+        + [
+            GATING_CONCEPT,
+        ]
+    )
+    base = RecursiveNode(
+        input_concepts=arguments,
+        output_concepts=outputs,
+        environment=environment,
+        parents=([parent] if (arguments or local_optional) else []),
+        # preexisting_conditions=conditions
+    )
+    # TODO:
+    # recursion will result in a union; group up to our final targets
+    wrapped_base = StrategyNode(
+        input_concepts=outputs,
+        output_concepts=outputs,
+        environment=environment,
+        parents=[base],
+        depth=depth,
+        conditions=BuildComparison(
+            left=GATING_CONCEPT, right=True, operator=ComparisonOperator.IS
+        ),
+    )
+    return wrapped_base

trilogy/core/processing/node_generators/rowset_node.py

@@ -0,0 +1,165 @@
+from typing import List
+
+from trilogy.constants import logger
+from trilogy.core.enums import Derivation
+from trilogy.core.exceptions import UnresolvableQueryException
+from trilogy.core.models.author import MultiSelectLineage, SelectLineage
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildGrain,
+    BuildRowsetItem,
+    BuildRowsetLineage,
+    BuildWhereClause,
+    Factory,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
+from trilogy.core.processing.utility import concept_to_relevant_joins, padding
+
+LOGGER_PREFIX = "[GEN_ROWSET_NODE]"
+
+
+def gen_rowset_node(
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History,
+    conditions: BuildWhereClause | None = None,
+) -> StrategyNode | None:
+    from trilogy.core.query_processor import get_query_node
+
+    if not isinstance(concept.lineage, BuildRowsetItem):
+        raise SyntaxError(
+            f"Invalid lineage passed into rowset fetch, got {type(concept.lineage)}, expected {BuildRowsetItem}"
+        )
+    lineage: BuildRowsetItem = concept.lineage
+    rowset: BuildRowsetLineage = lineage.rowset
+    select: SelectLineage | MultiSelectLineage = lineage.rowset.select
+
+    node = get_query_node(history.base_environment, select)
+
+    if not node:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate parent rowset node for {concept}"
+        )
+        raise UnresolvableQueryException(
+            f"Cannot generate parent select for concept {concept} in rowset {rowset.name}; ensure the rowset is a valid statement."
+        )
+    enrichment = set([x.address for x in local_optional])
+
+    factory = Factory(environment=history.base_environment, grain=select.grain)
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} rowset derived concepts are {lineage.rowset.derived_concepts}"
+    )
+    concept_pool = list(environment.concepts.values()) + list(
+        environment.alias_origin_lookup.values()
+    )
+    rowset_outputs = [
+        x.address for x in concept_pool if x.address in lineage.rowset.derived_concepts
+    ]
+    rowset_relevant: list[BuildConcept] = [
+        v for v in concept_pool if v.address in rowset_outputs
+    ]
+
+    additional_relevant = [
+        factory.build(x) for x in select.output_components if x.address in enrichment
+    ]
+    # add in other other concepts
+    node.set_output_concepts(rowset_relevant + additional_relevant)
+    if select.where_clause:
+        for item in additional_relevant:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} adding {item} to partial concepts"
+            )
+            node.partial_concepts.append(item)
+
+    node.grain = BuildGrain.from_concepts(
+        [
+            x
+            for x in node.output_concepts
+            if x.address
+            not in [
+                y
+                for y in node.hidden_concepts
+                if y in environment.concepts
+                and environment.concepts[y].derivation != Derivation.ROWSET
+            ]
+        ],
+    )
+
+    node.rebuild_cache()
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} final output is {[x.address for x in node.output_concepts]} with grain {node.grain}"
+    )
+    if not local_optional or all(
+        (
+            x.address in node.output_concepts
+            or (z in x.pseudonyms for z in node.output_concepts)
+        )
+        and x.address not in node.partial_concepts
+        for x in local_optional
+    ):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node as all optional {[x.address for x in local_optional]} found or no optional; exiting early."
+        )
+        return node
+    remaining = [
+        x
+        for x in local_optional
+        if x not in node.output_concepts or x in node.partial_concepts
+    ]
+    possible_joins = concept_to_relevant_joins(
+        [x for x in node.output_concepts if x.derivation != Derivation.ROWSET]
+    )
+    if not possible_joins:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node to get {[x.address for x in local_optional]}; have {[x.address for x in node.output_concepts]}"
+        )
+        return node
+    if any(x.derivation == Derivation.ROWSET for x in possible_joins):
+
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} cannot enrich rowset node with rowset concepts; exiting early"
+        )
+        return node
+    logger.info([x.address for x in possible_joins + local_optional])
+    enrich_node: MergeNode = source_concepts(  # this fetches the parent + join keys
+        # to then connect to the rest of the query
+        mandatory_list=possible_joins + remaining,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        conditions=conditions,
+        history=history,
+    )
+    if not enrich_node:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset enrichment node for {concept} with optional {local_optional}, returning just rowset node"
+        )
+        return node
+
+    non_hidden = [
+        x for x in node.output_concepts if x.address not in node.hidden_concepts
+    ]
+    for x in possible_joins:
+        if x.address in node.hidden_concepts:
+            node.unhide_output_concepts([x])
+    non_hidden_enrich = [
+        x
+        for x in enrich_node.output_concepts
+        if x.address not in enrich_node.hidden_concepts
+    ]
+    return MergeNode(
+        input_concepts=non_hidden + non_hidden_enrich,
+        output_concepts=non_hidden + local_optional,
+        environment=environment,
+        depth=depth,
+        parents=[
+            node,
+            enrich_node,
+        ],
+        preexisting_conditions=conditions.conditional if conditions else None,
+    )

trilogy/core/processing/node_generators/select_helpers/datasource_injection.py

@@ -0,0 +1,261 @@
+import sys
+from collections import defaultdict
+from datetime import date, datetime, timedelta
+from typing import List, Tuple, TypeVar
+
+from trilogy.core.enums import ComparisonOperator, FunctionType
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildConcept,
+    BuildConditional,
+    BuildDatasource,
+    BuildFunction,
+    BuildParenthetical,
+)
+from trilogy.core.models.core import DataType
+
+# Define a generic type that ensures start and end are the same type
+T = TypeVar("T", int, float, date, datetime)
+
+
+def reduce_expression(
+    var: BuildConcept, group_tuple: list[tuple[ComparisonOperator, T]]
+) -> bool:
+    # Track ranges
+    lower_check: T
+    upper_check: T
+
+    # if var.datatype in (DataType.FLOAT,):
+    #     lower_check = float("-inf")  # type: ignore
+    #     upper_check = float("inf")  # type: ignore
+    if var.datatype == DataType.INTEGER:
+        lower_check = float("-inf")  # type: ignore
+        upper_check = float("inf")  # type: ignore
+    elif var.datatype == DataType.DATE:
+        lower_check = date.min  # type: ignore
+        upper_check = date.max  # type: ignore
+
+    elif var.datatype == DataType.DATETIME:
+        lower_check = datetime.min  # type: ignore
+        upper_check = datetime.max  # type: ignore
+    elif var.datatype == DataType.BOOL:
+        lower_check = False  # type: ignore
+        upper_check = True  # type: ignore
+    elif var.datatype == DataType.FLOAT:
+        lower_check = float("-inf")  # type: ignore
+        upper_check = float("inf")  # type: ignore
+    else:
+        return False
+
+    ranges: list[Tuple[T, T]] = []
+    for op, value in group_tuple:
+        increment: int | timedelta | float
+        if isinstance(value, date):
+            increment = timedelta(days=1)
+        elif isinstance(value, datetime):
+            increment = timedelta(seconds=1)
+        elif isinstance(value, int):
+            increment = 1
+        elif isinstance(value, float):
+            increment = sys.float_info.epsilon
+
+        if op == ">":
+            ranges.append(
+                (
+                    value + increment,
+                    upper_check,
+                )
+            )
+        elif op == ">=":
+            ranges.append(
+                (
+                    value,
+                    upper_check,
+                )
+            )
+        elif op == "<":
+            ranges.append(
+                (
+                    lower_check,
+                    value - increment,
+                )
+            )
+        elif op == "<=":
+            ranges.append(
+                (
+                    lower_check,
+                    value,
+                )
+            )
+        elif op == "=":
+            ranges.append(
+                (
+                    value,
+                    value,
+                )
+            )
+        elif op == ComparisonOperator.IS:
+            ranges.append(
+                (
+                    value,
+                    value,
+                )
+            )
+        elif op == ComparisonOperator.NE:
+            pass
+        else:
+            return False
+    return is_fully_covered(lower_check, upper_check, ranges, increment)
+
+
+TARGET_TYPES = (
+    int,
+    date,
+    float,
+    datetime,
+    bool,
+)
+REDUCABLE_TYPES = (int, float, date, bool, datetime, BuildFunction)
+
+
+def simplify_conditions(
+    conditions: list[BuildComparison | BuildConditional | BuildParenthetical],
+) -> bool:
+    # Group conditions by variable
+    grouped: dict[
+        BuildConcept, list[tuple[ComparisonOperator, datetime | int | date | float]]
+    ] = defaultdict(list)
+    for condition in conditions:
+        if not isinstance(condition, BuildComparison):
+            return False
+        left_is_concept = False
+        left_is_reducable = False
+        right_is_concept = False
+        right_is_reducable = False
+        if isinstance(condition.left, BuildConcept):
+            left_is_concept = True
+        elif isinstance(condition.left, REDUCABLE_TYPES):
+            left_is_reducable = True
+
+        if isinstance(condition.right, BuildConcept):
+            right_is_concept = True
+        elif isinstance(condition.right, REDUCABLE_TYPES):
+            right_is_reducable = True
+
+        if not (
+            (left_is_concept and right_is_reducable)
+            or (right_is_concept and left_is_reducable)
+        ):
+            return False
+        if left_is_concept:
+            concept = condition.left
+            raw_comparison = condition.right
+        else:
+            concept = condition.right
+            raw_comparison = condition.left
+
+        if isinstance(raw_comparison, BuildFunction):
+            if not raw_comparison.operator == FunctionType.CONSTANT:
+                return False
+            first_arg = raw_comparison.arguments[0]
+            if not isinstance(first_arg, TARGET_TYPES):
+                return False
+            comparison = first_arg
+        else:
+            if not isinstance(raw_comparison, TARGET_TYPES):
+                return False
+            comparison = raw_comparison
+
+        if not isinstance(comparison, REDUCABLE_TYPES):
+            return False
+
+        var: BuildConcept = concept  # type: ignore
+        op = condition.operator
+        grouped[var].append((op, comparison))
+
+    simplified = []
+    for var, group_tuple in grouped.items():
+        simplified.append(reduce_expression(var, group_tuple))  # type: ignore
+
+    # Final simplification
+    return True if all(isinstance(s, bool) and s for s in simplified) else False
+
+
+def boolean_fully_covered(
+    start: bool,
+    end: bool,
+    ranges: List[Tuple[bool, bool]],
+):
+    all = []
+    for r_start, r_end in ranges:
+        if r_start is True and r_end is True:
+            all.append(True)
+        elif r_start is False and r_end is False:
+            all.append(False)
+    return set(all) == {False, True}
+
+
+def is_fully_covered(
+    start: T,
+    end: T,
+    ranges: List[Tuple[T, T]],
+    increment: int | timedelta | float,
+):
+    """
+    Check if the list of range pairs fully covers the set [start, end].
+
+    Parameters:
+    - start (int or float): The starting value of the set to cover.
+    - end (int or float): The ending value of the set to cover.
+    - ranges (list of tuples): List of range pairs [(start1, end1), (start2, end2), ...].
+
+    Returns:
+    - bool: True if the ranges fully cover [start, end], False otherwise.
+    """
+    if isinstance(start, bool) and isinstance(end, bool):
+        # convert each element of each tuple to a boolean
+        bool_ranges = [(bool(r_start), bool(r_end)) for r_start, r_end in ranges]
+
+        return boolean_fully_covered(start, end, bool_ranges)
+    # Sort ranges by their start values (and by end values for ties)
+    ranges.sort()
+
+    # Check for gaps
+    current_end = start
+    for r_start, r_end in ranges:
+        # If there's a gap between the current range and the previous coverage
+        if (r_start - current_end) > increment:  # type: ignore
+            return False
+        # Extend the current coverage
+        current_end = max(current_end, r_end)
+
+    # If the loop ends and we haven't reached the end, return False
+    return current_end >= end
+
+
+def get_union_sources(
+    datasources: list[BuildDatasource], concepts: list[BuildConcept]
+) -> List[list[BuildDatasource]]:
+    candidates: list[BuildDatasource] = []
+
+    for x in datasources:
+        if any([c.address in x.output_concepts for c in concepts]):
+            if (
+                any([c.address in x.partial_concepts for c in concepts])
+                and x.non_partial_for
+            ):
+                candidates.append(x)
+    assocs: dict[str, list[BuildDatasource]] = defaultdict(list[BuildDatasource])
+    for x in candidates:
+        if not x.non_partial_for:
+            continue
+        if not len(x.non_partial_for.concept_arguments) == 1:
+            continue
+        merge_key = x.non_partial_for.concept_arguments[0]
+        assocs[merge_key.address].append(x)
+    final: list[list[BuildDatasource]] = []
+    for _, dses in assocs.items():
+        conditions = [c.non_partial_for.conditional for c in dses if c.non_partial_for]
+        if simplify_conditions(conditions):
+            final.append(dses)
+    return final