pytrilogy 0.0.2.49__py3-none-any.whl → 0.0.2.51__py3-none-any.whl

trilogy/core/processing/concept_strategies_v3.py

@@ -359,7 +359,6 @@ def generate_node(
             input_concepts=[],
             output_concepts=constant_targets,
             environment=environment,
-            g=g,
             parents=[],
             depth=depth + 1,
         )
@@ -450,6 +449,7 @@ def generate_node(
             conditions=conditions,
         )
         if not check:
+
             logger.info(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Could not resolve root concepts, checking for expanded concepts"
             )
@@ -471,7 +471,6 @@ def generate_node(
                         x
                         for x in ex_resolve.output_concepts
                         if x.address not in [y.address for y in root_targets]
-                        and x not in ex_resolve.grain.components
                     ]
 
                     pseudonyms = [
@@ -479,10 +478,19 @@ def generate_node(
                         for x in extra
                         if any(x.address in y.pseudonyms for y in root_targets)
                     ]
-                    # if we're only connected by a pseudonym, keep those in output
-                    expanded.set_output_concepts(root_targets + pseudonyms)
+                    logger.info(
+                        f"{depth_to_prefix(depth)}{LOGGER_PREFIX} reducing final outputs, was {[c.address for c in ex_resolve.output_concepts]} with extra {[c.address for c in extra]}"
+                    )
+                    base = [
+                        x for x in ex_resolve.output_concepts if x.address not in extra
+                    ]
+                    for x in root_targets:
+                        if x.address not in base:
+                            base.append(x)
+                    expanded.set_output_concepts(base)
                     # but hide them
                     if pseudonyms:
+                        expanded.add_output_concepts(pseudonyms)
                         logger.info(
                             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Hiding pseudonyms{[c.address for c in pseudonyms]}"
                         )
@@ -906,10 +914,10 @@ def _search_concepts(
                 input_concepts=non_virtual,
                 output_concepts=non_virtual,
                 environment=environment,
-                g=g,
                 parents=stack,
                 depth=depth,
             )
+
         # ensure we can resolve our final merge
         output.resolve()
         if condition_required and conditions:
@@ -919,7 +927,7 @@ def _search_concepts(
                     output, environment, g, where=conditions, history=history
                 )
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Graph is connected, returning merge node, partial {[c.address for c in output.partial_concepts]}"
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node partial {[c.address for c in output.partial_concepts]}"
         )
         return output
 
@@ -987,7 +995,6 @@ def source_query_concepts(
             x for x in root.output_concepts if x.address not in root.hidden_concepts
         ],
         environment=environment,
-        g=g,
         parents=[root],
         partial_concepts=root.partial_concepts,
     )

trilogy/core/processing/node_generators/basic_node.py

@@ -44,7 +44,7 @@ def gen_basic_node(
     conditions: WhereClause | None = None,
 ):
     depth_prefix = "\t" * depth
-    parent_concepts = resolve_function_parent_concepts(concept)
+    parent_concepts = resolve_function_parent_concepts(concept, environment=environment)
 
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has parents {[x.address for x in parent_concepts]}"
@@ -61,12 +61,16 @@ def gen_basic_node(
             f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has equivalent optional {[x.address for x in equivalent_optional]}"
         )
     for eo in equivalent_optional:
-        parent_concepts += resolve_function_parent_concepts(eo)
+        parent_concepts += resolve_function_parent_concepts(eo, environment=environment)
     non_equivalent_optional = [
         x for x in local_optional if x not in equivalent_optional
     ]
+    all_parents = parent_concepts + non_equivalent_optional
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} Fetching parents {[x.address for x in all_parents]}"
+    )
     parent_node: StrategyNode = source_concepts(
-        mandatory_list=parent_concepts + non_equivalent_optional,
+        mandatory_list=all_parents,
         environment=environment,
         g=g,
         depth=depth + 1,

trilogy/core/processing/node_generators/common.py

@@ -20,12 +20,16 @@ from trilogy.core.processing.nodes.merge_node import MergeNode
 from trilogy.utility import unique
 
 
-def resolve_function_parent_concepts(concept: Concept) -> List[Concept]:
+def resolve_function_parent_concepts(
+    concept: Concept, environment: Environment
+) -> List[Concept]:
     if not isinstance(concept.lineage, (Function, AggregateWrapper)):
         raise ValueError(f"Concept {concept} lineage is not function or aggregate")
     if concept.derivation == PurposeLineage.AGGREGATE:
         if not concept.grain.abstract:
-            base = concept.lineage.concept_arguments + concept.grain.components_copy
+            base = concept.lineage.concept_arguments + [
+                environment.concepts[c] for c in concept.grain.components
+            ]
             # if the base concept being aggregated is a property with a key
             # keep the key as a parent
         else:
@@ -56,6 +60,7 @@ def resolve_condition_parent_concepts(
 
 def resolve_filter_parent_concepts(
     concept: Concept,
+    environment: Environment,
 ) -> Tuple[Concept, List[Concept], List[Tuple[Concept, ...]]]:
     if not isinstance(concept.lineage, FilterItem):
         raise ValueError(
@@ -70,7 +75,7 @@ def resolve_filter_parent_concepts(
     base_rows += condition_rows
     base_existence += condition_existence
     if direct_parent.grain:
-        base_rows += direct_parent.grain.components_copy
+        base_rows += [environment.concepts[c] for c in direct_parent.grain.components]
     if (
         isinstance(direct_parent, Concept)
         and direct_parent.purpose == Purpose.PROPERTY
@@ -130,7 +135,6 @@ def gen_property_enrichment_node(
         ),
         output_concepts=base_node.output_concepts + extra_properties,
         environment=environment,
-        g=g,
         parents=[
             base_node,
         ]
@@ -209,7 +213,6 @@ def gen_enrichment_node(
         input_concepts=unique(join_keys + extra_required + non_hidden, "address"),
         output_concepts=unique(join_keys + extra_required + non_hidden, "address"),
         environment=environment,
-        g=g,
         parents=[enrich_node, base_node],
         force_group=False,
         preexisting_conditions=conditions.conditional if conditions else None,

trilogy/core/processing/node_generators/filter_node.py

@@ -28,7 +28,7 @@ def gen_filter_node(
     conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
     immediate_parent, parent_row_concepts, parent_existence_concepts = (
-        resolve_filter_parent_concepts(concept)
+        resolve_filter_parent_concepts(concept, environment)
     )
     if not isinstance(concept.lineage, FilterItem):
         raise SyntaxError('Filter node must have a lineage of type "FilterItem"')
@@ -117,7 +117,6 @@ def gen_filter_node(
                 input_concepts=row_parent.output_concepts,
                 output_concepts=[concept] + row_parent.output_concepts,
                 environment=row_parent.environment,
-                g=row_parent.g,
                 parents=[row_parent],
                 depth=row_parent.depth,
                 partial_concepts=row_parent.partial_concepts,
@@ -137,8 +136,8 @@ def gen_filter_node(
         parent.add_existence_concepts(flattened_existence, False).set_output_concepts(
             expected_output, False
         )
-        parent.grain = Grain(
-            components=(
+        parent.grain = Grain.from_concepts(
+            (
                 list(immediate_parent.keys)
                 if immediate_parent.keys
                 else [immediate_parent]
@@ -161,10 +160,9 @@ def gen_filter_node(
             ),
             output_concepts=[concept, immediate_parent] + parent_row_concepts,
             environment=environment,
-            g=g,
             parents=core_parents,
-            grain=Grain(
-                components=[immediate_parent] + parent_row_concepts,
+            grain=Grain.from_concepts(
+                [immediate_parent] + parent_row_concepts,
             ),
             preexisting_conditions=conditions.conditional if conditions else None,
         )
@@ -202,7 +200,6 @@ def gen_filter_node(
         ]
         + local_optional,
         environment=environment,
-        g=g,
         parents=[
             # this node fetches only what we need to filter
             filter_node,

trilogy/core/processing/node_generators/group_node.py

@@ -34,7 +34,7 @@ def gen_group_node(
     # aggregates MUST always group to the proper grain
     # except when the
     parent_concepts: List[Concept] = unique(
-        resolve_function_parent_concepts(concept), "address"
+        resolve_function_parent_concepts(concept, environment=environment), "address"
     )
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} parent concepts are {[x.address for x in parent_concepts]} from group grain {concept.grain}"
@@ -43,18 +43,28 @@ def gen_group_node(
     # if the aggregation has a grain, we need to ensure these are the ONLY optional in the output of the select
     output_concepts = [concept]
 
-    if concept.grain and len(concept.grain.components_copy) > 0:
-        grain_components = (
-            concept.grain.components_copy if not concept.grain.abstract else []
-        )
+    if (
+        concept.grain
+        and len(concept.grain.components) > 0
+        and not concept.grain.abstract
+    ):
+        grain_components = [environment.concepts[c] for c in concept.grain.components]
         parent_concepts += grain_components
         output_concepts += grain_components
         for possible_agg in local_optional:
+
             if not isinstance(possible_agg.lineage, (AggregateWrapper, Function)):
                 continue
+            logger.info(possible_agg)
+            if possible_agg.grain and possible_agg.grain != concept.grain:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} mismatched equivalent group by with grain {possible_agg.grain} for {concept.address}"
+                )
+
             if possible_agg.grain and possible_agg.grain == concept.grain:
                 agg_parents: List[Concept] = resolve_function_parent_concepts(
-                    possible_agg
+                    possible_agg,
+                    environment=environment,
                 )
                 if set([x.address for x in agg_parents]).issubset(
                     set([x.address for x in parent_concepts])
@@ -63,13 +73,19 @@ def gen_group_node(
                     logger.info(
                         f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
                     )
-                elif Grain(components=agg_parents) == Grain(components=parent_concepts):
+                elif Grain.from_concepts(agg_parents) == Grain.from_concepts(
+                    parent_concepts
+                ):
                     extra = [x for x in agg_parents if x.address not in parent_concepts]
                     parent_concepts += extra
                     output_concepts.append(possible_agg)
                     logger.info(
                         f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
                     )
+                else:
+                    logger.info(
+                        f"{padding(depth)}{LOGGER_PREFIX} mismatched grain {Grain.from_concepts(agg_parents)} vs {Grain.from_concepts(parent_concepts)}"
+                    )
     if parent_concepts:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} fetching group node parents {LooseConceptList(concepts=parent_concepts)}"
@@ -94,13 +110,12 @@ def gen_group_node(
 
     # the keys we group by
     # are what we can use for enrichment
-    group_key_parents = concept.grain.components_copy
+    group_key_parents = [environment.concepts[c] for c in concept.grain.components]
 
     group_node = GroupNode(
         output_concepts=output_concepts,
         input_concepts=parent_concepts,
         environment=environment,
-        g=g,
         parents=parents,
         depth=depth,
         preexisting_conditions=conditions.conditional if conditions else None,

trilogy/core/processing/node_generators/group_to_node.py

@@ -45,7 +45,6 @@ def gen_group_to_node(
         output_concepts=parent_concepts + [concept],
         input_concepts=parent_concepts,
         environment=environment,
-        g=g,
         parents=parents,
         depth=depth,
     )
@@ -76,7 +75,6 @@ def gen_group_to_node(
         + [x for x in parent_concepts if x.address != concept.address],
         output_concepts=[concept] + local_optional,
         environment=environment,
-        g=g,
         parents=[
             # this node gets the group
             group_node,

trilogy/core/processing/node_generators/multiselect_node.py

@@ -8,12 +8,13 @@ from trilogy.core.models import (
     Concept,
     Conditional,
     Environment,
+    Grain,
     MultiSelectStatement,
     WhereClause,
 )
 from trilogy.core.processing.node_generators.common import resolve_join_order
 from trilogy.core.processing.nodes import History, MergeNode, NodeJoin
-from trilogy.core.processing.nodes.base_node import StrategyNode, concept_list_to_grain
+from trilogy.core.processing.nodes.base_node import StrategyNode
 from trilogy.core.processing.utility import concept_to_relevant_joins, padding
 
 LOGGER_PREFIX = "[GEN_MULTISELECT_NODE]"
@@ -108,7 +109,6 @@ def gen_multiselect_node(
         input_concepts=[x for y in base_parents for x in y.output_concepts],
         output_concepts=[x for y in base_parents for x in y.output_concepts],
         environment=environment,
-        g=g,
         depth=depth,
         parents=base_parents,
         node_joins=node_joins,
@@ -138,8 +138,8 @@ def gen_multiselect_node(
 
     # assume grain to be output of select
     # but don't include anything aggregate at this point
-    node.resolution_cache.grain = concept_list_to_grain(
-        node.output_concepts, parent_sources=node.resolution_cache.datasources
+    node.resolution_cache.grain = Grain.from_concepts(
+        node.output_concepts,
     )
     possible_joins = concept_to_relevant_joins(additional_relevant)
     if not local_optional:
@@ -178,7 +178,6 @@ def gen_multiselect_node(
         input_concepts=enrich_node.output_concepts + node.output_concepts,
         output_concepts=node.output_concepts + local_optional,
         environment=environment,
-        g=g,
         depth=depth,
         parents=[
             # this node gets the multiselect

trilogy/core/processing/node_generators/node_merge_node.py

@@ -327,13 +327,18 @@ def subgraphs_to_merge_node(
         for y in x.output_concepts:
             input_c.append(y)
     if len(parents) == 1 and enable_early_exit:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} only one parent node, exiting early w/ {[c.address for c in parents[0].output_concepts]}"
+        )
         return parents[0]
-
+    base_output = [x for x in all_concepts]
+    # for x in base_output:
+    #     if x not in input_c:
+    #         input_c.append(x)
     return MergeNode(
         input_concepts=unique(input_c, "address"),
-        output_concepts=[x for x in all_concepts],
+        output_concepts=base_output,
         environment=environment,
-        g=g,
         parents=parents,
         depth=depth,
         # conditions=conditions,
@@ -369,6 +374,12 @@ def gen_merge_node(
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} Was able to resolve graph through weak component resolution - final graph {log_graph}"
             )
+            for flat in log_graph:
+                if set(flat) == set([x.address for x in all_concepts]):
+                    logger.info(
+                        f"{padding(depth)}{LOGGER_PREFIX} expanded concept resolution was identical to search resolution; breaking to avoid recursion error."
+                    )
+                    return None
             return subgraphs_to_merge_node(
                 weak_resolve,
                 depth=depth,

trilogy/core/processing/node_generators/rowset_node.py

@@ -5,6 +5,7 @@ from trilogy.core.enums import PurposeLineage
 from trilogy.core.models import (
     Concept,
     Environment,
+    Grain,
     MultiSelectStatement,
     RowsetDerivationStatement,
     RowsetItem,
@@ -12,7 +13,6 @@ from trilogy.core.models import (
     WhereClause,
 )
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
-from trilogy.core.processing.nodes.base_node import concept_list_to_grain
 from trilogy.core.processing.utility import concept_to_relevant_joins, padding
 
 LOGGER_PREFIX = "[GEN_ROWSET_NODE]"
@@ -74,7 +74,7 @@ def gen_rowset_node(
     assert node.resolution_cache
     # assume grain to be output of select
     # but don't include anything hidden(the non-rowset concepts)
-    node.grain = concept_list_to_grain(
+    node.grain = Grain.from_concepts(
         [
             x
             for x in node.output_concepts
@@ -83,7 +83,6 @@ def gen_rowset_node(
                 y for y in node.hidden_concepts if y.derivation != PurposeLineage.ROWSET
             ]
         ],
-        parent_sources=node.resolution_cache.datasources,
     )
 
     node.rebuild_cache()
@@ -92,7 +91,7 @@ def gen_rowset_node(
         x.address in node.output_concepts for x in local_optional
     ):
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node as all optional found or no optional; exiting early."
+            f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node as all optional {[x.address for x in local_optional]} found or no optional; exiting early."
         )
         return node
     possible_joins = concept_to_relevant_joins(
@@ -140,7 +139,6 @@ def gen_rowset_node(
         input_concepts=non_hidden + non_hidden_enrich,
         output_concepts=non_hidden + local_optional,
         environment=environment,
-        g=g,
         depth=depth,
         parents=[
             node,

trilogy/core/processing/node_generators/select_helpers/datasource_injection.py

@@ -0,0 +1,203 @@
+from collections import defaultdict
+from datetime import date, datetime, timedelta
+from typing import List, Tuple, TypeVar
+
+from trilogy.core.enums import ComparisonOperator
+from trilogy.core.models import (
+    Comparison,
+    Concept,
+    Conditional,
+    Datasource,
+    DataType,
+    Function,
+    FunctionType,
+    Parenthetical,
+)
+
+# Define a generic type that ensures start and end are the same type
+T = TypeVar("T", int, date, datetime)
+
+
+def reduce_expression(
+    var: Concept, group_tuple: list[tuple[ComparisonOperator, T]]
+) -> bool:
+    # Track ranges
+    lower_check: T
+    upper_check: T
+
+    # if var.datatype in (DataType.FLOAT,):
+    #     lower_check = float("-inf")  # type: ignore
+    #     upper_check = float("inf")  # type: ignore
+    if var.datatype == DataType.INTEGER:
+        lower_check = float("-inf")  # type: ignore
+        upper_check = float("inf")  # type: ignore
+    elif var.datatype == DataType.DATE:
+        lower_check = date.min  # type: ignore
+        upper_check = date.max  # type: ignore
+
+    elif var.datatype == DataType.DATETIME:
+        lower_check = datetime.min  # type: ignore
+        upper_check = datetime.max  # type: ignore
+    else:
+        raise ValueError(f"Invalid datatype: {var.datatype}")
+
+    ranges: list[Tuple[T, T]] = []
+    for op, value in group_tuple:
+        increment: int | timedelta
+        if isinstance(value, date):
+            increment = timedelta(days=1)
+        elif isinstance(value, datetime):
+            increment = timedelta(seconds=1)
+        elif isinstance(value, int):
+            increment = 1
+        # elif isinstance(value, float):
+        #     value = Decimal(value)
+        #     increment = Decimal(0.0000000001)
+
+        if op == ">":
+            ranges.append(
+                (
+                    value + increment,
+                    upper_check,
+                )
+            )
+        elif op == ">=":
+            ranges.append(
+                (
+                    value,
+                    upper_check,
+                )
+            )
+        elif op == "<":
+            ranges.append(
+                (
+                    lower_check,
+                    value - increment,
+                )
+            )
+        elif op == "<=":
+            ranges.append(
+                (
+                    lower_check,
+                    value,
+                )
+            )
+        elif op == "=":
+            ranges.append(
+                (
+                    value,
+                    value,
+                )
+            )
+        else:
+            raise ValueError(f"Invalid operator: {op}")
+    return is_fully_covered(lower_check, upper_check, ranges, increment)
+
+
+def simplify_conditions(
+    conditions: list[Comparison | Conditional | Parenthetical],
+) -> bool:
+    # Group conditions by variable
+    grouped: dict[Concept, list[tuple[ComparisonOperator, datetime | int | date]]] = (
+        defaultdict(list)
+    )
+    for condition in conditions:
+        if not isinstance(condition, Comparison):
+            return False
+        if not isinstance(
+            condition.left, (int, date, datetime, Function)
+        ) and not isinstance(condition.right, (int, date, datetime, Function)):
+            return False
+        if not isinstance(condition.left, Concept) and not isinstance(
+            condition.right, Concept
+        ):
+            return False
+        vars = [condition.left, condition.right]
+        concept = [x for x in vars if isinstance(x, Concept)][0]
+        comparison = [x for x in vars if not isinstance(x, Concept)][0]
+        if isinstance(comparison, Function):
+            if not comparison.operator == FunctionType.CONSTANT:
+                return False
+            first_arg = comparison.arguments[0]
+            if not isinstance(first_arg, (int, date, datetime)):
+                return False
+            comparison = first_arg
+        if not isinstance(comparison, (int, date, datetime)):
+            return False
+
+        var = concept
+        op = condition.operator
+        grouped[var].append((op, comparison))
+
+    simplified = []
+    for var, group_tuple in grouped.items():
+        simplified.append(reduce_expression(var, group_tuple))  # type: ignore
+
+    # Final simplification
+    return True if all(isinstance(s, bool) and s for s in simplified) else False
+
+
+def is_fully_covered(
+    start: T,
+    end: T,
+    ranges: List[Tuple[T, T]],
+    increment: int | timedelta,
+):
+    """
+    Check if the list of range pairs fully covers the set [start, end].
+
+    Parameters:
+    - start (int or float): The starting value of the set to cover.
+    - end (int or float): The ending value of the set to cover.
+    - ranges (list of tuples): List of range pairs [(start1, end1), (start2, end2), ...].
+
+    Returns:
+    - bool: True if the ranges fully cover [start, end], False otherwise.
+    """
+    # Sort ranges by their start values (and by end values for ties)
+    ranges.sort()
+
+    # Check for gaps
+    current_end = start
+    print(ranges)
+    for r_start, r_end in ranges:
+        print(r_start, r_end)
+        # If there's a gap between the current range and the previous coverage
+        print(r_start - current_end)
+        if (r_start - current_end) > increment:  # type: ignore
+            print("gap")
+            return False
+        print("okay")
+        # Extend the current coverage
+        current_end = max(current_end, r_end)
+
+    # If the loop ends and we haven't reached the end, return False
+    print(current_end, end)
+    print(current_end >= end)
+    return current_end >= end
+
+
+def get_union_sources(datasources: list[Datasource], concepts: list[Concept]):
+    candidates: list[Datasource] = []
+    for x in datasources:
+        if all([c.address in x.output_concepts for c in concepts]):
+            if (
+                any([c.address in x.partial_concepts for c in concepts])
+                and x.non_partial_for
+            ):
+                candidates.append(x)
+
+    assocs: dict[str, list[Datasource]] = defaultdict(list[Datasource])
+    for x in candidates:
+        if not x.non_partial_for:
+            continue
+        if not len(x.non_partial_for.concept_arguments) == 1:
+            continue
+        merge_key = x.non_partial_for.concept_arguments[0]
+        assocs[merge_key.address].append(x)
+    final: list[list[Datasource]] = []
+    for _, dses in assocs.items():
+        conditions = [c.non_partial_for.conditional for c in dses if c.non_partial_for]
+        if simplify_conditions(conditions):
+            final.append(dses)
+    return final