pytrilogy 0.0.3.105__tar.gz → 0.0.3.107__tar.gz

{pytrilogy-0.0.3.105/pytrilogy.egg-info → pytrilogy-0.0.3.107}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.105
+Version: 0.0.3.107
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.105 → pytrilogy-0.0.3.107/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.105
+Version: 0.0.3.107
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.105 → pytrilogy-0.0.3.107}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.105"
+__version__ = "0.0.3.107"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.105 → pytrilogy-0.0.3.107}/trilogy/core/processing/concept_strategies_v3.py

@@ -350,9 +350,6 @@ def check_for_early_exit(
 def generate_loop_completion(context: LoopContext, virtual: set[str]) -> StrategyNode:
     condition_required = True
     non_virtual = [c for c in context.completion_mandatory if c.address not in virtual]
-    non_virtual_output = [
-        c for c in context.original_mandatory if c.address not in virtual
-    ]
     non_virtual_different = len(context.completion_mandatory) != len(
         context.original_mandatory
     )
@@ -380,11 +377,12 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
         logger.info(
             f"Condition {context.conditions} not required, parents included filtering! {parent_map}"
         )
+
     if len(context.stack) == 1:
         output: StrategyNode = context.stack[0]
         if non_virtual_different:
             logger.info(
-                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Found different non-virtual output concepts ({non_virtual_difference_values}), removing condition injected values by setting outputs to {[x.address for x in output.output_concepts if x.address in non_virtual_output]}"
+                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Found added non-virtual output concepts ({non_virtual_difference_values})"
             )
             # output.set_output_concepts(
             #     [
@@ -398,13 +396,6 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
             # )
             # output.set_output_concepts(context.original_mandatory)
 
-            # if isinstance(output, MergeNode):
-            #     output.force_group = True
-            #     output.rebuild_cache()
-
-        logger.info(
-            f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)}"
-        )
     else:
         logger.info(
             f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} wrapping multiple parent nodes {[type(x) for x in context.stack]} in merge node"
@@ -441,10 +432,18 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
             f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Conditions {context.conditions} were injected, checking if we need a group to restore grain"
         )
         return group_if_required_v2(
-            output, context.original_mandatory, context.environment
+            output,
+            context.original_mandatory,
+            context.environment,
+            non_virtual_difference_values,
         )
 
-    return group_if_required_v2(output, context.original_mandatory, context.environment)
+    return group_if_required_v2(
+        output,
+        context.original_mandatory,
+        context.environment,
+        non_virtual_difference_values,
+    )
 
 
 def _search_concepts(

{pytrilogy-0.0.3.105 → pytrilogy-0.0.3.107}/trilogy/core/processing/discovery_utility.py

@@ -180,8 +180,12 @@ def check_if_group_required(
 
 
 def group_if_required_v2(
-    root: StrategyNode, final: List[BuildConcept], environment: BuildEnvironment
+    root: StrategyNode,
+    final: List[BuildConcept],
+    environment: BuildEnvironment,
+    where_injected: set[str] | None = None,
 ):
+    where_injected = where_injected or set()
     required = check_if_group_required(
         downstream_concepts=final, parents=[root.resolve()], environment=environment
     )
@@ -197,8 +201,23 @@ def group_if_required_v2(
             root.rebuild_cache()
             return root
         elif isinstance(root, GroupNode):
-            # root.set_output_concepts(final, rebuild=False)
-            # root.rebuild_cache()
+
+            if set(x.address for x in final) != set(
+                x.address for x in root.output_concepts
+            ):
+                allowed_outputs = [
+                    x
+                    for x in root.output_concepts
+                    if not (
+                        x.address in where_injected
+                        and x.address not in (root.required_outputs or set())
+                    )
+                ]
+
+                logger.info(
+                    f"Adjusting group node outputs to remove injected concepts {where_injected}: remaining {allowed_outputs}"
+                )
+                root.set_output_concepts(allowed_outputs)
             return root
         return GroupNode(
             output_concepts=targets,

{pytrilogy-0.0.3.105 → pytrilogy-0.0.3.107}/trilogy/core/processing/node_generators/group_node.py

@@ -176,6 +176,7 @@ def gen_group_node(
         parents=parents,
         depth=depth,
         preexisting_conditions=conditions.conditional if conditions else None,
+        required_outputs=parent_concepts,
     )
 
     # early exit if no optional

{pytrilogy-0.0.3.105 → pytrilogy-0.0.3.107}/trilogy/core/processing/node_generators/select_merge_node.py

@@ -224,6 +224,72 @@ def create_pruned_concept_graph(
     return g
 
 
+# def deduplicate_nodes(subgraph: nx.DiGraph, nodes: list[str], partial_map: dict[str, list[str]], depth: int) -> list[str]:
+#     """
+#     Remove duplicate datasource nodes that are connected to the same concepts
+#     and have the same partial state, keeping the one with the most unique concepts.
+
+#     Args:
+#         subgraph: NetworkX DiGraph containing the nodes and edges
+#         nodes: List of node names to deduplicate
+#         partial_map: Map of datasource to partial nodes
+
+#     Returns:
+#         List of deduplicated node names
+#     """
+#     # Filter for datasource nodes only
+#     ds_nodes = [node for node in nodes if node.startswith("ds~")]
+#     non_ds_nodes = [node for node in nodes if not node.startswith("ds~")]
+
+#     if len(ds_nodes) <= 1:
+#         return nodes  # No deduplication needed
+
+#     # Build a map of each datasource to its connected concepts and partial state
+#     ds_info = {}
+
+#     for ds_node in ds_nodes:
+#         # Get connected concept nodes (nodes starting with "c~")
+#         connected_concepts = set()
+#         for neighbor in subgraph.neighbors(ds_node):
+#             if neighbor.startswith("c~"):
+#                 connected_concepts.add(neighbor)
+
+#         # Get partial state for this datasource
+#         partial_state = tuple(sorted(partial_map.get(ds_node, [])))
+
+#         ds_info[ds_node] = {
+#             'concepts': connected_concepts,
+#             'partial_state': partial_state
+#         }
+
+#     # Find datasources to remove (those that are subsets of others)
+#     nodes_to_remove = set()
+#     logger.info('LOOK HERE')
+#     logger.info(ds_info)
+#     for ds_a, info_a in ds_info.items():
+#         for ds_b, info_b in ds_info.items():
+#             if ds_a != ds_b and ds_a not in nodes_to_remove:
+#                 # Check if ds_a is a subset of ds_b (same partial state and concepts are subset)
+#                 if (info_a['partial_state'] == info_b['partial_state'] and
+#                     info_a['concepts'].issubset(info_b['concepts']) and
+#                     len(info_a['concepts']) < len(info_b['concepts'])):
+#                     # ds_a connects to fewer concepts than ds_b, so remove ds_a
+#                     nodes_to_remove.add(ds_a)
+#                 elif (info_a['partial_state'] == info_b['partial_state'] and
+#                       info_a['concepts'] == info_b['concepts']):
+#                     # Exact same concepts and partial state - keep one arbitrarily
+#                     # (keep the lexicographically smaller one for consistency)
+#                     if ds_a > ds_b:
+#                         nodes_to_remove.add(ds_a)
+
+#     # Keep datasource nodes that weren't marked for removal
+#     logger.info(f"{padding(depth)}{LOGGER_PREFIX} Removing duplicate datasource nodes: {nodes_to_remove}")
+#     deduplicated_ds_nodes = [ds for ds in ds_nodes if ds not in nodes_to_remove]
+
+#     # Return deduplicated datasource nodes plus all non-datasource nodes
+#     return deduplicated_ds_nodes + non_ds_nodes
+
+
 def resolve_subgraphs(
     g: ReferenceGraph,
     relevant: list[BuildConcept],

{pytrilogy-0.0.3.105 → pytrilogy-0.0.3.107}/trilogy/core/processing/node_generators/window_node.py

@@ -27,7 +27,7 @@ WINDOW_TYPES = (BuildWindowItem,)
 
 
 def resolve_window_parent_concepts(
-    concept: BuildConcept, environment: BuildEnvironment
+    concept: BuildConcept, environment: BuildEnvironment, depth: int
 ) -> tuple[BuildConcept, List[BuildConcept]]:
     if not isinstance(concept.lineage, WINDOW_TYPES):
         raise ValueError
@@ -39,7 +39,9 @@ def resolve_window_parent_concepts(
             base += item.concept_arguments
     if concept.grain:
         for gitem in concept.grain.components:
-            logger.info(f"{LOGGER_PREFIX} appending grain item {gitem} to base")
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} appending grain item {gitem} to base"
+            )
             base.append(environment.concepts[gitem])
     return concept.lineage.content, unique(base, "address")
 
@@ -54,7 +56,7 @@ def gen_window_node(
     history: History,
     conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
-    base, parent_concepts = resolve_window_parent_concepts(concept, environment)
+    base, parent_concepts = resolve_window_parent_concepts(concept, environment, depth)
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} generating window node for {concept} with parents {[x.address for x in parent_concepts]} and optional {local_optional}"
     )
@@ -62,7 +64,7 @@ def gen_window_node(
         x
         for x in local_optional
         if isinstance(x.lineage, WINDOW_TYPES)
-        and resolve_window_parent_concepts(x, environment)[1] == parent_concepts
+        and resolve_window_parent_concepts(x, environment, depth)[1] == parent_concepts
     ]
 
     targets = [base]
@@ -79,7 +81,7 @@ def gen_window_node(
     if equivalent_optional:
         for x in equivalent_optional:
             assert isinstance(x.lineage, WINDOW_TYPES)
-            base, parents = resolve_window_parent_concepts(x, environment)
+            base, parents = resolve_window_parent_concepts(x, environment, depth)
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} found equivalent optional {x} with parents {parents}"
             )

{pytrilogy-0.0.3.105 → pytrilogy-0.0.3.107}/trilogy/core/processing/nodes/group_node.py

@@ -49,6 +49,7 @@ class GroupNode(StrategyNode):
         existence_concepts: List[BuildConcept] | None = None,
         hidden_concepts: set[str] | None = None,
         ordering: BuildOrderBy | None = None,
+        required_outputs: List[BuildConcept] | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -66,6 +67,9 @@ class GroupNode(StrategyNode):
             hidden_concepts=hidden_concepts,
             ordering=ordering,
         )
+        # the set of concepts required to preserve grain
+        # set by group by node generation with aggregates
+        self.required_outputs = required_outputs
 
     @classmethod
     def check_if_required(
@@ -184,4 +188,7 @@ class GroupNode(StrategyNode):
             existence_concepts=list(self.existence_concepts),
             hidden_concepts=set(self.hidden_concepts),
             ordering=self.ordering,
+            required_outputs=(
+                list(self.required_outputs) if self.required_outputs else None
+            ),
         )

{pytrilogy-0.0.3.105 → pytrilogy-0.0.3.107}/trilogy/core/processing/utility.py

@@ -90,13 +90,86 @@ class GroupRequiredResponse:
     required: bool
 
 
+def find_all_connecting_concepts(g: nx.Graph, ds1: str, ds2: str) -> set[str]:
+    """Find all concepts that connect two datasources"""
+    concepts1 = set(g.neighbors(ds1))
+    concepts2 = set(g.neighbors(ds2))
+    return concepts1 & concepts2
+
+
+def get_connection_keys(
+    all_connections: dict[tuple[str, str], set[str]], left: str, right: str
+) -> set[str]:
+    """Get all concepts that connect two datasources"""
+    lookup = sorted([left, right])
+    key: tuple[str, str] = (lookup[0], lookup[1])
+    return all_connections.get(key, set())
+
+
+def get_join_type(
+    left: str,
+    right: str,
+    partials: dict[str, list[str]],
+    nullables: dict[str, list[str]],
+    all_connecting_keys: set[str],
+) -> JoinType:
+    left_is_partial = any(key in partials.get(left, []) for key in all_connecting_keys)
+    left_is_nullable = any(
+        key in nullables.get(left, []) for key in all_connecting_keys
+    )
+    right_is_partial = any(
+        key in partials.get(right, []) for key in all_connecting_keys
+    )
+    right_is_nullable = any(
+        key in nullables.get(right, []) for key in all_connecting_keys
+    )
+
+    if left_is_nullable and right_is_nullable:
+        join_type = JoinType.FULL
+    elif left_is_partial and right_is_partial:
+        join_type = JoinType.FULL
+    elif left_is_partial:
+        join_type = JoinType.FULL
+    elif right_is_nullable:
+        join_type = JoinType.RIGHT_OUTER
+    elif right_is_partial or left_is_nullable:
+        join_type = JoinType.LEFT_OUTER
+    # we can't inner join if the left was an outer join
+    else:
+        join_type = JoinType.INNER
+    return join_type
+
+
+def reduce_join_types(join_types: Set[JoinType]) -> JoinType:
+    final_join_type = JoinType.INNER
+    if any([x == JoinType.FULL for x in join_types]):
+        final_join_type = JoinType.FULL
+    elif any([x == JoinType.LEFT_OUTER for x in join_types]):
+        final_join_type = JoinType.LEFT_OUTER
+    elif any([x == JoinType.RIGHT_OUTER for x in join_types]):
+        final_join_type = JoinType.RIGHT_OUTER
+
+    return final_join_type
+
+
 def resolve_join_order_v2(
     g: nx.Graph, partials: dict[str, list[str]], nullables: dict[str, list[str]]
 ) -> list[JoinOrderOutput]:
     datasources = [x for x in g.nodes if x.startswith("ds~")]
     concepts = [x for x in g.nodes if x.startswith("c~")]
 
+    # Pre-compute all possible connections between datasources
+    all_connections: dict[tuple[str, str], set[str]] = {}
+    for i, ds1 in enumerate(datasources):
+        for ds2 in datasources[i + 1 :]:
+            connecting_concepts = find_all_connecting_concepts(g, ds1, ds2)
+            if connecting_concepts:
+                key = tuple(sorted([ds1, ds2]))
+                all_connections[key] = connecting_concepts
+
     output: list[JoinOrderOutput] = []
+
+    # create our map of pivots, or common join concepts
     pivot_map = {
         concept: [x for x in g.neighbors(concept) if x in datasources]
         for concept in concepts
@@ -108,8 +181,9 @@ def resolve_join_order_v2(
         )
     )
     solo = [x for x in pivot_map if len(pivot_map[x]) == 1]
-    eligible_left = set()
+    eligible_left: set[str] = set()
 
+    # while we have pivots, keep joining them in
     while pivots:
         next_pivots = [
             x for x in pivots if any(y in eligible_left for y in pivot_map[x])
@@ -120,7 +194,7 @@ def resolve_join_order_v2(
         else:
             root = pivots.pop(0)
 
-        # sort so less partials is last and eligible lefts are
+        # sort so less partials is last and eligible lefts are first
         def score_key(x: str) -> tuple[int, int, str]:
             base = 1
             # if it's left, higher weight
@@ -133,79 +207,56 @@ def resolve_join_order_v2(
                 base -= 1
             return (base, len(x), x)
 
-        # get remainig un-joined datasets
+        # get remaining un-joined datasets
         to_join = sorted(
             [x for x in pivot_map[root] if x not in eligible_left], key=score_key
         )
         while to_join:
             # need to sort this to ensure we join on the best match
-            base = sorted(
-                [x for x in pivot_map[root] if x in eligible_left], key=score_key
-            )
+            # but check ALL left in case there are non-pivt keys to join on
+            base = sorted([x for x in eligible_left], key=score_key)
             if not base:
                 new = to_join.pop()
                 eligible_left.add(new)
                 base = [new]
             right = to_join.pop()
             # we already joined it
-            # this could happen if the same pivot is shared with multiple Dses
+            # this could happen if the same pivot is shared with multiple DSes
             if right in eligible_left:
                 continue
+
             joinkeys: dict[str, set[str]] = {}
             # sorting puts the best candidate last for pop
             # so iterate over the reversed list
             join_types = set()
+
             for left_candidate in reversed(base):
-                common = nx.common_neighbors(g, left_candidate, right)
+                # Get all concepts that connect these two datasources
+                all_connecting_keys = get_connection_keys(
+                    all_connections, left_candidate, right
+                )
 
-                if not common:
+                if not all_connecting_keys:
                     continue
+
+                # Check if we already have this exact set of keys
                 exists = False
                 for _, v in joinkeys.items():
-                    if v == common:
+                    if v == all_connecting_keys:
                         exists = True
                 if exists:
                     continue
-                left_is_partial = any(
-                    key in partials.get(left_candidate, []) for key in common
-                )
-                left_is_nullable = any(
-                    key in nullables.get(left_candidate, []) for key in common
-                )
-                right_is_partial = any(key in partials.get(right, []) for key in common)
-                # we don't care if left is nullable for join type (just keys), but if we did
-                # left_is_nullable = any(
-                #     key in nullables.get(left_candidate, []) for key in common
-                # )
-                right_is_nullable = any(
-                    key in nullables.get(right, []) for key in common
-                )
-                if left_is_nullable and right_is_nullable:
-                    join_type = JoinType.FULL
-                elif left_is_partial and right_is_partial:
-                    join_type = JoinType.FULL
-                elif left_is_partial:
-                    join_type = JoinType.FULL
-                elif right_is_nullable:
-                    join_type = JoinType.RIGHT_OUTER
-                elif right_is_partial or left_is_nullable:
-                    join_type = JoinType.LEFT_OUTER
-                # we can't inner join if the left was an outer join
-                else:
-                    join_type = JoinType.INNER
 
+                join_type = get_join_type(
+                    left_candidate, right, partials, nullables, all_connecting_keys
+                )
                 join_types.add(join_type)
-                joinkeys[left_candidate] = common
-            final_join_type = JoinType.INNER
-            if any([x == JoinType.FULL for x in join_types]):
-                final_join_type = JoinType.FULL
-            elif any([x == JoinType.LEFT_OUTER for x in join_types]):
-                final_join_type = JoinType.LEFT_OUTER
-            elif any([x == JoinType.RIGHT_OUTER for x in join_types]):
-                final_join_type = JoinType.RIGHT_OUTER
+                joinkeys[left_candidate] = all_connecting_keys
+
+            final_join_type = reduce_join_types(join_types)
+
             output.append(
                 JoinOrderOutput(
-                    # left=left_candidate,
                     right=right,
                     type=final_join_type,
                     keys=joinkeys,
@@ -216,7 +267,6 @@ def resolve_join_order_v2(
     for concept in solo:
         for ds in pivot_map[concept]:
             # if we already have it, skip it
-
             if ds in eligible_left:
                 continue
             # if we haven't had ANY left datasources yet
@@ -224,17 +274,39 @@ def resolve_join_order_v2(
             if not eligible_left:
                 eligible_left.add(ds)
                 continue
-            # otherwise do a full out join
-            output.append(
-                JoinOrderOutput(
-                    # pick random one to be left
-                    left=list(eligible_left)[0],
-                    right=ds,
-                    type=JoinType.FULL,
-                    keys={},
+            # otherwise do a full outer join
+            # Try to find if there are any connecting keys with existing left tables
+            best_left = None
+            best_keys: set[str] = set()
+            for existing_left in eligible_left:
+                connecting_keys = get_connection_keys(
+                    all_connections, existing_left, ds
+                )
+                if connecting_keys and len(connecting_keys) > len(best_keys):
+                    best_left = existing_left
+                    best_keys = connecting_keys
+
+            if best_left and best_keys:
+                output.append(
+                    JoinOrderOutput(
+                        left=best_left,
+                        right=ds,
+                        type=JoinType.FULL,
+                        keys={best_left: best_keys},
+                    )
+                )
+            else:
+                output.append(
+                    JoinOrderOutput(
+                        # pick random one to be left
+                        left=list(eligible_left)[0],
+                        right=ds,
+                        type=JoinType.FULL,
+                        keys={},
+                    )
                 )
-            )
             eligible_left.add(ds)
+
     # only once we have all joins
     # do we know if some inners need to be left outers
     for review_join in output:
@@ -248,6 +320,7 @@ def resolve_join_order_v2(
             ]
         ):
             review_join.type = JoinType.LEFT_OUTER
+
     return output
 
 
@@ -352,7 +425,9 @@ def resolve_instantiated_concept(
     )
 
 
-def reduce_concept_pairs(input: list[ConceptPair]) -> list[ConceptPair]:
+def reduce_concept_pairs(
+    input: list[ConceptPair], right_source: QueryDatasource | BuildDatasource
+) -> list[ConceptPair]:
     left_keys = set()
     right_keys = set()
     for pair in input:
@@ -361,7 +436,10 @@ def reduce_concept_pairs(input: list[ConceptPair]) -> list[ConceptPair]:
         if pair.right.purpose == Purpose.KEY:
             right_keys.add(pair.right.address)
     final: list[ConceptPair] = []
+    seen_right_keys = set()
     for pair in input:
+        if pair.right.address in seen_right_keys:
+            continue
         if (
             pair.left.purpose == Purpose.PROPERTY
             and pair.left.keys
@@ -374,7 +452,15 @@ def reduce_concept_pairs(input: list[ConceptPair]) -> list[ConceptPair]:
             and pair.right.keys.issubset(right_keys)
         ):
             continue
+
+        seen_right_keys.add(pair.right.address)
         final.append(pair)
+    all_keys = set([x.right.address for x in final])
+    if right_source.grain.components and right_source.grain.components.issubset(
+        all_keys
+    ):
+        return [x for x in final if x.right.address in right_source.grain.components]
+
     return final
 
 
@@ -443,7 +529,8 @@ def get_node_joins(
                     )
                     for k, v in j.keys.items()
                     for concept in v
-                ]
+                ],
+                ds_node_map[j.right],
             ),
         )
         for j in joins

{pytrilogy-0.0.3.105 → pytrilogy-0.0.3.107}/trilogy/parsing/render.py

@@ -8,7 +8,14 @@ from typing import Any
 from jinja2 import Template
 
 from trilogy.constants import DEFAULT_NAMESPACE, VIRTUAL_CONCEPT_PREFIX, MagicConstants
-from trilogy.core.enums import ConceptSource, DatePart, FunctionType, Modifier, Purpose
+from trilogy.core.enums import (
+    ConceptSource,
+    DatePart,
+    FunctionType,
+    Modifier,
+    Purpose,
+    ValidationScope,
+)
 from trilogy.core.models.author import (
     AggregateWrapper,
     AlignClause,
@@ -66,6 +73,7 @@ from trilogy.core.statements.author import (
     SelectItem,
     SelectStatement,
     TypeDeclaration,
+    ValidateStatement,
 )
 
 QUERY_TEMPLATE = Template(
@@ -445,6 +453,13 @@ class Renderer:
         final = "".join(prefixes)
         return f"{final}{self.to_string(arg.content)}"
 
+    @to_string.register
+    def _(self, arg: ValidateStatement):
+        targets = ",".join(arg.targets) if arg.targets else "*"
+        if arg.scope.value == ValidationScope.ALL:
+            return "validate all;"
+        return f"validate {arg.scope.value} {targets};"
+
     @to_string.register
     def _(self, arg: SelectStatement):
         with self.indented():