pytrilogy 0.0.3.66__tar.gz → 0.0.3.68__tar.gz

{pytrilogy-0.0.3.66/pytrilogy.egg-info → pytrilogy-0.0.3.68}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.66
+Version: 0.0.3.68
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.66
+Version: 0.0.3.68
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/tests/test_statements.py

@@ -59,7 +59,7 @@ def test_io_statement():
 
 auto x <- unnest(array);
 
-copy into csv '{target}' from select x -> test;
+copy into csv '{target}' from select x -> test order by test asc;
 """
     exec = Dialects.DUCK_DB.default_executor()
     results = exec.parse_text(text)

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.66"
+__version__ = "0.0.3.68"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/trilogy/core/exceptions.py

@@ -8,7 +8,7 @@ class UndefinedConceptException(Exception):
         self.suggestions = suggestions
 
 
-class UnresolvableQueryException(Exception):
+class FrozenEnvironmentException(Exception):
     pass
 
 
@@ -16,15 +16,15 @@ class InvalidSyntaxException(Exception):
     pass
 
 
-class NoDatasourceException(Exception):
+class UnresolvableQueryException(Exception):
     pass
 
 
-class FrozenEnvironmentException(Exception):
+class NoDatasourceException(UnresolvableQueryException):
     pass
 
 
-class AmbiguousRelationshipResolutionException(Exception):
+class AmbiguousRelationshipResolutionException(UnresolvableQueryException):
     def __init__(self, message, parents: List[set[str]]):
         super().__init__(self, message)
         self.message = message

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/trilogy/core/models/build.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from abc import ABC
+from collections import defaultdict
 from datetime import date, datetime
 from functools import cached_property, singledispatchmethod
 from typing import (
@@ -1466,6 +1467,20 @@ BuildExpr = (
 BuildConcept.model_rebuild()
 
 
+def get_canonical_pseudonyms(environment: Environment) -> dict[str, set[str]]:
+    roots: dict[str, set[str]] = defaultdict(set)
+    for k, v in environment.concepts.items():
+        roots[v.address].add(k)
+        for x in v.pseudonyms:
+            roots[v.address].add(x)
+    for k, v in environment.alias_origin_lookup.items():
+        lookup = environment.concepts[k].address
+        roots[lookup].add(v.address)
+        for x2 in v.pseudonyms:
+            roots[lookup].add(x2)
+    return roots
+
+
 class Factory:
 
     def __init__(
@@ -1479,6 +1494,7 @@ class Factory:
         self.local_concepts: dict[str, BuildConcept] = (
             {} if local_concepts is None else local_concepts
         )
+        self.pseudonym_map = get_canonical_pseudonyms(environment)
 
     def instantiate_concept(
         self,
@@ -1656,6 +1672,17 @@ class Factory:
         )
         is_aggregate = Concept.calculate_is_aggregate(build_lineage)
 
+        # if this is a pseudonym, we need to look up the base address
+        if base.address in self.environment.alias_origin_lookup:
+            lookup_address = self.environment.concepts[base.address].address
+            # map only to the canonical concept, not to other merged concepts
+            base_pseudonyms = {lookup_address}
+        else:
+            base_pseudonyms = {
+                x
+                for x in self.pseudonym_map.get(base.address, set())
+                if x != base.address
+            }
         rval = BuildConcept.model_construct(
             name=base.name,
             datatype=base.datatype,
@@ -1666,7 +1693,7 @@ class Factory:
             namespace=base.namespace,
             keys=base.keys,
             modifiers=base.modifiers,
-            pseudonyms=base.pseudonyms,
+            pseudonyms=base_pseudonyms,
             ## instantiated values
             derivation=derivation,
             granularity=granularity,
@@ -1688,14 +1715,22 @@ class Factory:
 
     @build.register
     def _(self, base: ColumnAssignment) -> BuildColumnAssignment:
-        fetched = self.environment.concepts[base.concept.address]
+        address = base.concept.address
+        fetched = (
+            self.build(
+                self.environment.alias_origin_lookup[address].with_grain(self.grain)
+            )
+            if address in self.environment.alias_origin_lookup
+            else self.build(self.environment.concepts[address].with_grain(self.grain))
+        )
+
         return BuildColumnAssignment.model_construct(
             alias=(
                 self.build(base.alias)
                 if isinstance(base.alias, Function)
                 else base.alias
             ),
-            concept=self.build(fetched.with_grain(self.grain)),
+            concept=fetched,
             modifiers=base.modifiers,
         )
 

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/trilogy/core/models/environment.py

@@ -362,9 +362,6 @@ class Environment(BaseModel):
                         and x.concept.address != deriv_lookup
                     ]
                     assert len(datasource.columns) < clen
-                    for x in datasource.columns:
-                        logger.info(x)
-
             return None
 
         if existing and self.config.allow_duplicate_declaration:
@@ -607,15 +604,15 @@ class Environment(BaseModel):
                 )
                 persisted = f"{PERSISTED_CONCEPT_PREFIX}_" + new_persisted_concept.name
                 # override the current concept source to reflect that it's now coming from a datasource
+                base_pseudonyms = new_persisted_concept.pseudonyms or set()
+                original_pseudonyms = {*base_pseudonyms, new_persisted_concept.address}
                 if (
                     new_persisted_concept.metadata.concept_source
                     != ConceptSource.PERSIST_STATEMENT
                 ):
                     original_concept = new_persisted_concept.model_copy(
                         deep=True,
-                        update={
-                            "name": persisted,
-                        },
+                        update={"name": persisted, "pseudonyms": original_pseudonyms},
                     )
                     self.add_concept(
                         original_concept,
@@ -629,6 +626,7 @@ class Environment(BaseModel):
                         ),
                         "derivation": Derivation.ROOT,
                         "purpose": new_persisted_concept.purpose,
+                        "pseudonyms": {*original_pseudonyms, original_concept.address},
                     }
                     # purpose is used in derivation calculation
                     # which should be fixed, but we'll do in a followup
@@ -650,6 +648,7 @@ class Environment(BaseModel):
                         new_persisted_concept,
                         meta=meta,
                     )
+
         return datasource
 
     def delete_datasource(

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/trilogy/core/processing/concept_strategies_v3.py

@@ -224,6 +224,9 @@ def initialize_loop_context(
     else:
 
         completion_mandatory = mandatory_list
+    logger.info(
+        f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Initialized loop context with mandatory list {[c.address for c in mandatory_list]} and completion mandatory {[c.address for c in completion_mandatory]}"
+    )
     return LoopContext(
         mandatory_list=mandatory_list,
         environment=environment,
@@ -330,7 +333,7 @@ def check_for_early_exit(
     return False
 
 
-def generate_loop_completion(context: LoopContext, virtual) -> StrategyNode:
+def generate_loop_completion(context: LoopContext, virtual: set[str]) -> StrategyNode:
     condition_required = True
     non_virtual = [c for c in context.completion_mandatory if c.address not in virtual]
     non_virtual_output = [
@@ -367,10 +370,15 @@ def generate_loop_completion(context: LoopContext, virtual) -> StrategyNode:
         output: StrategyNode = context.stack[0]
         if non_virtual_different:
             logger.info(
-                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Found different non-virtual output concepts ({non_virtual_difference_values}), removing condition injected values"
+                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Found different non-virtual output concepts ({non_virtual_difference_values}), removing condition injected values by setting outputs to {[x.address for x in output.output_concepts if x.address in non_virtual_output]}"
             )
             output.set_output_concepts(
-                [x for x in output.output_concepts if x.address in non_virtual_output],
+                [
+                    x
+                    for x in output.output_concepts
+                    if x.address in non_virtual_output
+                    or any(c in non_virtual_output for c in x.pseudonyms)
+                ],
                 rebuild=False,
             )
 
@@ -404,7 +412,7 @@ def generate_loop_completion(context: LoopContext, virtual) -> StrategyNode:
     elif context.conditions:
         output.preexisting_conditions = context.conditions.conditional
     logger.info(
-        f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node partial {[c.address for c in output.partial_concepts]} with {context.conditions}"
+        f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node output {[x.address for x in output.usable_outputs]} partial {[c.address for c in output.partial_concepts]} with {context.conditions}"
     )
     if condition_required and context.conditions and non_virtual_different:
         logger.info(
@@ -419,7 +427,7 @@ def generate_loop_completion(context: LoopContext, virtual) -> StrategyNode:
         )
         if result.required:
             logger.info(
-                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Adding group node"
+                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Adding group node with outputs {[x.address for x in context.original_mandatory]}"
             )
             return GroupNode(
                 output_concepts=context.original_mandatory,
@@ -466,6 +474,7 @@ def _search_concepts(
     )
 
     while context.incomplete:
+
         priority_concept = get_priority_concept(
             context.mandatory_list,
             context.attempted,
@@ -478,7 +487,7 @@ def _search_concepts(
         candidates = [
             c for c in context.mandatory_list if c.address != priority_concept.address
         ]
-        # the local conditions list may be override if we end up injecting conditions
+        # the local conditions list may be overriden if we end up injecting conditions
         candidate_list, local_conditions = generate_candidates_restrictive(
             priority_concept,
             candidates,

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/trilogy/core/processing/discovery_node_factory.py

@@ -307,16 +307,17 @@ class RootNodeHandler:
     def _resolve_root_concepts(
         self, root_targets: List[BuildConcept]
     ) -> Optional[StrategyNode]:
-        synonym_node = self._try_synonym_resolution(root_targets)
-        if synonym_node:
-            logger.info(
-                f"{depth_to_prefix(self.ctx.depth)}{LOGGER_PREFIX} "
-                f"resolved root concepts through synonyms"
-            )
-            return synonym_node
         expanded_node = self._try_merge_expansion(root_targets)
         if expanded_node:
             return expanded_node
+        if self.ctx.accept_partial:
+            synonym_node = self._try_synonym_resolution(root_targets)
+            if synonym_node:
+                logger.info(
+                    f"{depth_to_prefix(self.ctx.depth)}{LOGGER_PREFIX} "
+                    f"resolved root concepts through synonyms"
+                )
+                return synonym_node
 
         return None
 
@@ -351,7 +352,10 @@ class RootNodeHandler:
         extra = restrict_node_outputs_targets(expanded, root_targets, self.ctx.depth)
 
         pseudonyms = [
-            x for x in extra if any(x.address in y.pseudonyms for y in root_targets)
+            x
+            for x in extra
+            if any(x.address in y.pseudonyms for y in root_targets)
+            and x.address not in root_targets
         ]
 
         if pseudonyms:

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/trilogy/core/processing/discovery_utility.py

@@ -1,7 +1,7 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.enums import Derivation, Granularity
+from trilogy.core.enums import Derivation
 from trilogy.core.models.build import (
     BuildConcept,
     BuildRowsetItem,
@@ -52,14 +52,6 @@ def get_priority_concept(
     # pass_two = [c for c in all_concepts+filter_only if c.address not in attempted_addresses]
     for remaining_concept in (pass_one,):
         priority = (
-            # find anything that needs no joins first, so we can exit early
-            [
-                c
-                for c in remaining_concept
-                if c.derivation == Derivation.CONSTANT
-                and c.granularity == Granularity.SINGLE_ROW
-            ]
-            +
             # then multiselects to remove them from scope
             [c for c in remaining_concept if c.derivation == Derivation.MULTISELECT]
             +

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/trilogy/core/processing/discovery_validation.py

@@ -34,6 +34,9 @@ def validate_concept(
     seen: set[str],
     environment: BuildEnvironment,
 ):
+    # logger.debug(
+    #     f"Validating concept {concept.address} with accept_partial={accept_partial}"
+    # )
     found_map[str(node)].add(concept)
     seen.add(concept.address)
     if concept not in node.partial_concepts:
@@ -53,12 +56,21 @@ def validate_concept(
             found_map[str(node)].add(concept)
     for v_address in concept.pseudonyms:
         if v_address in seen:
-            return
-        v = environment.concepts[v_address]
+            continue
+        if v_address in environment.alias_origin_lookup:
+            # logger.debug(
+            #     f"Found alias origin for {v_address}: {environment.alias_origin_lookup[v_address]} mapped to {environment.concepts[v_address]}")
+            v = environment.alias_origin_lookup[v_address]
+        else:
+            v = environment.concepts[v_address]
+
         if v.address in seen:
-            return
+
+            continue
+
         if v.address == concept.address:
-            return
+
+            continue
         validate_concept(
             v,
             node,
@@ -93,6 +105,7 @@ def validate_stack(
 
         for concept in resolved.output_concepts:
             if concept.address in resolved.hidden_concepts:
+
                 continue
 
             validate_concept(

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/trilogy/core/processing/node_generators/filter_node.py

@@ -57,7 +57,13 @@ def build_parent_concepts(
     local_optional: List[BuildConcept],
     conditions: BuildWhereClause | None = None,
     depth: int = 0,
-):
+) -> tuple[
+    list[BuildConcept],
+    list[tuple[BuildConcept, ...]],
+    list[BuildConcept],
+    bool,
+    bool,
+]:
     parent_row_concepts, parent_existence_concepts = resolve_filter_parent_concepts(
         concept, environment
     )
@@ -66,6 +72,10 @@ def build_parent_concepts(
     filter_where = concept.lineage.where
 
     same_filter_optional: list[BuildConcept] = []
+    # mypy struggled here? we shouldn't need explicit bools
+    global_filter_is_local_filter: bool = (
+        True if (conditions and conditions == filter_where) else False
+    )
 
     for x in local_optional:
         if isinstance(x.lineage, FILTER_TYPES):
@@ -79,7 +89,7 @@ def build_parent_concepts(
                         parent_row_concepts.append(arg)
                 same_filter_optional.append(x)
                 continue
-        elif conditions and conditions == filter_where:
+        elif global_filter_is_local_filter:
             same_filter_optional.append(x)
 
     # sometimes, it's okay to include other local optional above the filter
@@ -100,6 +110,7 @@ def build_parent_concepts(
         parent_existence_concepts,
         same_filter_optional,
         is_optimized_pushdown,
+        global_filter_is_local_filter,
     )
 
 
@@ -152,6 +163,7 @@ def gen_filter_node(
         parent_existence_concepts,
         same_filter_optional,
         optimized_pushdown,
+        global_filter_is_local_filter,
     ) = build_parent_concepts(
         concept,
         environment=environment,
@@ -187,7 +199,13 @@ def gen_filter_node(
             f"{padding(depth)}{LOGGER_PREFIX} filter node row parents {[x.address for x in parent_row_concepts]} could not be found"
         )
         return None
-
+    if global_filter_is_local_filter:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} filter node conditions match global conditions adding row parent {row_parent.output_concepts} with condition {where.conditional}"
+        )
+        row_parent.add_parents(core_parent_nodes)
+        row_parent.set_output_concepts([concept] + local_optional)
+        return row_parent
     if optimized_pushdown:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional} across {[concept] + same_filter_optional + row_parent.output_concepts} "
@@ -211,7 +229,8 @@ def gen_filter_node(
             parent = row_parent
             parent.add_output_concepts([concept] + same_filter_optional)
         parent.add_parents(core_parent_nodes)
-        parent.add_condition(where.conditional)
+        if not parent.preexisting_conditions == where.conditional:
+            parent.add_condition(where.conditional)
         parent.add_existence_concepts(flattened_existence, False)
         parent.grain = BuildGrain.from_concepts(
             parent.output_concepts,
@@ -225,7 +244,8 @@ def gen_filter_node(
         parents_for_grain = [
             x.lineage.content
             for x in filters
-            if isinstance(x.lineage.content, BuildConcept)
+            if isinstance(x.lineage, BuildFilterItem)
+            and isinstance(x.lineage.content, BuildConcept)
         ]
         filter_node = FilterNode(
             input_concepts=unique(

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/trilogy/core/processing/node_generators/group_node.py

@@ -133,7 +133,7 @@ def gen_group_node(
         )
         return group_node
     missing_optional = [
-        x.address for x in local_optional if x.address not in group_node.output_concepts
+        x.address for x in local_optional if x.address not in group_node.usable_outputs
     ]
     if not missing_optional:
         logger.info(

{pytrilogy-0.0.3.66 → pytrilogy-0.0.3.68}/trilogy/core/processing/node_generators/node_merge_node.py

@@ -42,6 +42,9 @@ def extract_concept(node: str, env: BuildEnvironment):
 
 def filter_unique_graphs(graphs: list[list[str]]) -> list[list[str]]:
     unique_graphs: list[set[str]] = []
+
+    # sort graphs from largest to smallest
+    graphs.sort(key=lambda x: len(x), reverse=True)
     for graph in graphs:
         if not any(set(graph).issubset(x) for x in unique_graphs):
             unique_graphs.append(set(graph))
@@ -110,12 +113,13 @@ def determine_induced_minimal_nodes(
 
     try:
         paths = nx.multi_source_dijkstra_path(H, nodelist)
-    except nx.exception.NodeNotFound as e:
-        logger.debug(f"Unable to find paths for {nodelist}- {str(e)}")
+        # logger.debug(f"Paths found for {nodelist}")
+    except nx.exception.NodeNotFound:
+        # logger.debug(f"Unable to find paths for {nodelist}- {str(e)}")
         return None
     H.remove_nodes_from(list(x for x in H.nodes if x not in paths))
     sG: nx.Graph = ax.steinertree.steiner_tree(H, nodelist).copy()
-    logger.debug("Steiner tree found for nodes %s", nodelist)
+    # logger.debug(f"Steiner tree found for nodes {nodelist} {sG.nodes}")
     final: nx.DiGraph = nx.subgraph(G, sG.nodes).copy()
 
     for edge in G.edges:
@@ -154,12 +158,31 @@ def determine_induced_minimal_nodes(
     return final
 
 
+def canonicalize_addresses(
+    reduced_concept_set: set[str], environment: BuildEnvironment
+) -> set[str]:
+    """
+    Convert a set of concept addresses to their canonical form.
+    This is necessary to ensure that we can compare concepts correctly,
+    especially when dealing with aliases or pseudonyms.
+    """
+    return set(
+        environment.concepts[x].address if x in environment.concepts else x
+        for x in reduced_concept_set
+    )
+
+
 def detect_ambiguity_and_raise(
-    all_concepts: list[BuildConcept], reduced_concept_sets: list[set[str]]
+    all_concepts: list[BuildConcept],
+    reduced_concept_sets_raw: list[set[str]],
+    environment: BuildEnvironment,
 ) -> None:
     final_candidates: list[set[str]] = []
     common: set[str] = set()
     # find all values that show up in every join_additions
+    reduced_concept_sets = [
+        canonicalize_addresses(x, environment) for x in reduced_concept_sets_raw
+    ]
     for ja in reduced_concept_sets:
         if not common:
             common = ja
@@ -198,18 +221,21 @@ def filter_relevant_subgraphs(
 
 
 def filter_duplicate_subgraphs(
-    subgraphs: list[list[BuildConcept]],
+    subgraphs: list[list[BuildConcept]], environment
 ) -> list[list[BuildConcept]]:
     seen: list[set[str]] = []
 
     for graph in subgraphs:
-        seen.append(set([x.address for x in graph]))
+        seen.append(
+            canonicalize_addresses(set([x.address for x in graph]), environment)
+        )
     final = []
     # sometimes w can get two subcomponents that are the same
     # due to alias resolution
     # if so, drop any that are strict subsets.
     for graph in subgraphs:
-        set_x = set([x.address for x in graph])
+        logger.info(f"Checking graph {graph} for duplicates in {seen}")
+        set_x = canonicalize_addresses(set([x.address for x in graph]), environment)
         if any([set_x.issubset(y) and set_x != y for y in seen]):
             continue
         final.append(graph)
@@ -295,7 +321,7 @@ def resolve_weak_components(
     if not found:
         return None
 
-    detect_ambiguity_and_raise(all_concepts, reduced_concept_sets)
+    detect_ambiguity_and_raise(all_concepts, reduced_concept_sets, environment)
 
     # take our first one as the actual graph
     g = found[0]
@@ -316,7 +342,7 @@ def resolve_weak_components(
         if not sub_component:
             continue
         subgraphs.append(sub_component)
-    final = filter_duplicate_subgraphs(subgraphs)
+    final = filter_duplicate_subgraphs(subgraphs, environment)
     return final
     # return filter_relevant_subgraphs(subgraphs)
 
@@ -361,17 +387,25 @@ def subgraphs_to_merge_node(
         )
         parents.append(parent)
     input_c = []
+    output_c = []
     for x in parents:
         for y in x.usable_outputs:
             input_c.append(y)
+            if y in output_concepts:
+                output_c.append(y)
+            elif any(y.address in c.pseudonyms for c in output_concepts) or any(
+                c.address in y.pseudonyms for c in output_concepts
+            ):
+                output_c.append(y)
+
     if len(parents) == 1 and enable_early_exit:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} only one parent node, exiting early w/ {[c.address for c in parents[0].output_concepts]}"
         )
         return parents[0]
-    return MergeNode(
+    rval = MergeNode(
         input_concepts=unique(input_c, "address"),
-        output_concepts=output_concepts,
+        output_concepts=output_c,
         environment=environment,
         parents=parents,
         depth=depth,
@@ -381,6 +415,7 @@ def subgraphs_to_merge_node(
         # preexisting_conditions=search_conditions.conditional,
         # node_joins=[]
     )
+    return rval
 
 
 def gen_merge_node(
@@ -437,24 +472,4 @@ def gen_merge_node(
                 search_conditions=search_conditions,
                 output_concepts=all_concepts,
             )
-
-    # one concept handling may need to be kicked to alias
-    if len(all_search_concepts) == 1:
-        concept = all_search_concepts[0]
-        for v in concept.pseudonyms:
-            test = subgraphs_to_merge_node(
-                [[concept, environment.alias_origin_lookup[v]]],
-                g=g,
-                all_concepts=[concept],
-                environment=environment,
-                depth=depth,
-                source_concepts=source_concepts,
-                history=history,
-                conditions=conditions,
-                enable_early_exit=False,
-                search_conditions=search_conditions,
-                output_concepts=[concept],
-            )
-            if test:
-                return test
     return None