pytrilogy 0.0.3.93__tar.gz → 0.0.3.94__tar.gz

{pytrilogy-0.0.3.93/pytrilogy.egg-info → pytrilogy-0.0.3.94}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.93
+Version: 0.0.3.94
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.93 → pytrilogy-0.0.3.94/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.93
+Version: 0.0.3.94
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.93 → pytrilogy-0.0.3.94}/pytrilogy.egg-info/SOURCES.txt

@@ -90,7 +90,6 @@ trilogy/core/processing/node_generators/node_merge_node.py
 trilogy/core/processing/node_generators/recursive_node.py
 trilogy/core/processing/node_generators/rowset_node.py
 trilogy/core/processing/node_generators/select_merge_node.py
-trilogy/core/processing/node_generators/select_merge_node_v2.py
 trilogy/core/processing/node_generators/select_node.py
 trilogy/core/processing/node_generators/synonym_node.py
 trilogy/core/processing/node_generators/union_node.py

{pytrilogy-0.0.3.93 → pytrilogy-0.0.3.94}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.93"
+__version__ = "0.0.3.94"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.93 → pytrilogy-0.0.3.94}/trilogy/core/env_processor.py

@@ -44,8 +44,10 @@ def add_concept(
             continue
         if pseudonym_node.split("@")[0] == node_name.split("@")[0]:
             continue
-        g.add_edge(pseudonym_node, node_name, pseudonym=True)
-        g.add_edge(node_name, pseudonym_node, pseudonym=True)
+        g.add_edge(pseudonym_node, node_name)
+        g.add_edge(node_name, pseudonym_node)
+        g.pseudonyms.add((pseudonym_node, node_name))
+        g.pseudonyms.add((node_name, pseudonym_node))
         add_concept(pseudonym, g, concept_mapping, default_concept_graph, seen)
 
 

pytrilogy-0.0.3.94/trilogy/core/graph_models.py

@@ -0,0 +1,132 @@
+from typing import Union
+
+import networkx as nx
+
+from trilogy.core.models.build import BuildConcept, BuildDatasource, BuildWhereClause
+
+
+def get_graph_exact_match(
+    g: Union[nx.DiGraph, "ReferenceGraph"],
+    accept_partial: bool,
+    conditions: BuildWhereClause | None,
+) -> set[str]:
+    exact: set[str] = set()
+    for node, ds in g.datasources.items():
+        if isinstance(ds, list):
+            exact.add(node)
+            continue
+
+        if not conditions and not ds.non_partial_for:
+            exact.add(node)
+            continue
+        elif not conditions and accept_partial and ds.non_partial_for:
+            exact.add(node)
+            continue
+        elif conditions:
+            if not ds.non_partial_for:
+                continue
+            if ds.non_partial_for and conditions == ds.non_partial_for:
+                exact.add(node)
+                continue
+        else:
+            continue
+
+    return exact
+
+
+def prune_sources_for_conditions(
+    g: "ReferenceGraph",
+    accept_partial: bool,
+    conditions: BuildWhereClause | None,
+):
+    complete = get_graph_exact_match(g, accept_partial, conditions)
+    to_remove = []
+    for node in g.datasources:
+        if node not in complete:
+            to_remove.append(node)
+
+    for node in to_remove:
+        g.remove_node(node)
+
+
+def concept_to_node(input: BuildConcept) -> str:
+    # if input.purpose == Purpose.METRIC:
+    #     return f"c~{input.namespace}.{input.name}@{input.grain}"
+    return f"c~{input.address}@{input.grain.str_no_condition}"
+
+
+def datasource_to_node(input: BuildDatasource) -> str:
+    # if isinstance(input, JoinedDataSource):
+    #     return "ds~join~" + ",".join(
+    #         [datasource_to_node(sub) for sub in input.datasources]
+    #     )
+    return f"ds~{input.identifier}"
+
+
+class ReferenceGraph(nx.DiGraph):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.concepts: dict[str, BuildConcept] = {}
+        self.datasources: dict[str, BuildDatasource] = {}
+        self.pseudonyms: set[tuple[str, str]] = set()
+
+    def copy(self):
+        g = ReferenceGraph()
+        g.concepts = self.concepts.copy()
+        g.datasources = self.datasources.copy()
+        g.pseudonyms = {*self.pseudonyms}
+        # g.add_nodes_from(self.nodes(data=True))
+        for node in self.nodes:
+            g.add_node(node, fast=True)
+        for edge in self.edges:
+            g.add_edge(edge[0], edge[1], fast=True)
+        # g.add_edges_from(self.edges(data=True))
+        return g
+
+    def remove_node(self, n):
+        if n in self.concepts:
+            del self.concepts[n]
+        if n in self.datasources:
+            del self.datasources[n]
+        super().remove_node(n)
+
+    def add_node(self, node_for_adding, fast: bool = False, **attr):
+        if fast:
+            return super().add_node(node_for_adding, **attr)
+        if isinstance(node_for_adding, BuildConcept):
+            node_name = concept_to_node(node_for_adding)
+            self.concepts[node_name] = node_for_adding
+        elif isinstance(node_for_adding, BuildDatasource):
+            node_name = datasource_to_node(node_for_adding)
+            self.datasources[node_name] = node_for_adding
+        else:
+            node_name = node_for_adding
+            if attr.get("datasource"):
+                self.datasources[node_name] = attr["datasource"]
+        super().add_node(node_name, **attr)
+
+    def add_edge(self, u_of_edge, v_of_edge, fast: bool = False, **attr):
+        if fast:
+            return super().add_edge(u_of_edge, v_of_edge, **attr)
+        if isinstance(u_of_edge, BuildConcept):
+            orig = u_of_edge
+            u_of_edge = concept_to_node(u_of_edge)
+            if u_of_edge not in self.nodes:
+                self.add_node(orig)
+        elif isinstance(u_of_edge, BuildDatasource):
+            origd = u_of_edge
+            u_of_edge = datasource_to_node(u_of_edge)
+            if u_of_edge not in self.nodes:
+                self.add_node(origd)
+
+        if isinstance(v_of_edge, BuildConcept):
+            orig = v_of_edge
+            v_of_edge = concept_to_node(v_of_edge)
+            if v_of_edge not in self.nodes:
+                self.add_node(orig)
+        elif isinstance(v_of_edge, BuildDatasource):
+            origd = v_of_edge
+            v_of_edge = datasource_to_node(v_of_edge)
+            if v_of_edge not in self.nodes:
+                self.add_node(origd)
+        super().add_edge(u_of_edge, v_of_edge)

{pytrilogy-0.0.3.93 → pytrilogy-0.0.3.94}/trilogy/core/models/author.py

@@ -902,7 +902,11 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
 
     @property
     def is_aggregate(self):
-        return self.calculate_is_aggregate(self.lineage)
+        base = getattr(self, "_is_aggregate", None)
+        if base:
+            return base
+        setattr(self, "_is_aggregate", self.calculate_is_aggregate(self.lineage))
+        return self._is_aggregate
 
     def with_merge(self, source: Self, target: Self, modifiers: List[Modifier]) -> Self:
         if self.address == source.address:
@@ -1069,18 +1073,25 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
         final_grain = grain if not self.grain.components else self.grain
         keys = self.keys
 
-        if self.is_aggregate and isinstance(new_lineage, Function) and grain.components:
+        if self.is_aggregate and grain.components and isinstance(new_lineage, Function):
             grain_components: list[ConceptRef | Concept] = [
                 environment.concepts[c].reference for c in grain.components
             ]
-            new_lineage = AggregateWrapper(function=new_lineage, by=grain_components)
+            new_lineage = AggregateWrapper.model_construct(
+                function=new_lineage, by=grain_components
+            )
             final_grain = grain
             keys = set(grain.components)
-        elif isinstance(new_lineage, AggregateWrapper) and not new_lineage.by and grain:
+        elif (
+            grain
+            and new_lineage
+            and isinstance(new_lineage, AggregateWrapper)
+            and not new_lineage.by
+        ):
             grain_components = [
                 environment.concepts[c].reference for c in grain.components
             ]
-            new_lineage = AggregateWrapper(
+            new_lineage = AggregateWrapper.model_construct(
                 function=new_lineage.function, by=grain_components
             )
             final_grain = grain
@@ -1670,15 +1681,6 @@ class Function(DataTyped, ConceptArgs, Mergeable, Namespaced, BaseModel):
     def datatype(self):
         return self.output_datatype
 
-    @field_validator("output_datatype")
-    @classmethod
-    def parse_output_datatype(cls, v, info: ValidationInfo):
-        values = info.data
-        if values.get("operator") == FunctionType.ATTR_ACCESS:
-            if isinstance(v, StructType):
-                raise SyntaxError
-        return v
-
     @field_validator("arguments", mode="before")
     @classmethod
     def parse_arguments(cls, v, info: ValidationInfo):
@@ -1845,17 +1847,6 @@ class Function(DataTyped, ConceptArgs, Mergeable, Namespaced, BaseModel):
             base += get_concept_arguments(arg)
         return base
 
-    @property
-    def output_grain(self):
-        # aggregates have an abstract grain
-        base_grain = Grain(components=[])
-        if self.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
-            return base_grain
-        # scalars have implicit grain of all arguments
-        for input in self.concept_arguments:
-            base_grain += input.grain
-        return base_grain
-
 
 class FunctionCallWrapper(
     DataTyped,

{pytrilogy-0.0.3.93 → pytrilogy-0.0.3.94}/trilogy/core/models/build.py

@@ -1533,6 +1533,7 @@ class Factory:
         )
         self.local_non_build_concepts: dict[str, Concept] = {}
         self.pseudonym_map = pseudonym_map or get_canonical_pseudonyms(environment)
+        self.build_grain = self.build(self.grain) if self.grain else None
 
     def instantiate_concept(
         self,
@@ -1792,11 +1793,11 @@ class Factory:
         address = base.concept.address
         fetched = (
             self._build_concept(
-                self.environment.alias_origin_lookup[address].with_grain(self.grain)
-            )
+                self.environment.alias_origin_lookup[address]
+            ).with_grain(self.build_grain)
             if address in self.environment.alias_origin_lookup
-            else self._build_concept(
-                self.environment.concepts[address].with_grain(self.grain)
+            else self._build_concept(self.environment.concepts[address]).with_grain(
+                self.build_grain
             )
         )
 

{pytrilogy-0.0.3.93 → pytrilogy-0.0.3.94}/trilogy/core/processing/node_generators/node_merge_node.py

@@ -6,7 +6,11 @@ from networkx.algorithms import approximation as ax
 from trilogy.constants import logger
 from trilogy.core.enums import Derivation
 from trilogy.core.exceptions import AmbiguousRelationshipResolutionException
-from trilogy.core.graph_models import concept_to_node, prune_sources_for_conditions
+from trilogy.core.graph_models import (
+    ReferenceGraph,
+    concept_to_node,
+    prune_sources_for_conditions,
+)
 from trilogy.core.models.build import BuildConcept, BuildConditional, BuildWhereClause
 from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
@@ -17,11 +21,12 @@ LOGGER_PREFIX = "[GEN_MERGE_NODE]"
 AMBIGUITY_CHECK_LIMIT = 20
 
 
-def filter_pseudonyms_for_source(ds_graph: nx.DiGraph, node: str):
+def filter_pseudonyms_for_source(
+    ds_graph: nx.DiGraph, node: str, pseudonyms: set[tuple[str, str]]
+):
     to_remove = set()
-
     for edge in ds_graph.edges:
-        if ds_graph.edges[edge].get("pseudonym", False):
+        if edge in pseudonyms:
             lengths = {}
             for n in edge:
                 lengths[n] = nx.shortest_path_length(ds_graph, node, n)
@@ -52,12 +57,14 @@ def filter_unique_graphs(graphs: list[list[str]]) -> list[list[str]]:
     return [list(x) for x in unique_graphs]
 
 
-def extract_ds_components(g: nx.DiGraph, nodelist: list[str]) -> list[list[str]]:
+def extract_ds_components(
+    g: nx.DiGraph, nodelist: list[str], pseudonyms: set[tuple[str, str]]
+) -> list[list[str]]:
     graphs = []
     for node in g.nodes:
         if node.startswith("ds~"):
             local = g.copy()
-            filter_pseudonyms_for_source(local, node)
+            filter_pseudonyms_for_source(local, node, pseudonyms)
             ds_graph: nx.DiGraph = nx.ego_graph(local, node, radius=10).copy()
             graphs.append(
                 [
@@ -78,7 +85,7 @@ def extract_ds_components(g: nx.DiGraph, nodelist: list[str]) -> list[list[str]]
 
 
 def determine_induced_minimal_nodes(
-    G: nx.DiGraph,
+    G: ReferenceGraph,
     nodelist: list[str],
     environment: BuildEnvironment,
     filter_downstream: bool,
@@ -86,23 +93,19 @@ def determine_induced_minimal_nodes(
 ) -> nx.DiGraph | None:
     H: nx.Graph = nx.to_undirected(G).copy()
     nodes_to_remove = []
-    concepts = nx.get_node_attributes(G, "concept")
-
-    for node in G.nodes:
-        if concepts.get(node):
-            lookup: BuildConcept = concepts[node]
-            # inclusion of aggregates can create ambiguous node relation chains
-            # there may be a better way to handle this
-            # can be revisited if we need to connect a derived synonym based on an aggregate
-            if lookup.derivation in (
-                Derivation.CONSTANT,
-                Derivation.AGGREGATE,
-                Derivation.FILTER,
-            ):
-                nodes_to_remove.append(node)
-            # purge a node if we're already looking for all it's parents
-            if filter_downstream and lookup.derivation not in (Derivation.ROOT,):
-                nodes_to_remove.append(node)
+    for node, lookup in G.concepts.items():
+        # inclusion of aggregates can create ambiguous node relation chains
+        # there may be a better way to handle this
+        # can be revisited if we need to connect a derived synonym based on an aggregate
+        if lookup.derivation in (
+            Derivation.CONSTANT,
+            Derivation.AGGREGATE,
+            Derivation.FILTER,
+        ):
+            nodes_to_remove.append(node)
+        # purge a node if we're already looking for all it's parents
+        if filter_downstream and lookup.derivation not in (Derivation.ROOT,):
+            nodes_to_remove.append(node)
     if nodes_to_remove:
         # logger.debug(f"Removing nodes {nodes_to_remove} from graph")
         H.remove_nodes_from(nodes_to_remove)
@@ -259,7 +262,7 @@ def filter_duplicate_subgraphs(
 def resolve_weak_components(
     all_concepts: List[BuildConcept],
     environment: BuildEnvironment,
-    environment_graph: nx.DiGraph,
+    environment_graph: ReferenceGraph,
     filter_downstream: bool = True,
     accept_partial: bool = False,
     search_conditions: BuildWhereClause | None = None,
@@ -316,8 +319,6 @@ def resolve_weak_components(
             ]
             new = [x for x in all_graph_concepts if x.address not in all_concepts]
 
-            new_addresses = set([x.address for x in new if x.address not in synonyms])
-
             if not new:
                 break_flag = True
             # remove our new nodes for the next search path
@@ -329,6 +330,7 @@ def resolve_weak_components(
             # from trilogy.hooks.graph_hook import GraphHook
             # GraphHook().query_graph_built(g, highlight_nodes=[concept_to_node(c.with_default_grain()) for c in all_concepts if "__preql_internal" not in c.address])
             found.append(g)
+            new_addresses = set([x.address for x in new if x.address not in synonyms])
             reduced_concept_sets.append(new_addresses)
 
         except nx.exception.NetworkXNoPath:
@@ -346,7 +348,7 @@ def resolve_weak_components(
     subgraphs: list[list[BuildConcept]] = []
     # components = nx.strongly_connected_components(g)
     node_list = [x for x in g.nodes if x.startswith("c~")]
-    components = extract_ds_components(g, node_list)
+    components = extract_ds_components(g, node_list, environment_graph.pseudonyms)
     logger.debug(f"Extracted components {components} from {node_list}")
     for component in components:
         # we need to take unique again as different addresses may map to the same concept

{pytrilogy-0.0.3.93 → pytrilogy-0.0.3.94}/trilogy/core/processing/node_generators/select_helpers/datasource_injection.py

@@ -128,17 +128,32 @@ def simplify_conditions(
     for condition in conditions:
         if not isinstance(condition, BuildComparison):
             return False
-        if not isinstance(condition.left, REDUCABLE_TYPES) and not isinstance(
-            condition.right, REDUCABLE_TYPES
-        ):
-            return False
-        if not isinstance(condition.left, BuildConcept) and not isinstance(
-            condition.right, BuildConcept
+        left_is_concept = False
+        left_is_reducable = False
+        right_is_concept = False
+        right_is_reducable = False
+        if isinstance(condition.left, BuildConcept):
+            left_is_concept = True
+        elif isinstance(condition.left, REDUCABLE_TYPES):
+            left_is_reducable = True
+
+        if isinstance(condition.right, BuildConcept):
+            right_is_concept = True
+        elif isinstance(condition.right, REDUCABLE_TYPES):
+            right_is_reducable = True
+
+        if not (
+            (left_is_concept and right_is_reducable)
+            or (right_is_concept and left_is_reducable)
         ):
             return False
-        vars = [condition.left, condition.right]
-        concept = [x for x in vars if isinstance(x, BuildConcept)][0]
-        raw_comparison = [x for x in vars if not isinstance(x, BuildConcept)][0]
+        if left_is_concept:
+            concept = condition.left
+            raw_comparison = condition.right
+        else:
+            concept = condition.right
+            raw_comparison = condition.left
+
         if isinstance(raw_comparison, BuildFunction):
             if not raw_comparison.operator == FunctionType.CONSTANT:
                 return False
@@ -154,7 +169,7 @@ def simplify_conditions(
         if not isinstance(comparison, REDUCABLE_TYPES):
             return False
 
-        var = concept
+        var: BuildConcept = concept  # type: ignore
         op = condition.operator
         grouped[var].append((op, comparison))
 
@@ -240,7 +255,6 @@ def get_union_sources(
         assocs[merge_key.address].append(x)
     final: list[list[BuildDatasource]] = []
     for _, dses in assocs.items():
-
         conditions = [c.non_partial_for.conditional for c in dses if c.non_partial_for]
         if simplify_conditions(conditions):
             final.append(dses)