pytrilogy 0.0.2.18__tar.gz → 0.0.2.20__tar.gz

{pytrilogy-0.0.2.18/pytrilogy.egg-info → pytrilogy-0.0.2.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.18
+Version: 0.0.2.20
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.18 → pytrilogy-0.0.2.20/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.18
+Version: 0.0.2.20
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.18 → pytrilogy-0.0.2.20}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.18"
+__version__ = "0.0.2.20"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.18 → pytrilogy-0.0.2.20}/trilogy/core/models.py

@@ -559,19 +559,16 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
         grain = ",".join([str(c.address) for c in self.grain.components])
         return f"{self.namespace}.{self.name}<{grain}>"
 
-    @property
+    @cached_property
     def address(self) -> str:
-        if not self._address_cache:
-            self._address_cache = f"{self.namespace}.{self.name}"
-        return self._address_cache
-
-    @address.setter
-    def address(self, address: str) -> None:
-        self._address_cache = address
+        return f"{self.namespace}.{self.name}"
 
     def set_name(self, name: str):
         self.name = name
-        self.address = f"{self.namespace}.{self.name}"
+        try:
+            del self.address
+        except AttributeError:
+            pass
 
     @property
     def output(self) -> "Concept":
@@ -3260,23 +3257,13 @@ class Environment(BaseModel):
         for datasource in self.datasources.values():
             for concept in datasource.output_concepts:
                 concrete_addresses.add(concept.address)
-        current_mat = [x.address for x in self.materialized_concepts]
         self.materialized_concepts = [
             c for c in self.concepts.values() if c.address in concrete_addresses
-        ]
-        # include aliased concepts
-        self.materialized_concepts += [
+        ] + [
             c
             for c in self.alias_origin_lookup.values()
             if c.address in concrete_addresses
         ]
-        new = [
-            x.address
-            for x in self.materialized_concepts
-            if x.address not in current_mat
-        ]
-        if new:
-            logger.debug(f"Environment added new materialized concepts {new}")
 
     def validate_concept(self, lookup: str, meta: Meta | None = None):
         existing: Concept = self.concepts.get(lookup)  # type: ignore
@@ -3400,6 +3387,7 @@ class Environment(BaseModel):
         meta: Meta | None = None,
         force: bool = False,
         add_derived: bool = True,
+        _ignore_cache: bool = False,
     ):
         if not force:
             self.validate_concept(concept.address, meta=meta)
@@ -3410,13 +3398,15 @@ class Environment(BaseModel):
         from trilogy.core.environment_helpers import generate_related_concepts
 
         generate_related_concepts(concept, self, meta=meta, add_derived=add_derived)
-        self.gen_concept_list_caches()
+        if not _ignore_cache:
+            self.gen_concept_list_caches()
         return concept
 
     def add_datasource(
         self,
         datasource: Datasource,
         meta: Meta | None = None,
+        _ignore_cache: bool = False,
     ):
 
         self.datasources[datasource.env_label] = datasource
@@ -3428,11 +3418,13 @@ class Environment(BaseModel):
                 new_concept.set_name("_pre_persist_" + current_concept.name)
                 # remove the associated lineage
                 current_concept.lineage = None
-                self.add_concept(new_concept, meta=meta, force=True)
-                self.add_concept(current_concept, meta=meta, force=True)
+                self.add_concept(new_concept, meta=meta, force=True, _ignore_cache=True)
+                self.add_concept(
+                    current_concept, meta=meta, force=True, _ignore_cache=True
+                )
                 self.merge_concept(new_concept, current_concept, [])
-
-        self.gen_concept_list_caches()
+        if not _ignore_cache:
+            self.gen_concept_list_caches()
         return datasource
 
     def delete_datasource(

{pytrilogy-0.0.2.18 → pytrilogy-0.0.2.20}/trilogy/core/processing/node_generators/group_node.py

@@ -5,6 +5,7 @@ from trilogy.core.models import (
     WhereClause,
     Function,
     AggregateWrapper,
+    Grain,
 )
 from trilogy.utility import unique
 from trilogy.core.processing.nodes import GroupNode, StrategyNode, History
@@ -50,9 +51,9 @@ def gen_group_node(
         parent_concepts += grain_components
         output_concepts += grain_components
         for possible_agg in local_optional:
+            if not isinstance(possible_agg.lineage, (AggregateWrapper, Function)):
+                continue
             if possible_agg.grain and possible_agg.grain == concept.grain:
-                if not isinstance(possible_agg.lineage, (AggregateWrapper, Function)):
-                    continue
                 agg_parents: List[Concept] = resolve_function_parent_concepts(
                     possible_agg
                 )
@@ -60,7 +61,16 @@ def gen_group_node(
                     set([x.address for x in parent_concepts])
                 ):
                     output_concepts.append(possible_agg)
-
+                    logger.info(
+                        f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                    )
+                elif Grain(components=agg_parents) == Grain(components=parent_concepts):
+                    extra = [x for x in agg_parents if x.address not in parent_concepts]
+                    parent_concepts += extra
+                    output_concepts.append(possible_agg)
+                    logger.info(
+                        f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                    )
     if parent_concepts:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} fetching group node parents {LooseConceptList(concepts=parent_concepts)}"

{pytrilogy-0.0.2.18 → pytrilogy-0.0.2.20}/trilogy/core/processing/node_generators/node_merge_node.py

@@ -148,7 +148,6 @@ def detect_ambiguity_and_raise(
             common = common.intersection(ja)
         if all(set(ja).issubset(y) for y in reduced_concept_sets):
             final_candidates.append(ja)
-
     if not final_candidates:
         filtered_paths = [x.difference(common) for x in reduced_concept_sets]
         raise AmbiguousRelationshipResolutionException(

{pytrilogy-0.0.2.18 → pytrilogy-0.0.2.20}/trilogy/core/processing/node_generators/select_merge_node.py

@@ -21,7 +21,6 @@ from trilogy.constants import logger
 from trilogy.core.processing.utility import padding
 from trilogy.core.enums import PurposeLineage
 
-
 LOGGER_PREFIX = "[GEN_ROOT_MERGE_NODE]"
 
 
@@ -52,18 +51,19 @@ def get_graph_grain_length(g: nx.DiGraph) -> dict[str, int]:
 def create_pruned_concept_graph(
     g: nx.DiGraph, all_concepts: List[Concept], accept_partial: bool = False
 ) -> nx.DiGraph:
+    orig_g = g
     g = g.copy()
     target_addresses = set([c.address for c in all_concepts])
-    concepts: dict[str, Concept] = nx.get_node_attributes(g, "concept")
-    datasources: dict[str, Datasource] = nx.get_node_attributes(g, "datasource")
+    concepts: dict[str, Concept] = nx.get_node_attributes(orig_g, "concept")
+    datasources: dict[str, Datasource] = nx.get_node_attributes(orig_g, "datasource")
     relevant_concepts_pre = {
         n: x.address
         for n in g.nodes()
         # filter out synonyms
         if (x := concepts.get(n, None)) and x.address in target_addresses
     }
-    relevant_concepts = list(relevant_concepts_pre.keys())
-    relevent_datasets = []
+    relevant_concepts: list[str] = list(relevant_concepts_pre.keys())
+    relevent_datasets: list[str] = []
     if not accept_partial:
         partial = {}
         for node in g.nodes:
@@ -95,15 +95,22 @@ def create_pruned_concept_graph(
         ]
         if actual_neighbors:
             relevent_datasets.append(n)
-    for n in g.nodes():
-        if n.startswith("c~") and n not in relevant_concepts:
-            neighbor_count = 0
-            for x in nx.all_neighbors(g, n):
-                if x in relevent_datasets:
-                    neighbor_count += 1
-            if neighbor_count > 1:
-                relevant_concepts.append(concepts.get(n))
 
+    # for injecting extra join concepts that are shared between datasets
+    # use the original graph, pre-partial pruning
+    for n in orig_g.nodes:
+        # readd ignoring grain
+        # we want to join inclusive of all concepts
+        roots: dict[str, set[str]] = {}
+        if n.startswith("c~") and n not in relevant_concepts:
+            root = n.split("@")[0]
+            neighbors = roots.get(root, set())
+            for neighbor in nx.all_neighbors(orig_g, n):
+                if neighbor in relevent_datasets:
+                    neighbors.add(neighbor)
+            if len(neighbors) > 1:
+                relevant_concepts.append(n)
+            roots[root] = set()
     g.remove_nodes_from(
         [
             n
@@ -111,6 +118,18 @@ def create_pruned_concept_graph(
             if n not in relevent_datasets and n not in relevant_concepts
         ]
     )
+
+    subgraphs = list(nx.connected_components(g.to_undirected()))
+    if not subgraphs:
+        return None
+    if subgraphs and len(subgraphs) != 1:
+        return None
+    # add back any relevant edges that might have been partially filtered
+    relevant = set(relevant_concepts + relevent_datasets)
+    for edge in orig_g.edges():
+        if edge[0] in relevant and edge[1] in relevant:
+            g.add_edge(edge[0], edge[1])
+
     return g
 
 
@@ -243,19 +262,15 @@ def gen_select_merge_node(
         )
     for attempt in [False, True]:
         pruned_concept_graph = create_pruned_concept_graph(g, non_constant, attempt)
-        subgraphs = list(nx.connected_components(pruned_concept_graph.to_undirected()))
-
-        if subgraphs and len(subgraphs) == 1:
+        if pruned_concept_graph:
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} found covering graph w/ partial flag {attempt}"
             )
             break
-    if len(subgraphs) > 1:
-        # from trilogy.hooks.graph_hook import GraphHook
-        # GraphHook().query_graph_built(pruned_concept_graph.to_undirected(), highlight_nodes=[concept_to_node(c.with_default_grain()) for c in all_concepts if "__preql_internal" not in c.address])
-        # raise SyntaxError(f'Too many subgraphs found for {[c.address for c in all_concepts]}: got {subgraphs}')
+
+    if not pruned_concept_graph:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Too many subgraphs found for {[c.address for c in non_constant]}: got {subgraphs}'"
+            f"{padding(depth)}{LOGGER_PREFIX} no covering graph found {attempt}"
         )
         return None
 

{pytrilogy-0.0.2.18 → pytrilogy-0.0.2.20}/trilogy/core/processing/utility.py

@@ -199,7 +199,6 @@ def get_node_joins(
     identifier_map: dict[str, Datasource | QueryDatasource] = {
         x.identifier: x for x in datasources
     }
-
     grain_pseudonyms: set[str] = set()
     for g in grain:
         env_lookup = environment.concepts[g.address]

{pytrilogy-0.0.2.18 → pytrilogy-0.0.2.20}/trilogy/parsing/parse_engine.py

@@ -814,14 +814,19 @@ class ParseToObjects(Transformer):
                 raise ImportError(f"Unable to import file {target}, parsing error: {e}")
 
         for _, concept in nparser.environment.concepts.items():
-            self.environment.add_concept(concept.with_namespace(alias))
+            self.environment.add_concept(
+                concept.with_namespace(alias), _ignore_cache=True
+            )
 
         for _, datasource in nparser.environment.datasources.items():
-            self.environment.add_datasource(datasource.with_namespace(alias))
+            self.environment.add_datasource(
+                datasource.with_namespace(alias), _ignore_cache=True
+            )
         imps = ImportStatement(
             alias=alias, path=Path(args[0]), environment=nparser.environment
         )
         self.environment.imports[alias] = imps
+        self.environment.gen_concept_list_caches()
         return imps
 
     @v_args(meta=True)