pytrilogy 0.0.2.23__py3-none-any.whl → 0.0.2.26__py3-none-any.whl

trilogy/core/models.py

@@ -73,6 +73,7 @@ from collections import UserList, UserDict
 from functools import cached_property
 from abc import ABC
 from collections import defaultdict
+import hashlib
 
 LOGGER_PREFIX = "[MODELS]"
 
@@ -102,6 +103,12 @@ def get_version():
     return __version__
 
 
+def address_with_namespace(address: str, namespace: str) -> str:
+    if address.split(".", 1)[0] == DEFAULT_NAMESPACE:
+        return f"{namespace}.{address.split('.',1)[1]}"
+    return f"{namespace}.{address}"
+
+
 def get_concept_arguments(expr) -> List["Concept"]:
     output = []
     if isinstance(expr, Concept):
@@ -184,6 +191,13 @@ class ConstantInlineable(ABC):
         raise NotImplementedError
 
 
+class HasUUID(ABC):
+
+    @property
+    def uuid(self) -> str:
+        return hashlib.md5(str(self).encode()).hexdigest()
+
+
 class SelectTypeMixin(BaseModel):
     where_clause: Union["WhereClause", None] = Field(default=None)
     having_clause: Union["HavingClause", None] = Field(default=None)
@@ -436,7 +450,7 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     keys: Optional[Tuple["Concept", ...]] = None
     grain: "Grain" = Field(default=None, validate_default=True)
     modifiers: Optional[List[Modifier]] = Field(default_factory=list)
-    pseudonyms: Dict[str, Concept] = Field(default_factory=dict)
+    pseudonyms: set[str] = Field(default_factory=set)
     _address_cache: str | None = None
 
     def __hash__(self):
@@ -462,7 +476,7 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     def with_merge(self, source: Concept, target: Concept, modifiers: List[Modifier]):
         if self.address == source.address:
             new = target.with_grain(self.grain.with_merge(source, target, modifiers))
-            new.pseudonyms[self.address] = self
+            new.pseudonyms.add(self.address)
             return new
         return self.__class__(
             name=self.name,
@@ -616,9 +630,7 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
                 else None
             ),
             modifiers=self.modifiers,
-            pseudonyms={
-                k: v.with_namespace(namespace) for k, v in self.pseudonyms.items()
-            },
+            pseudonyms={address_with_namespace(v, namespace) for v in self.pseudonyms},
         )
 
     def with_select_context(
@@ -862,7 +874,7 @@ class Grain(Mergeable, BaseModel):
             )
         else:
             v2 = unique(v, "address")
-        final = []
+        final: List[Concept] = []
         for sub in v2:
             if sub.purpose in (Purpose.PROPERTY, Purpose.METRIC) and sub.keys:
                 if all([c in v2 for c in sub.keys]):
@@ -916,6 +928,20 @@ class Grain(Mergeable, BaseModel):
             [c.name == ALL_ROWS_CONCEPT for c in self.components]
         )
 
+    @property
+    def synonym_set(self) -> set[str]:
+        base = []
+        for x in self.components_copy:
+            if isinstance(x.lineage, RowsetItem):
+                base.append(x.lineage.content.address)
+                for c in x.lineage.content.pseudonyms:
+                    base.append(c)
+            else:
+                base.append(x.address)
+                for c in x.pseudonyms:
+                    base.append(c)
+        return set(base)
+
     @cached_property
     def set(self) -> set[str]:
         base = []
@@ -931,7 +957,11 @@ class Grain(Mergeable, BaseModel):
             return self.set == set([c.address for c in other])
         if not isinstance(other, Grain):
             return False
-        return self.set == other.set
+        if self.set == other.set:
+            return True
+        elif self.synonym_set == other.synonym_set:
+            return True
+        return False
 
     def issubset(self, other: "Grain"):
         return self.set.issubset(other.set)
@@ -1584,14 +1614,7 @@ class RawSQLStatement(BaseModel):
     meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
 
-class CopyStatement(BaseModel):
-    target: str
-    target_type: IOType
-    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
-    select: SelectStatement
-
-
-class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
+class SelectStatement(HasUUID, Mergeable, Namespaced, SelectTypeMixin, BaseModel):
     selection: List[SelectItem]
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
@@ -1709,19 +1732,34 @@ class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
             # if the concept is a locally derived concept, it cannot ever be partial
             # but if it's a concept pulled in from upstream and we have a where clause, it should be partial
             ColumnAssignment(
-                alias=c.address.replace(".", "_"),
+                alias=(
+                    c.name.replace(".", "_")
+                    if c.namespace == DEFAULT_NAMESPACE
+                    else c.address.replace(".", "_")
+                ),
                 concept=c,
                 modifiers=modifiers if c.address not in self.locally_derived else [],
             )
             for c in self.output_components
         ]
 
+        condition = None
+        if self.where_clause:
+            condition = self.where_clause.conditional
+        if self.having_clause:
+            if condition:
+                condition = self.having_clause.conditional + condition
+            else:
+                condition = self.having_clause.conditional
+
         new_datasource = Datasource(
             identifier=identifier,
             address=address,
             grain=grain or self.grain,
             columns=columns,
             namespace=namespace,
+            where=WhereClause(conditional=condition) if condition else None,
+            non_partial_for=WhereClause(conditional=condition) if condition else None,
         )
         for column in columns:
             column.concept = column.concept.with_grain(new_datasource.grain)
@@ -1789,6 +1827,16 @@ class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
         )
 
 
+class CopyStatement(BaseModel):
+    target: str
+    target_type: IOType
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
+    select: SelectStatement
+
+    def refresh_bindings(self, environment: Environment):
+        self.select.refresh_bindings(environment)
+
+
 class AlignItem(Namespaced, BaseModel):
     alias: str
     concepts: List[Concept]
@@ -1834,7 +1882,7 @@ class AlignClause(Namespaced, BaseModel):
         return AlignClause(items=[x.with_namespace(namespace) for x in self.items])
 
 
-class MultiSelectStatement(SelectTypeMixin, Mergeable, Namespaced, BaseModel):
+class MultiSelectStatement(HasUUID, SelectTypeMixin, Mergeable, Namespaced, BaseModel):
     selects: List[SelectStatement]
     align: AlignClause
     namespace: str
@@ -1996,7 +2044,7 @@ class DatasourceMetadata(BaseModel):
     line_no: int | None = None
 
 
-class MergeStatementV2(Namespaced, BaseModel):
+class MergeStatementV2(HasUUID, Namespaced, BaseModel):
     source: Concept
     target: Concept
     modifiers: List[Modifier] = Field(default_factory=list)
@@ -2010,7 +2058,7 @@ class MergeStatementV2(Namespaced, BaseModel):
         return new
 
 
-class Datasource(Namespaced, BaseModel):
+class Datasource(HasUUID, Namespaced, BaseModel):
     identifier: str
     columns: List[ColumnAssignment]
     address: Union[Address, str]
@@ -2022,6 +2070,7 @@ class Datasource(Namespaced, BaseModel):
         default_factory=lambda: DatasourceMetadata(freshness_concept=None)
     )
     where: Optional[WhereClause] = None
+    non_partial_for: Optional[WhereClause] = None
 
     def merge_concept(
         self, source: Concept, target: Concept, modifiers: List[Modifier]
@@ -2222,6 +2271,7 @@ class InstantiatedUnnestJoin(BaseModel):
 class ConceptPair(BaseModel):
     left: Concept
     right: Concept
+    existing_datasource: Union[Datasource, "QueryDatasource"]
     modifiers: List[Modifier] = Field(default_factory=list)
 
     @property
@@ -2233,17 +2283,23 @@ class ConceptPair(BaseModel):
         return Modifier.NULLABLE in self.modifiers
 
 
+class CTEConceptPair(ConceptPair):
+    cte: CTE
+
+
 class BaseJoin(BaseModel):
-    left_datasource: Union[Datasource, "QueryDatasource"]
     right_datasource: Union[Datasource, "QueryDatasource"]
-    concepts: List[Concept]
     join_type: JoinType
-    filter_to_mutual: bool = False
+    concepts: Optional[List[Concept]] = None
+    left_datasource: Optional[Union[Datasource, "QueryDatasource"]] = None
     concept_pairs: list[ConceptPair] | None = None
 
     def __init__(self, **data: Any):
         super().__init__(**data)
-        if self.left_datasource.full_name == self.right_datasource.full_name:
+        if (
+            self.left_datasource
+            and self.left_datasource.full_name == self.right_datasource.full_name
+        ):
             raise SyntaxError(
                 f"Cannot join a dataself to itself, joining {self.left_datasource} and"
                 f" {self.right_datasource}"
@@ -2253,24 +2309,23 @@ class BaseJoin(BaseModel):
         # if we have a list of concept pairs
         if self.concept_pairs:
             return
-
-        for concept in self.concepts:
+        if self.concepts == []:
+            return
+        assert self.left_datasource and self.right_datasource
+        for concept in self.concepts or []:
             include = True
             for ds in [self.left_datasource, self.right_datasource]:
                 synonyms = []
                 for c in ds.output_concepts:
-                    synonyms += list(c.pseudonyms.keys())
+                    synonyms += list(c.pseudonyms)
                 if (
                     concept.address not in [c.address for c in ds.output_concepts]
                     and concept.address not in synonyms
                 ):
-                    if self.filter_to_mutual:
-                        include = False
-                    else:
-                        raise SyntaxError(
-                            f"Invalid join, missing {concept} on {ds.name}, have"
-                            f" {[c.address for c in ds.output_concepts]}"
-                        )
+                    raise SyntaxError(
+                        f"Invalid join, missing {concept} on {ds.name}, have"
+                        f" {[c.address for c in ds.output_concepts]}"
+                    )
             if include:
                 final_concepts.append(concept)
         if not final_concepts and self.concepts:
@@ -2287,7 +2342,7 @@ class BaseJoin(BaseModel):
                     self.concepts = []
                     return
                 # if everything is at abstract grain, we can skip joins
-                if all([c.grain == Grain() for c in ds.output_concepts]):
+                if all([c.grain.abstract for c in ds.output_concepts]):
                     self.concepts = []
                     return
 
@@ -2305,21 +2360,27 @@ class BaseJoin(BaseModel):
 
     @property
     def unique_id(self) -> str:
-        # TODO: include join type?
-        return (
-            self.left_datasource.name
-            + self.right_datasource.name
-            + self.join_type.value
-        )
+        return str(self)
+
+    @property
+    def input_concepts(self) -> List[Concept]:
+        base = []
+        if self.concept_pairs:
+            for pair in self.concept_pairs:
+                base += [pair.left, pair.right]
+        elif self.concepts:
+            base += self.concepts
+        return base
 
     def __str__(self):
         if self.concept_pairs:
             return (
-                f"{self.join_type.value} on"
-                f" {','.join([str(k.left)+'='+str(k.right) for k in self.concept_pairs])}"
+                f"{self.join_type.value} {self.right_datasource.name} on"
+                f" {','.join([str(k.existing_datasource.name) + '.'+ str(k.left)+'='+str(k.right) for k in self.concept_pairs])}"
             )
         return (
-            f"{self.join_type.value} on" f" {','.join([str(k) for k in self.concepts])}"
+            f"{self.join_type.value} {self.right_datasource.name} on"
+            f" {','.join([str(k) for k in self.concepts])}"
         )
 
 
@@ -2364,19 +2425,9 @@ class QueryDatasource(BaseModel):
         for join in v:
             if not isinstance(join, BaseJoin):
                 continue
-            if join.left_datasource.identifier == join.right_datasource.identifier:
-                raise SyntaxError(
-                    f"Cannot join a datasource to itself, joining {join.left_datasource}"
-                )
-            pairing = "".join(
-                sorted(
-                    [join.left_datasource.identifier, join.right_datasource.identifier]
-                )
-            )
+            pairing = str(join)
             if pairing in unique_pairs:
-                raise SyntaxError(
-                    f"Duplicate join {join.left_datasource.identifier} and {join.right_datasource.identifier}"
-                )
+                raise SyntaxError(f"Duplicate join {str(join)}")
             unique_pairs.add(pairing)
         return v
 
@@ -2641,7 +2692,12 @@ class CTE(BaseModel):
                         isinstance(join, Join)
                         and (
                             join.right_cte.name != removed_cte
-                            and join.left_cte.name != removed_cte
+                            and any(
+                                [
+                                    x.cte.name != removed_cte
+                                    for x in (join.joinkey_pairs or [])
+                                ]
+                            )
                         )
                     )
                 ]
@@ -2712,8 +2768,12 @@ class CTE(BaseModel):
         for join in self.joins:
             if isinstance(join, InstantiatedUnnestJoin):
                 continue
-            if join.left_cte.name == parent.name:
+            if join.left_cte and join.left_cte.name == parent.name:
                 join.inline_cte(parent)
+            if join.joinkey_pairs:
+                for pair in join.joinkey_pairs:
+                    if pair.cte and pair.cte.name == parent.name:
+                        join.inline_cte(parent)
             if join.right_cte.name == parent.name:
                 join.inline_cte(parent)
         for k, v in self.source_map.items():
@@ -2834,9 +2894,21 @@ class CTE(BaseModel):
             return self.parent_ctes[0].name
         return self.name
 
+    def get_concept(self, address: str) -> Concept | None:
+        for cte in self.parent_ctes:
+            if address in cte.output_columns:
+                match = [x for x in cte.output_columns if x.address == address].pop()
+                return match
+
+        for array in [self.source.input_concepts, self.source.output_concepts]:
+            match_list = [x for x in array if x.address == address]
+            if match_list:
+                return match_list.pop()
+        return None
+
     def get_alias(self, concept: Concept, source: str | None = None) -> str:
         for cte in self.parent_ctes:
-            if concept.address in [x.address for x in cte.output_columns]:
+            if concept.address in cte.output_columns:
                 if source and source != cte.name:
                     continue
                 return concept.safe_address
@@ -2924,21 +2996,26 @@ class JoinKey(BaseModel):
 
 
 class Join(BaseModel):
-    left_cte: CTE
+
     right_cte: CTE
     jointype: JoinType
-    joinkeys: List[JoinKey]
-    joinkey_pairs: List[ConceptPair] | None = None
+    left_cte: CTE | None = None
+    joinkey_pairs: List[CTEConceptPair] | None = None
     inlined_ctes: set[str] = Field(default_factory=set)
 
     def inline_cte(self, cte: CTE):
         self.inlined_ctes.add(cte.name)
 
-    @property
-    def left_name(self) -> str:
-        if self.left_cte.name in self.inlined_ctes:
-            return self.left_cte.source.datasources[0].identifier
-        return self.left_cte.name
+    # @property
+    # def left_name(self) -> str:
+    #     if self.left_cte.name in self.inlined_ctes:
+    #         return self.left_cte.source.datasources[0].identifier
+    #     return self.left_cte.name
+
+    def get_name(self, cte: CTE):
+        if cte.name in self.inlined_ctes:
+            return cte.source.datasources[0].identifier
+        return cte.name
 
     @property
     def right_name(self) -> str:
@@ -2946,12 +3023,6 @@ class Join(BaseModel):
             return self.right_cte.source.datasources[0].identifier
         return self.right_cte.name
 
-    @property
-    def left_ref(self) -> str:
-        if self.left_cte.name in self.inlined_ctes:
-            return f"{self.left_cte.source.datasources[0].safe_location} as {self.left_cte.source.datasources[0].identifier}"
-        return self.left_cte.name
-
     @property
     def right_ref(self) -> str:
         if self.right_cte.name in self.inlined_ctes:
@@ -2960,19 +3031,21 @@ class Join(BaseModel):
 
     @property
     def unique_id(self) -> str:
-        return self.left_name + self.right_name + self.jointype.value
+        return str(self)
 
     def __str__(self):
         if self.joinkey_pairs:
             return (
-                f"{self.jointype.value} JOIN {self.left_name} and"
+                f"{self.jointype.value} join"
                 f" {self.right_name} on"
-                f" {','.join([str(k.left)+'='+str(k.right)+str(k.modifiers) for k in self.joinkey_pairs])}"
+                f" {','.join([k.cte.name + '.'+str(k.left.address)+'='+str(k.right.address) for k in self.joinkey_pairs])}"
             )
-        return (
-            f"{self.jointype.value} JOIN {self.left_name} and"
-            f" {self.right_name} on {','.join([str(k) for k in self.joinkeys])}"
-        )
+        elif self.left_cte:
+            return (
+                f"{self.jointype.value} JOIN {self.left_cte.name} and"
+                f" {self.right_name} on {','.join([str(k) for k in self.joinkey_pairs])}"
+            )
+        return f"{self.jointype.value} JOIN  {self.right_name} on {','.join([str(k) for k in self.joinkey_pairs])}"
 
 
 class UndefinedConcept(Concept, Mergeable, Namespaced):
@@ -2988,7 +3061,7 @@ class UndefinedConcept(Concept, Mergeable, Namespaced):
     ) -> "UndefinedConcept" | Concept:
         if self.address == source.address:
             new = target.with_grain(self.grain.with_merge(source, target, modifiers))
-            new.pseudonyms[self.address] = self
+            new.pseudonyms.add(self.address)
             return new
         return self.__class__(
             name=self.name,
@@ -3190,7 +3263,7 @@ class EnvironmentConceptDict(dict):
         return super().items()
 
 
-class ImportStatement(BaseModel):
+class ImportStatement(HasUUID, BaseModel):
     alias: str
     path: Path
     environment: Union["Environment", None] = None
@@ -3520,6 +3593,7 @@ class Environment(BaseModel):
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ):
         replacements = {}
+
         # exit early if we've run this
         if source.address in self.alias_origin_lookup:
             if self.concepts[source.address] == target:
@@ -3528,11 +3602,11 @@ class Environment(BaseModel):
         for k, v in self.concepts.items():
 
             if v.address == target.address:
-                v.pseudonyms[source.address] = source
+                v.pseudonyms.add(source.address)
             if v.address == source.address:
                 replacements[k] = target
                 self.canonical_map[k] = target.address
-                v.pseudonyms[target.address] = target
+                v.pseudonyms.add(target.address)
             # we need to update keys and grains of all concepts
             else:
                 replacements[k] = v.with_merge(source, target, modifiers)
@@ -4185,6 +4259,9 @@ class AggregateWrapper(Mergeable, Namespaced, SelectContext, BaseModel):
 class WhereClause(Mergeable, ConceptArgs, Namespaced, SelectContext, BaseModel):
     conditional: Union[SubselectComparison, Comparison, Conditional, "Parenthetical"]
 
+    def __repr__(self):
+        return str(self.conditional)
+
     @property
     def input(self) -> List[Concept]:
         return self.conditional.input
@@ -4303,7 +4380,7 @@ class Limit(BaseModel):
     count: int
 
 
-class ConceptDeclarationStatement(BaseModel):
+class ConceptDeclarationStatement(HasUUID, BaseModel):
     concept: Concept
 
 
@@ -4311,7 +4388,7 @@ class ConceptDerivation(BaseModel):
     concept: Concept
 
 
-class RowsetDerivationStatement(Namespaced, BaseModel):
+class RowsetDerivationStatement(HasUUID, Namespaced, BaseModel):
     name: str
     select: SelectStatement | MultiSelectStatement
     namespace: str
@@ -4576,7 +4653,7 @@ class TupleWrapper(Generic[VT], tuple):
         return cls(v, type=arg_to_datatype(v[0]))
 
 
-class PersistStatement(BaseModel):
+class PersistStatement(HasUUID, BaseModel):
     datasource: Datasource
     select: SelectStatement
     meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())

trilogy/core/processing/concept_strategies_v3.py

@@ -455,7 +455,20 @@ def generate_node(
                         if x.address not in [y.address for y in root_targets]
                         and x not in ex_resolve.grain.components
                     ]
-                    expanded.set_output_concepts(root_targets)
+
+                    pseudonyms = [
+                        x
+                        for x in extra
+                        if any(x.address in y.pseudonyms for y in root_targets)
+                    ]
+                    # if we're only connected by a pseudonym, keep those in output
+                    expanded.set_output_concepts(root_targets + pseudonyms)
+                    # but hide them
+                    if pseudonyms:
+                        logger.info(
+                            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Hiding pseudonyms{[c.address for c in pseudonyms]}"
+                        )
+                        expanded.hide_output_concepts(pseudonyms)
 
                     logger.info(
                         f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in root_targets]} via concept addition; removing extra {[c.address for c in extra]}"
@@ -480,6 +493,7 @@ def validate_concept(
     found_map: dict[str, set[Concept]],
     accept_partial: bool,
     seen: set[str],
+    environment: Environment,
 ):
 
     found_map[str(node)].add(concept)
@@ -500,10 +514,11 @@ def validate_concept(
         if accept_partial:
             found_addresses.add(concept.address)
             found_map[str(node)].add(concept)
-    for _, v in concept.pseudonyms.items():
-        if v.address == concept.address:
+    for v_address in concept.pseudonyms:
+        v = environment.concepts[v_address]
+        if v == concept.address:
             return
-        if v.address in seen:
+        if v in seen:
             return
         validate_concept(
             v,
@@ -515,10 +530,12 @@ def validate_concept(
             found_map,
             accept_partial,
             seen=seen,
+            environment=environment,
         )
 
 
 def validate_stack(
+    environment: Environment,
     stack: List[StrategyNode],
     concepts: List[Concept],
     mandatory_with_filter: List[Concept],
@@ -546,6 +563,7 @@ def validate_stack(
                 found_map,
                 accept_partial,
                 seen,
+                environment,
             )
         for concept in node.virtual_output_concepts:
             if concept.address in non_partial_addresses:
@@ -807,6 +825,7 @@ def _search_concepts(
                 break
         attempted.add(priority_concept.address)
         complete, found, missing, partial, virtual = validate_stack(
+            environment,
             stack,
             mandatory_list,
             completion_mandatory,

trilogy/core/processing/node_generators/common.py

@@ -1,6 +1,5 @@
 from typing import List, Tuple, Callable
 
-
 from trilogy.core.enums import PurposeLineage, Purpose
 from trilogy.core.models import (
     Concept,

trilogy/core/processing/node_generators/node_merge_node.py

@@ -209,9 +209,9 @@ def resolve_weak_components(
         for c in all_concepts
         if "__preql_internal" not in c.address
     ]
-    synonyms: list[Concept] = []
+    synonyms: set[str] = set()
     for x in all_concepts:
-        synonyms += x.pseudonyms.values()
+        synonyms = synonyms.union(x.pseudonyms)
     while break_flag is not True:
         count += 1
         if count > AMBIGUITY_CHECK_LIMIT:
@@ -385,9 +385,9 @@ def gen_merge_node(
     # one concept handling may need to be kicked to alias
     if len(all_concepts) == 1:
         concept = all_concepts[0]
-        for k, v in concept.pseudonyms.items():
+        for v in concept.pseudonyms:
             test = subgraphs_to_merge_node(
-                [[concept, v]],
+                [[concept, environment.alias_origin_lookup[v]]],
                 g=g,
                 all_concepts=[concept],
                 environment=environment,