pytrilogy 0.0.3.93__py3-none-any.whl → 0.0.3.95__py3-none-any.whl

trilogy/core/env_processor.py

@@ -20,7 +20,8 @@ def add_concept(
     if node_name in seen:
         return
     seen.add(node_name)
-    g.add_node(concept)
+    g.concepts[node_name] = concept
+    g.add_node(node_name)
     if concept.concept_arguments:
         for source in concept.concept_arguments:
             if not isinstance(source, BuildConcept):
@@ -28,9 +29,10 @@ def add_concept(
                     f"Invalid non-build concept {source} passed into graph generation from {concept}"
                 )
             generic = get_default_grain_concept(source, default_concept_graph)
+            generic_node = concept_to_node(generic)
             add_concept(generic, g, concept_mapping, default_concept_graph, seen)
 
-            g.add_edge(generic, node_name)
+            g.add_edge(generic_node, node_name, fast=True)
     for ps_address in concept.pseudonyms:
         if ps_address not in concept_mapping:
             raise SyntaxError(f"Concept {concept} has invalid pseudonym {ps_address}")
@@ -44,8 +46,10 @@ def add_concept(
             continue
         if pseudonym_node.split("@")[0] == node_name.split("@")[0]:
             continue
-        g.add_edge(pseudonym_node, node_name, pseudonym=True)
-        g.add_edge(node_name, pseudonym_node, pseudonym=True)
+        g.add_edge(pseudonym_node, node_name, fast=True)
+        g.add_edge(node_name, pseudonym_node, fast=True)
+        g.pseudonyms.add((pseudonym_node, node_name))
+        g.pseudonyms.add((node_name, pseudonym_node))
         add_concept(pseudonym, g, concept_mapping, default_concept_graph, seen)
 
 
@@ -80,20 +84,27 @@ def generate_adhoc_graph(
 
     for dataset in datasources:
         node = datasource_to_node(dataset)
-        g.add_node(dataset, type="datasource", datasource=dataset)
+        g.add_datasource_node(node, dataset)
         for concept in dataset.concepts:
+            cnode = concept_to_node(concept)
+            g.concepts[cnode] = concept
+            g.add_node(cnode)
             if restrict_to_listed:
-                if concept_to_node(concept) not in g.nodes:
+                if cnode not in g.nodes:
                     continue
-            g.add_edge(node, concept)
-            g.add_edge(concept, node)
+            g.add_edge(node, cnode, fast=True)
+            g.add_edge(cnode, node, fast=True)
             # if there is a key on a table at a different grain
             # add an FK edge to the canonical source, if it exists
             # for example, order ID on order product table
             default = get_default_grain_concept(concept, default_concept_graph)
+
             if concept != default:
-                g.add_edge(concept, default)
-                g.add_edge(default, concept)
+                dcnode = concept_to_node(default)
+                g.concepts[dcnode] = default
+                g.add_node(dcnode)
+                g.add_edge(cnode, dcnode, fast=True)
+                g.add_edge(dcnode, cnode, fast=True)
     return g
 
 

trilogy/core/environment_helpers.py

@@ -169,6 +169,112 @@ def generate_key_concepts(concept: Concept, environment: Environment):
         environment.add_concept(new_concept, add_derived=False)
 
 
+def remove_date_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated date-related concepts for the given concept"""
+    date_suffixes = ["month", "year", "quarter", "day", "day_of_week"]
+    grain_suffixes = ["month_start", "year_start"]
+
+    for suffix in date_suffixes + grain_suffixes:
+        address = concept.address + f".{suffix}"
+        if address in environment.concepts:
+            derived_concept = environment.concepts[address]
+            # Only remove if it was auto-derived from this concept
+            if (
+                derived_concept.metadata
+                and derived_concept.metadata.concept_source
+                == ConceptSource.AUTO_DERIVED
+                and derived_concept.keys
+                and concept.address in derived_concept.keys
+            ):
+                environment.remove_concept(address)
+
+
+def remove_datetime_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated datetime-related concepts for the given concept"""
+    datetime_suffixes = ["date", "hour", "minute", "second"]
+
+    for suffix in datetime_suffixes:
+        address = concept.address + f".{suffix}"
+        if address in environment.concepts:
+            derived_concept = environment.concepts[address]
+            # Only remove if it was auto-derived from this concept
+            if (
+                derived_concept.metadata
+                and derived_concept.metadata.concept_source
+                == ConceptSource.AUTO_DERIVED
+                and derived_concept.keys
+                and concept.address in derived_concept.keys
+            ):
+                environment.remove_concept(address)
+
+
+def remove_key_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated key-related concepts for the given concept"""
+    key_suffixes = ["count"]
+
+    for suffix in key_suffixes:
+        address = concept.address + f".{suffix}"
+        if address in environment.concepts:
+            derived_concept = environment.concepts[address]
+            if (
+                derived_concept.metadata
+                and derived_concept.metadata.concept_source
+                == ConceptSource.AUTO_DERIVED
+            ):
+                environment.remove_concept(address)
+
+
+def remove_struct_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated struct field concepts for the given concept"""
+    if not isinstance(concept.datatype, StructType):
+        return
+
+    target_namespace = (
+        environment.namespace + "." + concept.name
+        if environment.namespace and environment.namespace != DEFAULT_NAMESPACE
+        else concept.name
+    )
+
+    # Get all concepts in the target namespace that were auto-derived
+    concepts_to_remove = []
+    for address, derived_concept in environment.concepts.items():
+        if (
+            derived_concept.namespace == target_namespace
+            and derived_concept.metadata
+            and derived_concept.metadata.concept_source == ConceptSource.AUTO_DERIVED
+            and isinstance(derived_concept.lineage, Function)
+            and derived_concept.lineage.operator == FunctionType.ATTR_ACCESS
+            and len(derived_concept.lineage.arguments) >= 1
+            and derived_concept.lineage.arguments[0] == concept.reference
+        ):
+            concepts_to_remove.append(address)
+
+    for address in concepts_to_remove:
+        environment.remove_concept(address)
+
+
+def remove_related_concepts(concept: Concept, environment: Environment):
+    """Remove all auto-generated concepts that were derived from the given concept"""
+
+    # Remove key-related concepts
+    if concept.purpose == Purpose.KEY:
+        remove_key_concepts(concept, environment)
+
+    # Remove datatype-specific concepts
+    if concept.datatype == DataType.DATE:
+        remove_date_concepts(concept, environment)
+    elif concept.datatype == DataType.DATETIME:
+        remove_date_concepts(concept, environment)
+        remove_datetime_concepts(concept, environment)
+    elif concept.datatype == DataType.TIMESTAMP:
+        remove_date_concepts(concept, environment)
+        remove_datetime_concepts(concept, environment)
+
+    # Remove struct field concepts
+    if isinstance(concept.datatype, StructType):
+        remove_struct_concepts(concept, environment)
+
+
 def generate_related_concepts(
     concept: Concept,
     environment: Environment,
@@ -183,6 +289,7 @@ def generate_related_concepts(
     if concept.datatype == DataType.DATE and add_derived:
         generate_date_concepts(concept, environment)
     elif concept.datatype == DataType.DATETIME and add_derived:
+
         generate_date_concepts(concept, environment)
         generate_datetime_concepts(concept, environment)
     elif concept.datatype == DataType.TIMESTAMP and add_derived:
@@ -203,6 +310,10 @@ def generate_related_concepts(
                 ),
                 lineage=AttrAccess([concept.reference, key], environment=environment),
                 grain=concept.grain,
+                metadata=Metadata(
+                    concept_source=ConceptSource.AUTO_DERIVED,
+                ),
+                keys=concept.keys,
             )
             environment.add_concept(auto, meta=meta)
             if isinstance(value, Concept):

trilogy/core/exceptions.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Sequence
 
 
 class UndefinedConceptException(Exception):
@@ -24,6 +24,26 @@ class NoDatasourceException(UnresolvableQueryException):
     pass
 
 
+class ModelValidationError(Exception):
+    def __init__(
+        self,
+        message,
+        children: Sequence["ModelValidationError"] | None = None,
+        **kwargs
+    ):
+        super().__init__(self, message, **kwargs)
+        self.message = message
+        self.children = children
+
+
+class DatasourceModelValidationError(ModelValidationError):
+    pass
+
+
+class ConceptModelValidationError(ModelValidationError):
+    pass
+
+
 class AmbiguousRelationshipResolutionException(UnresolvableQueryException):
     def __init__(self, message, parents: List[set[str]]):
         super().__init__(self, message)

trilogy/core/functions.py

@@ -380,7 +380,12 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
     ),
     FunctionType.CURRENT_DATETIME: FunctionConfig(
         output_purpose=Purpose.CONSTANT,
-        output_type=DataType.DATE,
+        output_type=DataType.DATETIME,
+        arg_count=0,
+    ),
+    FunctionType.CURRENT_TIMESTAMP: FunctionConfig(
+        output_purpose=Purpose.CONSTANT,
+        output_type=DataType.TIMESTAMP,
         arg_count=0,
     ),
     FunctionType.BOOL: FunctionConfig(

trilogy/core/graph_models.py

@@ -1,50 +1,48 @@
+from typing import Union
+
 import networkx as nx
 
 from trilogy.core.models.build import BuildConcept, BuildDatasource, BuildWhereClause
 
 
 def get_graph_exact_match(
-    g: nx.DiGraph, accept_partial: bool, conditions: BuildWhereClause | None
+    g: Union[nx.DiGraph, "ReferenceGraph"],
+    accept_partial: bool,
+    conditions: BuildWhereClause | None,
 ) -> set[str]:
-    datasources: dict[str, BuildDatasource | list[BuildDatasource]] = (
-        nx.get_node_attributes(g, "datasource")
-    )
     exact: set[str] = set()
-    for node in g.nodes:
-        if node in datasources:
-            ds = datasources[node]
-            if isinstance(ds, list):
-                exact.add(node)
-                continue
-
-            if not conditions and not ds.non_partial_for:
-                exact.add(node)
+    for node, ds in g.datasources.items():
+        if isinstance(ds, list):
+            exact.add(node)
+            continue
+
+        if not conditions and not ds.non_partial_for:
+            exact.add(node)
+            continue
+        elif not conditions and accept_partial and ds.non_partial_for:
+            exact.add(node)
+            continue
+        elif conditions:
+            if not ds.non_partial_for:
                 continue
-            elif not conditions and accept_partial and ds.non_partial_for:
+            if ds.non_partial_for and conditions == ds.non_partial_for:
                 exact.add(node)
                 continue
-            elif conditions:
-                if not ds.non_partial_for:
-                    continue
-                if ds.non_partial_for and conditions == ds.non_partial_for:
-                    exact.add(node)
-                    continue
-            else:
-                continue
+        else:
+            continue
 
     return exact
 
 
 def prune_sources_for_conditions(
-    g: nx.DiGraph,
+    g: "ReferenceGraph",
     accept_partial: bool,
     conditions: BuildWhereClause | None,
 ):
-
     complete = get_graph_exact_match(g, accept_partial, conditions)
     to_remove = []
-    for node in g.nodes:
-        if node.startswith("ds~") and node not in complete:
+    for node in g.datasources:
+        if node not in complete:
             to_remove.append(node)
 
     for node in to_remove:
@@ -68,46 +66,41 @@ def datasource_to_node(input: BuildDatasource) -> str:
 class ReferenceGraph(nx.DiGraph):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-
-    def add_node(self, node_for_adding, **attr):
-        if isinstance(node_for_adding, BuildConcept):
-            node_name = concept_to_node(node_for_adding)
-            # if node_name in self.nodes:
-            #     return
-            attr["type"] = "concept"
-            attr["concept"] = node_for_adding
-            attr["grain"] = node_for_adding.grain
-        elif isinstance(node_for_adding, BuildDatasource):
-            node_name = datasource_to_node(node_for_adding)
-            # if node_name in self.nodes:
-            #     return
-            attr["type"] = "datasource"
-            attr["ds"] = node_for_adding
-            attr["grain"] = node_for_adding.grain
-        else:
-            node_name = node_for_adding
+        self.concepts: dict[str, BuildConcept] = {}
+        self.datasources: dict[str, BuildDatasource] = {}
+        self.pseudonyms: set[tuple[str, str]] = set()
+
+    def copy(self):
+        g = ReferenceGraph()
+        g.concepts = self.concepts.copy()
+        g.datasources = self.datasources.copy()
+        g.pseudonyms = {*self.pseudonyms}
+        # g.add_nodes_from(self.nodes(data=True))
+        for node in self.nodes:
+            g.add_node(node, fast=True)
+        for edge in self.edges:
+            g.add_edge(edge[0], edge[1], fast=True)
+        # g.add_edges_from(self.edges(data=True))
+        return g
+
+    def remove_node(self, n):
+        if n in self.concepts:
+            del self.concepts[n]
+        if n in self.datasources:
+            del self.datasources[n]
+        super().remove_node(n)
+
+    def add_node(self, node_for_adding, fast: bool = False, **attr):
+        if fast:
+            return super().add_node(node_for_adding, **attr)
+        node_name = node_for_adding
+        if attr.get("datasource"):
+            self.datasources[node_name] = attr["datasource"]
         super().add_node(node_name, **attr)
 
-    def add_edge(self, u_of_edge, v_of_edge, **attr):
-        if isinstance(u_of_edge, BuildConcept):
-            orig = u_of_edge
-            u_of_edge = concept_to_node(u_of_edge)
-            if u_of_edge not in self.nodes:
-                self.add_node(orig)
-        elif isinstance(u_of_edge, BuildDatasource):
-            orig = u_of_edge
-            u_of_edge = datasource_to_node(u_of_edge)
-            if u_of_edge not in self.nodes:
-                self.add_node(orig)
-
-        if isinstance(v_of_edge, BuildConcept):
-            orig = v_of_edge
-            v_of_edge = concept_to_node(v_of_edge)
-            if v_of_edge not in self.nodes:
-                self.add_node(orig)
-        elif isinstance(v_of_edge, BuildDatasource):
-            orig = v_of_edge
-            v_of_edge = datasource_to_node(v_of_edge)
-            if v_of_edge not in self.nodes:
-                self.add_node(orig)
-        super().add_edge(u_of_edge, v_of_edge, **attr)
+    def add_datasource_node(self, node_name, datasource):
+        self.datasources[node_name] = datasource
+        super().add_node(node_name, datasource=datasource)
+
+    def add_edge(self, u_of_edge, v_of_edge, fast: bool = False, **attr):
+        return super().add_edge(u_of_edge, v_of_edge, **attr)

trilogy/core/internal.py

@@ -64,4 +64,22 @@ DEFAULT_CONCEPTS = {
         granularity=Granularity.SINGLE_ROW,
         derivation=Derivation.CONSTANT,
     ),
+    "label": Concept(
+        name="label",
+        namespace=INTERNAL_NAMESPACE,
+        datatype=DataType.STRING,
+        purpose=Purpose.KEY,
+        grain=Grain(),
+        granularity=Granularity.SINGLE_ROW,
+        derivation=Derivation.CONSTANT,
+    ),
+    "expected": Concept(
+        name="expected_value",
+        namespace=INTERNAL_NAMESPACE,
+        datatype=DataType.STRING,
+        purpose=Purpose.KEY,
+        grain=Grain(),
+        granularity=Granularity.SINGLE_ROW,
+        derivation=Derivation.CONSTANT,
+    ),
 }

trilogy/core/models/author.py

@@ -902,7 +902,11 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
 
     @property
     def is_aggregate(self):
-        return self.calculate_is_aggregate(self.lineage)
+        base = getattr(self, "_is_aggregate", None)
+        if base:
+            return base
+        setattr(self, "_is_aggregate", self.calculate_is_aggregate(self.lineage))
+        return self._is_aggregate
 
     def with_merge(self, source: Self, target: Self, modifiers: List[Modifier]) -> Self:
         if self.address == source.address:
@@ -1069,18 +1073,25 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
         final_grain = grain if not self.grain.components else self.grain
         keys = self.keys
 
-        if self.is_aggregate and isinstance(new_lineage, Function) and grain.components:
+        if self.is_aggregate and grain.components and isinstance(new_lineage, Function):
             grain_components: list[ConceptRef | Concept] = [
                 environment.concepts[c].reference for c in grain.components
             ]
-            new_lineage = AggregateWrapper(function=new_lineage, by=grain_components)
+            new_lineage = AggregateWrapper.model_construct(
+                function=new_lineage, by=grain_components
+            )
             final_grain = grain
             keys = set(grain.components)
-        elif isinstance(new_lineage, AggregateWrapper) and not new_lineage.by and grain:
+        elif (
+            grain
+            and new_lineage
+            and isinstance(new_lineage, AggregateWrapper)
+            and not new_lineage.by
+        ):
             grain_components = [
                 environment.concepts[c].reference for c in grain.components
             ]
-            new_lineage = AggregateWrapper(
+            new_lineage = AggregateWrapper.model_construct(
                 function=new_lineage.function, by=grain_components
             )
             final_grain = grain
@@ -1670,15 +1681,6 @@ class Function(DataTyped, ConceptArgs, Mergeable, Namespaced, BaseModel):
     def datatype(self):
         return self.output_datatype
 
-    @field_validator("output_datatype")
-    @classmethod
-    def parse_output_datatype(cls, v, info: ValidationInfo):
-        values = info.data
-        if values.get("operator") == FunctionType.ATTR_ACCESS:
-            if isinstance(v, StructType):
-                raise SyntaxError
-        return v
-
     @field_validator("arguments", mode="before")
     @classmethod
     def parse_arguments(cls, v, info: ValidationInfo):
@@ -1845,17 +1847,6 @@ class Function(DataTyped, ConceptArgs, Mergeable, Namespaced, BaseModel):
             base += get_concept_arguments(arg)
         return base
 
-    @property
-    def output_grain(self):
-        # aggregates have an abstract grain
-        base_grain = Grain(components=[])
-        if self.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
-            return base_grain
-        # scalars have implicit grain of all arguments
-        for input in self.concept_arguments:
-            base_grain += input.grain
-        return base_grain
-
 
 class FunctionCallWrapper(
     DataTyped,

trilogy/core/models/build.py

@@ -1533,6 +1533,7 @@ class Factory:
         )
         self.local_non_build_concepts: dict[str, Concept] = {}
         self.pseudonym_map = pseudonym_map or get_canonical_pseudonyms(environment)
+        self.build_grain = self.build(self.grain) if self.grain else None
 
     def instantiate_concept(
         self,
@@ -1792,11 +1793,11 @@ class Factory:
         address = base.concept.address
         fetched = (
             self._build_concept(
-                self.environment.alias_origin_lookup[address].with_grain(self.grain)
-            )
+                self.environment.alias_origin_lookup[address]
+            ).with_grain(self.build_grain)
             if address in self.environment.alias_origin_lookup
-            else self._build_concept(
-                self.environment.concepts[address].with_grain(self.grain)
+            else self._build_concept(self.environment.concepts[address]).with_grain(
+                self.build_grain
             )
         )
 

trilogy/core/models/core.py

@@ -103,6 +103,9 @@ class DataType(Enum):
     def data_type(self):
         return self
 
+    def __str__(self) -> str:
+        return self.name
+
 
 class TraitDataType(BaseModel):
     type: DataType | NumericType | StructType | ArrayType | MapType

trilogy/core/models/environment.py

@@ -571,6 +571,34 @@ class Environment(BaseModel):
 
         return concept
 
+    def remove_concept(
+        self,
+        concept: Concept | str,
+    ) -> bool:
+        if self.frozen:
+            raise FrozenEnvironmentException(
+                "Environment is frozen, cannot remove concepts"
+            )
+        if isinstance(concept, Concept):
+            address = concept.address
+            c_instance = concept
+        else:
+            address = concept
+            c_instance_check = self.concepts.get(address)
+            if not c_instance_check:
+                return False
+            c_instance = c_instance_check
+        from trilogy.core.environment_helpers import remove_related_concepts
+
+        remove_related_concepts(c_instance, self)
+        if address in self.concepts:
+            del self.concepts[address]
+            return True
+        if address in self.alias_origin_lookup:
+            del self.alias_origin_lookup[address]
+
+        return False
+
     def add_datasource(
         self,
         datasource: Datasource,

trilogy/core/models/execute.py

@@ -23,6 +23,7 @@ from trilogy.core.constants import CONSTANT_DATASET
 from trilogy.core.enums import (
     ComparisonOperator,
     Derivation,
+    FunctionClass,
     FunctionType,
     JoinType,
     Modifier,
@@ -375,6 +376,12 @@ class CTE(BaseModel):
                 return check_is_not_in_group(c.lineage.content)
             if c.derivation == Derivation.CONSTANT:
                 return True
+            if (
+                c.purpose == Purpose.CONSTANT
+                and isinstance(c.lineage, BuildFunction)
+                and c.lineage.operator in FunctionClass.AGGREGATE_FUNCTIONS.value
+            ):
+                return True
             if c.purpose == Purpose.METRIC:
                 return True