pytrilogy 0.0.3.94__py3-none-any.whl → 0.0.3.96__py3-none-any.whl

trilogy/core/statements/execute.py

@@ -1,49 +1,106 @@
-from typing import Annotated, List, Optional, Union
+from dataclasses import dataclass, field
+from typing import List, Optional, Union
 
-from pydantic import BaseModel, Field
-from pydantic.functional_validators import PlainValidator
-
-from trilogy.core.models.author import ConceptRef
+from trilogy.core.enums import IOType, ValidationScope
+from trilogy.core.models.author import ConceptRef, HavingClause, WhereClause
 from trilogy.core.models.build import (
     BuildConcept,
     BuildDatasource,
     BuildOrderBy,
 )
-from trilogy.core.models.environment import EnvironmentConceptDict, validate_concepts
+from trilogy.core.models.datasource import Address, Datasource
+from trilogy.core.models.environment import EnvironmentConceptDict
 from trilogy.core.models.execute import CTE, UnionCTE
-from trilogy.core.statements.common import CopyQueryMixin, PersistQueryMixin
 
 
-class ProcessedQuery(BaseModel):
+@dataclass
+class CopyQueryMixin:
+    target: str
+    target_type: IOType
+
+
+@dataclass
+class MaterializedDataset:
+    address: Address
+
+
+@dataclass
+class PersistQueryMixin:
+    output_to: MaterializedDataset
+    datasource: Datasource
+
+
+@dataclass
+class SelectTypeMixin:
+    where_clause: Union["WhereClause", None] = field(default=None)
+    having_clause: Union["HavingClause", None] = field(default=None)
+
+    @property
+    def output_components(self) -> List[ConceptRef]:
+        raise NotImplementedError
+
+
+@dataclass
+class ProcessedQuery:
     output_columns: List[ConceptRef]
     ctes: List[CTE | UnionCTE]
     base: CTE | UnionCTE
-    hidden_columns: set[str] = Field(default_factory=set)
+    hidden_columns: set[str] = field(default_factory=set)
     limit: Optional[int] = None
     order_by: Optional[BuildOrderBy] = None
-    local_concepts: Annotated[
-        EnvironmentConceptDict, PlainValidator(validate_concepts)
-    ] = Field(default_factory=EnvironmentConceptDict)
+    local_concepts: EnvironmentConceptDict = field(
+        default_factory=EnvironmentConceptDict
+    )
+    locally_derived: set[str] = field(default_factory=set)
 
 
+@dataclass
 class ProcessedQueryPersist(ProcessedQuery, PersistQueryMixin):
     pass
 
 
+@dataclass
 class ProcessedCopyStatement(ProcessedQuery, CopyQueryMixin):
     pass
 
 
-class ProcessedRawSQLStatement(BaseModel):
+@dataclass
+class ProcessedRawSQLStatement:
     text: str
 
 
-class ProcessedStaticValueOutput(BaseModel):
+@dataclass
+class ProcessedValidateStatement:
+    scope: ValidationScope
+    targets: Optional[List[str]]
+
+
+@dataclass
+class ProcessedStaticValueOutput:
     values: List[dict]
 
 
-class ProcessedShowStatement(BaseModel):
+@dataclass
+class ProcessedShowStatement:
     output_columns: List[ConceptRef]
     output_values: List[
-        Union[BuildConcept, BuildDatasource, ProcessedQuery, ProcessedStaticValueOutput]
+        Union[
+            BuildConcept,
+            BuildDatasource,
+            ProcessedQuery,
+            ProcessedQueryPersist,
+            ProcessedCopyStatement,
+            ProcessedValidateStatement,
+            ProcessedStaticValueOutput,
+        ]
     ]
+
+
+PROCESSED_STATEMENT_TYPES = (
+    ProcessedCopyStatement
+    | ProcessedQuery
+    | ProcessedRawSQLStatement
+    | ProcessedQueryPersist
+    | ProcessedShowStatement
+    | ProcessedValidateStatement
+)

trilogy/core/validation/common.py

@@ -0,0 +1,110 @@
+from dataclasses import dataclass
+from enum import Enum
+
+from trilogy import Environment
+from trilogy.authoring import ConceptRef
+from trilogy.core.exceptions import ModelValidationError
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildConcept,
+    BuildConditional,
+    BuildDatasource,
+)
+from trilogy.core.models.environment import EnvironmentConceptDict
+from trilogy.core.models.execute import (
+    CTE,
+    QueryDatasource,
+)
+from trilogy.core.statements.execute import ProcessedQuery
+
+
+class ExpectationType(Enum):
+    LOGICAL = "logical"
+    ROWCOUNT = "rowcount"
+    DATA_TYPE_LIST = "data_type_list"
+
+
+@dataclass
+class ValidationTest:
+    check_type: ExpectationType
+    raw_query: ProcessedQuery | None = None
+    generated_query: str | None = None
+    expected: str | None = None
+    result: ModelValidationError | None = None
+    ran: bool = True
+
+
+class ValidationType(Enum):
+    DATASOURCES = "datasources"
+    CONCEPTS = "concepts"
+
+
+def easy_query(
+    concepts: list[BuildConcept],
+    datasource: BuildDatasource,
+    env: Environment,
+    condition: BuildConditional | BuildComparison | None = None,
+    limit: int = 100,
+):
+    """
+    Build basic datasource specific queries.
+    """
+    datasource_outputs = {c.address: c for c in datasource.concepts}
+    first_qds_concepts = datasource.concepts + concepts
+    root_qds = QueryDatasource(
+        input_concepts=first_qds_concepts,
+        output_concepts=concepts,
+        datasources=[datasource],
+        joins=[],
+        source_map={
+            concept.address: (
+                set([datasource]) if concept.address in datasource_outputs else set()
+            )
+            # include all base datasource conepts for convenience
+            for concept in first_qds_concepts
+        },
+        grain=datasource.grain,
+    )
+    cte = CTE(
+        name=f"datasource_{datasource.name}_base",
+        source=root_qds,
+        output_columns=concepts,
+        source_map={
+            concept.address: (
+                [datasource.safe_identifier]
+                if concept.address in datasource_outputs
+                else []
+            )
+            for concept in first_qds_concepts
+        },
+        grain=datasource.grain,
+        group_to_grain=True,
+        base_alias_override=datasource.safe_identifier,
+    )
+
+    filter_cte = CTE(
+        name=f"datasource_{datasource.name}_filter",
+        source=QueryDatasource(
+            datasources=[root_qds],
+            input_concepts=cte.output_columns,
+            output_concepts=cte.output_columns,
+            joins=[],
+            source_map={concept.address: (set([root_qds])) for concept in concepts},
+            grain=cte.grain,
+        ),
+        parent_ctes=[cte],
+        output_columns=cte.output_columns,
+        source_map={
+            concept.address: [cte.identifier] for concept in cte.output_columns
+        },
+        grain=cte.grain,
+        condition=condition,
+        limit=limit,
+    )
+
+    return ProcessedQuery(
+        output_columns=[ConceptRef(address=concept.address) for concept in concepts],
+        ctes=[cte, filter_cte],
+        base=cte,
+        local_concepts=EnvironmentConceptDict(**{}),
+    )

trilogy/core/validation/concept.py

@@ -0,0 +1,125 @@
+from trilogy import Environment, Executor
+from trilogy.core.enums import Derivation, Purpose
+from trilogy.core.exceptions import (
+    ConceptModelValidationError,
+    DatasourceModelValidationError,
+)
+from trilogy.core.models.build import (
+    BuildConcept,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.validation.common import ExpectationType, ValidationTest, easy_query
+
+
+def validate_property_concept(
+    concept: BuildConcept, exec: Executor | None = None
+) -> list[ValidationTest]:
+    return []
+
+
+def validate_key_concept(
+    concept: BuildConcept,
+    env: Environment,
+    build_env: BuildEnvironment,
+    exec: Executor | None = None,
+):
+    results: list[ValidationTest] = []
+    seen: dict[str, int] = {}
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            assignment = [
+                x for x in datasource.columns if x.concept.address == concept.address
+            ][0]
+            # if it's not a partial, skip it
+            if not assignment.is_complete:
+                continue
+            type_query = easy_query(
+                concepts=[
+                    # build_env.concepts[concept.address],
+                    build_env.concepts[f"grain_check_{concept.safe_address}"],
+                ],
+                datasource=datasource,
+                env=env,
+                limit=1,
+            )
+            if exec:
+                type_sql = exec.generate_sql(type_query)[-1]
+
+                rows = exec.execute_raw_sql(type_sql).fetchall()
+                seen[datasource.name] = rows[0][0] if rows else 0
+            else:
+                results.append(
+                    ValidationTest(
+                        raw_query=type_query,
+                        check_type=ExpectationType.ROWCOUNT,
+                        expected=f"equal_max_{concept.safe_address}",
+                        result=None,
+                        ran=False,
+                    )
+                )
+
+    if not exec:
+        return results
+    max_seen: int = max([v for v in seen.values() if v is not None], default=0)
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            assignment = [
+                x for x in datasource.columns if x.concept.address == concept.address
+            ][0]
+            err = None
+            datasource_count: int = seen.get(datasource.name, 0)
+            if datasource_count < max_seen and assignment.is_complete:
+                err = DatasourceModelValidationError(
+                    f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial."
+                )
+            results.append(
+                ValidationTest(
+                    check_type=ExpectationType.ROWCOUNT,
+                    expected=str(max_seen),
+                    result=err,
+                    ran=True,
+                )
+            )
+
+    return results
+
+
+def validate_datasources(
+    concept: BuildConcept, build_env: BuildEnvironment
+) -> list[ValidationTest]:
+    if concept.lineage:
+        return []
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            return []
+    if not concept.derivation == Derivation.ROOT:
+        return []
+    if concept.name.startswith("__") or (
+        concept.namespace and concept.namespace.startswith("__")
+    ):
+        return []
+    return [
+        ValidationTest(
+            check_type=ExpectationType.LOGICAL,
+            expected=None,
+            result=ConceptModelValidationError(
+                f"Concept {concept.address} is a root concept but has no datasources bound"
+            ),
+            ran=True,
+        )
+    ]
+
+
+def validate_concept(
+    concept: BuildConcept,
+    env: Environment,
+    build_env: BuildEnvironment,
+    exec: Executor | None = None,
+) -> list[ValidationTest]:
+    base: list[ValidationTest] = []
+    base += validate_datasources(concept, build_env)
+    if concept.purpose == Purpose.PROPERTY:
+        base += validate_property_concept(concept)
+    elif concept.purpose == Purpose.KEY:
+        base += validate_key_concept(concept, env, build_env, exec)
+    return base

trilogy/core/validation/datasource.py

@@ -0,0 +1,194 @@
+from datetime import date, datetime
+from decimal import Decimal
+from typing import Any
+
+from trilogy import Environment, Executor
+from trilogy.authoring import (
+    ArrayType,
+    DataType,
+    MapType,
+    NumericType,
+    StructType,
+    TraitDataType,
+)
+from trilogy.core.enums import ComparisonOperator
+from trilogy.core.exceptions import DatasourceModelValidationError
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildDatasource,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.validation.common import ExpectationType, ValidationTest, easy_query
+from trilogy.utility import unique
+
+
+def type_check(
+    input: Any,
+    expected_type: (
+        DataType | ArrayType | StructType | MapType | NumericType | TraitDataType
+    ),
+    nullable: bool = True,
+) -> bool:
+    if input is None and nullable:
+        return True
+    target_type = expected_type
+    while isinstance(target_type, TraitDataType):
+        return type_check(input, target_type.data_type, nullable)
+    if target_type == DataType.STRING:
+        return isinstance(input, str)
+    if target_type == DataType.INTEGER:
+        return isinstance(input, int)
+    if target_type == DataType.FLOAT or isinstance(target_type, NumericType):
+        return (
+            isinstance(input, float)
+            or isinstance(input, int)
+            or isinstance(input, Decimal)
+        )
+    if target_type == DataType.BOOL:
+        return isinstance(input, bool)
+    if target_type == DataType.DATE:
+        return isinstance(input, date)
+    if target_type == DataType.DATETIME:
+        return isinstance(input, datetime)
+    if target_type == DataType.ARRAY or isinstance(target_type, ArrayType):
+        return isinstance(input, list)
+    if target_type == DataType.MAP or isinstance(target_type, MapType):
+        return isinstance(input, dict)
+    if target_type == DataType.STRUCT or isinstance(target_type, StructType):
+        return isinstance(input, dict)
+    return False
+
+
+def validate_datasource(
+    datasource: BuildDatasource,
+    env: Environment,
+    build_env: BuildEnvironment,
+    exec: Executor | None = None,
+) -> list[ValidationTest]:
+    results: list[ValidationTest] = []
+    # we might have merged concepts, where both will map out to the same
+    unique_outputs = unique(
+        [build_env.concepts[col.concept.address] for col in datasource.columns],
+        "address",
+    )
+    type_query = easy_query(
+        concepts=unique_outputs,
+        datasource=datasource,
+        env=env,
+        limit=100,
+    )
+
+    rows = []
+    if exec:
+        type_sql = exec.generate_sql(type_query)[-1]
+        try:
+            rows = exec.execute_raw_sql(type_sql).fetchall()
+        except Exception as e:
+            results.append(
+                ValidationTest(
+                    raw_query=type_query,
+                    generated_query=type_sql,
+                    check_type=ExpectationType.LOGICAL,
+                    expected="valid_sql",
+                    result=DatasourceModelValidationError(
+                        f"Datasource {datasource.name} failed validation. Error executing type query {type_sql}: {e}"
+                    ),
+                    ran=True,
+                )
+            )
+            return results
+    else:
+
+        results.append(
+            ValidationTest(
+                raw_query=type_query,
+                check_type=ExpectationType.LOGICAL,
+                expected="datatype_match",
+                result=None,
+                ran=False,
+            )
+        )
+        return results
+    failures: list[
+        tuple[
+            str,
+            Any,
+            DataType | ArrayType | StructType | MapType | NumericType | TraitDataType,
+            bool,
+        ]
+    ] = []
+    cols_with_error = set()
+    for row in rows:
+        for col in datasource.columns:
+            actual_address = build_env.concepts[col.concept.address].safe_address
+            if actual_address in cols_with_error:
+                continue
+            rval = row[actual_address]
+            passed = type_check(rval, col.concept.datatype, col.is_nullable)
+            if not passed:
+                failures.append(
+                    (
+                        col.concept.address,
+                        rval,
+                        col.concept.datatype,
+                        col.is_nullable,
+                    )
+                )
+                cols_with_error.add(actual_address)
+
+    def format_failure(failure):
+        return f"Concept {failure[0]} value '{failure[1]}' does not conform to expected type {str(failure[2])} (nullable={failure[3]})"
+
+    if failures:
+        results.append(
+            ValidationTest(
+                check_type=ExpectationType.LOGICAL,
+                expected="datatype_match",
+                ran=True,
+                result=DatasourceModelValidationError(
+                    f"Datasource {datasource.name} failed validation. Found rows that do not conform to types: {[format_failure(failure) for failure in failures]}",
+                ),
+            )
+        )
+
+    query = easy_query(
+        concepts=[build_env.concepts[name] for name in datasource.grain.components]
+        + [build_env.concepts["grain_check"]],
+        datasource=datasource,
+        env=exec.environment,
+        condition=BuildComparison(
+            left=build_env.concepts["grain_check"],
+            right=1,
+            operator=ComparisonOperator.GT,
+        ),
+    )
+    if not exec:
+        results.append(
+            ValidationTest(
+                raw_query=query,
+                check_type=ExpectationType.ROWCOUNT,
+                expected="0",
+                result=None,
+                ran=False,
+            )
+        )
+
+    else:
+        sql = exec.generate_sql(query)[-1]
+
+        rows = exec.execute_raw_sql(sql).fetchmany(10)
+        if rows:
+            results.append(
+                ValidationTest(
+                    raw_query=query,
+                    generated_query=sql,
+                    check_type=ExpectationType.ROWCOUNT,
+                    expected="0",
+                    result=DatasourceModelValidationError(
+                        f"Datasource {datasource.name} failed validation. Found rows that do not conform to grain: {rows}"
+                    ),
+                    ran=True,
+                )
+            )
+
+    return results

trilogy/core/validation/environment.py

@@ -0,0 +1,71 @@
+from trilogy import Environment, Executor
+from trilogy.authoring import DataType, Function
+from trilogy.core.enums import FunctionType, Purpose, ValidationScope
+from trilogy.core.exceptions import (
+    ModelValidationError,
+)
+from trilogy.core.validation.common import ValidationTest
+from trilogy.core.validation.concept import validate_concept
+from trilogy.core.validation.datasource import validate_datasource
+from trilogy.parsing.common import function_to_concept
+
+
+def validate_environment(
+    env: Environment,
+    scope: ValidationScope = ValidationScope.ALL,
+    targets: list[str] | None = None,
+    exec: Executor | None = None,
+) -> list[ValidationTest]:
+    # avoid mutating the environment for validation
+    generate_only = exec is None
+    env = env.duplicate()
+    grain_check = function_to_concept(
+        parent=Function(
+            operator=FunctionType.SUM,
+            arguments=[1],
+            output_datatype=DataType.INTEGER,
+            output_purpose=Purpose.METRIC,
+        ),
+        name="grain_check",
+        environment=env,
+    )
+    env.add_concept(grain_check)
+    new_concepts = []
+    for concept in env.concepts.values():
+        concept_grain_check = function_to_concept(
+            parent=Function(
+                operator=FunctionType.COUNT_DISTINCT,
+                arguments=[concept.reference],
+                output_datatype=DataType.INTEGER,
+                output_purpose=Purpose.METRIC,
+            ),
+            name=f"grain_check_{concept.safe_address}",
+            environment=env,
+        )
+        new_concepts.append(concept_grain_check)
+    for concept in new_concepts:
+        env.add_concept(concept)
+    build_env = env.materialize_for_select()
+    results: list[ValidationTest] = []
+    if scope == ValidationScope.ALL or scope == ValidationScope.DATASOURCES:
+        for datasource in build_env.datasources.values():
+            if targets and datasource.name not in targets:
+                continue
+            results += validate_datasource(datasource, env, build_env, exec)
+    if scope == ValidationScope.ALL or scope == ValidationScope.CONCEPTS:
+
+        for bconcept in build_env.concepts.values():
+            if targets and bconcept.address not in targets:
+                continue
+            results += validate_concept(bconcept, env, build_env, exec)
+
+    # raise a nicely formatted union of all exceptions
+    exceptions: list[ModelValidationError] = [e.result for e in results if e.result]
+    if exceptions:
+        if not generate_only:
+            messages = "\n".join([str(e) for e in exceptions])
+            raise ModelValidationError(
+                f"Environment validation failed with the following errors:\n{messages}",
+                children=exceptions,
+            )
+    return results