pytrilogy 0.0.3.94__py3-none-any.whl → 0.0.3.95__py3-none-any.whl

trilogy/core/validation/common.py

@@ -0,0 +1,109 @@
+from dataclasses import dataclass
+from enum import Enum
+
+from trilogy import Environment
+from trilogy.authoring import ConceptRef
+from trilogy.core.exceptions import ModelValidationError
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildConcept,
+    BuildConditional,
+    BuildDatasource,
+)
+from trilogy.core.models.environment import EnvironmentConceptDict
+from trilogy.core.models.execute import (
+    CTE,
+    QueryDatasource,
+)
+from trilogy.core.statements.execute import ProcessedQuery
+
+
+class ExpectationType(Enum):
+    LOGICAL = "logical"
+    ROWCOUNT = "rowcount"
+    DATA_TYPE_LIST = "data_type_list"
+
+
+@dataclass
+class ValidationTest:
+    check_type: ExpectationType
+    query: str | None = None
+    expected: str | None = None
+    result: ModelValidationError | None = None
+    ran: bool = True
+
+
+class ValidationType(Enum):
+    DATASOURCES = "datasources"
+    CONCEPTS = "concepts"
+
+
+def easy_query(
+    concepts: list[BuildConcept],
+    datasource: BuildDatasource,
+    env: Environment,
+    condition: BuildConditional | BuildComparison | None = None,
+    limit: int = 100,
+):
+    """
+    Build basic datasource specific queries.
+    """
+    datasource_outputs = {c.address: c for c in datasource.concepts}
+    first_qds_concepts = datasource.concepts + concepts
+    root_qds = QueryDatasource(
+        input_concepts=first_qds_concepts,
+        output_concepts=concepts,
+        datasources=[datasource],
+        joins=[],
+        source_map={
+            concept.address: (
+                set([datasource]) if concept.address in datasource_outputs else set()
+            )
+            # include all base datasource conepts for convenience
+            for concept in first_qds_concepts
+        },
+        grain=datasource.grain,
+    )
+    cte = CTE(
+        name=f"datasource_{datasource.name}_base",
+        source=root_qds,
+        output_columns=concepts,
+        source_map={
+            concept.address: (
+                [datasource.safe_identifier]
+                if concept.address in datasource_outputs
+                else []
+            )
+            for concept in first_qds_concepts
+        },
+        grain=datasource.grain,
+        group_to_grain=True,
+        base_alias_override=datasource.safe_identifier,
+    )
+
+    filter_cte = CTE(
+        name=f"datasource_{datasource.name}_filter",
+        source=QueryDatasource(
+            datasources=[root_qds],
+            input_concepts=cte.output_columns,
+            output_concepts=cte.output_columns,
+            joins=[],
+            source_map={concept.address: (set([root_qds])) for concept in concepts},
+            grain=cte.grain,
+        ),
+        parent_ctes=[cte],
+        output_columns=cte.output_columns,
+        source_map={
+            concept.address: [cte.identifier] for concept in cte.output_columns
+        },
+        grain=cte.grain,
+        condition=condition,
+        limit=limit,
+    )
+
+    return ProcessedQuery(
+        output_columns=[ConceptRef(address=concept.address) for concept in concepts],
+        ctes=[cte, filter_cte],
+        base=cte,
+        local_concepts=EnvironmentConceptDict(**{}),
+    )

trilogy/core/validation/concept.py

@@ -0,0 +1,122 @@
+from trilogy import Executor
+from trilogy.core.enums import Derivation, Purpose
+from trilogy.core.exceptions import (
+    ConceptModelValidationError,
+    DatasourceModelValidationError,
+)
+from trilogy.core.models.build import (
+    BuildConcept,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.validation.common import ExpectationType, ValidationTest, easy_query
+
+
+def validate_property_concept(
+    concept: BuildConcept, generate_only: bool = False
+) -> list[ValidationTest]:
+    return []
+
+
+def validate_key_concept(
+    concept: BuildConcept,
+    build_env: BuildEnvironment,
+    exec: Executor,
+    generate_only: bool = False,
+):
+    results: list[ValidationTest] = []
+    seen = {}
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            assignment = [
+                x for x in datasource.columns if x.concept.address == concept.address
+            ][0]
+            type_query = easy_query(
+                concepts=[
+                    # build_env.concepts[concept.address],
+                    build_env.concepts[f"grain_check_{concept.safe_address}"],
+                ],
+                datasource=datasource,
+                env=exec.environment,
+                limit=1,
+            )
+            type_sql = exec.generate_sql(type_query)[-1]
+
+            rows = exec.execute_raw_sql(type_sql).fetchall()
+            if generate_only and assignment.is_complete:
+                results.append(
+                    ValidationTest(
+                        query=type_sql,
+                        check_type=ExpectationType.ROWCOUNT,
+                        expected=f"equal_max_{concept.safe_address}",
+                        result=None,
+                        ran=False,
+                    )
+                )
+                continue
+            seen[datasource.name] = rows[0][0] if rows else None
+    if generate_only:
+        return results
+    max_seen = max([v for v in seen.values() if v is not None], default=0)
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            assignment = [
+                x for x in datasource.columns if x.concept.address == concept.address
+            ][0]
+            err = None
+            if (seen[datasource.name] or 0) < max_seen and assignment.is_complete:
+                err = DatasourceModelValidationError(
+                    f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial."
+                )
+            results.append(
+                ValidationTest(
+                    query=None,
+                    check_type=ExpectationType.ROWCOUNT,
+                    expected=str(max_seen),
+                    result=err,
+                    ran=True,
+                )
+            )
+
+    return results
+
+
+def validate_datasources(
+    concept: BuildConcept, build_env: BuildEnvironment
+) -> list[ValidationTest]:
+    if concept.lineage:
+        return []
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            return []
+    if not concept.derivation == Derivation.ROOT:
+        return []
+    if concept.name.startswith("__") or (
+        concept.namespace and concept.namespace.startswith("__")
+    ):
+        return []
+    return [
+        ValidationTest(
+            query=None,
+            check_type=ExpectationType.LOGICAL,
+            expected=None,
+            result=ConceptModelValidationError(
+                f"Concept {concept.address} is a root concept but has no datasources bound"
+            ),
+            ran=True,
+        )
+    ]
+
+
+def validate_concept(
+    concept: BuildConcept,
+    build_env: BuildEnvironment,
+    exec: Executor,
+    generate_only: bool = False,
+) -> list[ValidationTest]:
+    base: list[ValidationTest] = []
+    base += validate_datasources(concept, build_env)
+    if concept.purpose == Purpose.PROPERTY:
+        base += validate_property_concept(concept, generate_only)
+    elif concept.purpose == Purpose.KEY:
+        base += validate_key_concept(concept, build_env, exec, generate_only)
+    return base

trilogy/core/validation/datasource.py

@@ -0,0 +1,192 @@
+from datetime import date, datetime
+from decimal import Decimal
+from typing import Any
+
+from trilogy import Executor
+from trilogy.authoring import (
+    ArrayType,
+    DataType,
+    MapType,
+    NumericType,
+    StructType,
+    TraitDataType,
+)
+from trilogy.core.enums import ComparisonOperator
+from trilogy.core.exceptions import DatasourceModelValidationError
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildDatasource,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.validation.common import ExpectationType, ValidationTest, easy_query
+from trilogy.utility import unique
+
+
+def type_check(
+    input: Any,
+    expected_type: (
+        DataType | ArrayType | StructType | MapType | NumericType | TraitDataType
+    ),
+    nullable: bool = True,
+) -> bool:
+    if input is None and nullable:
+        return True
+    target_type = expected_type
+    while isinstance(target_type, TraitDataType):
+        return type_check(input, target_type.data_type, nullable)
+    if target_type == DataType.STRING:
+        return isinstance(input, str)
+    if target_type == DataType.INTEGER:
+        return isinstance(input, int)
+    if target_type == DataType.FLOAT or isinstance(target_type, NumericType):
+        return (
+            isinstance(input, float)
+            or isinstance(input, int)
+            or isinstance(input, Decimal)
+        )
+    if target_type == DataType.BOOL:
+        return isinstance(input, bool)
+    if target_type == DataType.DATE:
+        return isinstance(input, date)
+    if target_type == DataType.DATETIME:
+        return isinstance(input, datetime)
+    if target_type == DataType.ARRAY or isinstance(target_type, ArrayType):
+        return isinstance(input, list)
+    if target_type == DataType.MAP or isinstance(target_type, MapType):
+        return isinstance(input, dict)
+    if target_type == DataType.STRUCT or isinstance(target_type, StructType):
+        return isinstance(input, dict)
+    return False
+
+
+def validate_datasource(
+    datasource: BuildDatasource,
+    build_env: BuildEnvironment,
+    exec: Executor,
+    generate_only: bool = False,
+) -> list[ValidationTest]:
+    results: list[ValidationTest] = []
+    # we might have merged concepts, where both wil lmap out to the same
+    unique_outputs = unique(
+        [build_env.concepts[col.concept.address] for col in datasource.columns],
+        "address",
+    )
+    type_query = easy_query(
+        concepts=unique_outputs,
+        datasource=datasource,
+        env=exec.environment,
+        limit=100,
+    )
+    type_sql = exec.generate_sql(type_query)[-1]
+    rows = []
+    if not generate_only:
+        try:
+            rows = exec.execute_raw_sql(type_sql).fetchall()
+        except Exception as e:
+            results.append(
+                ValidationTest(
+                    query=type_sql,
+                    check_type=ExpectationType.LOGICAL,
+                    expected="valid_sql",
+                    result=DatasourceModelValidationError(
+                        f"Datasource {datasource.name} failed validation. Error executing type query {type_sql}: {e}"
+                    ),
+                    ran=True,
+                )
+            )
+            return results
+    else:
+        results.append(
+            ValidationTest(
+                query=type_sql,
+                check_type=ExpectationType.LOGICAL,
+                expected="datatype_match",
+                result=None,
+                ran=False,
+            )
+        )
+        return results
+    failures: list[
+        tuple[
+            str,
+            Any,
+            DataType | ArrayType | StructType | MapType | NumericType | TraitDataType,
+            bool,
+        ]
+    ] = []
+    cols_with_error = set()
+    for row in rows:
+        for col in datasource.columns:
+
+            actual_address = build_env.concepts[col.concept.address].safe_address
+            if actual_address in cols_with_error:
+                continue
+            rval = row[actual_address]
+            passed = type_check(rval, col.concept.datatype, col.is_nullable)
+            if not passed:
+                failures.append(
+                    (
+                        col.concept.address,
+                        rval,
+                        col.concept.datatype,
+                        col.is_nullable,
+                    )
+                )
+                cols_with_error.add(actual_address)
+
+    def format_failure(failure):
+        return f"Concept {failure[0]} value '{failure[1]}' does not conform to expected type {str(failure[2])} (nullable={failure[3]})"
+
+    if failures:
+        results.append(
+            ValidationTest(
+                query=None,
+                check_type=ExpectationType.LOGICAL,
+                expected="datatype_match",
+                ran=True,
+                result=DatasourceModelValidationError(
+                    f"Datasource {datasource.name} failed validation. Found rows that do not conform to types: {[format_failure(failure) for failure in failures]}",
+                ),
+            )
+        )
+
+    query = easy_query(
+        concepts=[build_env.concepts[name] for name in datasource.grain.components]
+        + [build_env.concepts["grain_check"]],
+        datasource=datasource,
+        env=exec.environment,
+        condition=BuildComparison(
+            left=build_env.concepts["grain_check"],
+            right=1,
+            operator=ComparisonOperator.GT,
+        ),
+    )
+    if generate_only:
+        results.append(
+            ValidationTest(
+                query=exec.generate_sql(query)[-1],
+                check_type=ExpectationType.ROWCOUNT,
+                expected="0",
+                result=None,
+                ran=False,
+            )
+        )
+
+    else:
+        sql = exec.generate_sql(query)[-1]
+
+        rows = exec.execute_raw_sql(sql).fetchmany(10)
+        if rows:
+            results.append(
+                ValidationTest(
+                    query=sql,
+                    check_type=ExpectationType.ROWCOUNT,
+                    expected="0",
+                    result=DatasourceModelValidationError(
+                        f"Datasource {datasource.name} failed validation. Found rows that do not conform to grain: {rows}"
+                    ),
+                    ran=True,
+                )
+            )
+
+    return results

trilogy/core/validation/environment.py

@@ -0,0 +1,71 @@
+from trilogy import Environment, Executor
+from trilogy.authoring import DataType, Function
+from trilogy.core.enums import FunctionType, Purpose, ValidationScope
+from trilogy.core.exceptions import (
+    ModelValidationError,
+)
+from trilogy.core.validation.common import ValidationTest
+from trilogy.core.validation.concept import validate_concept
+from trilogy.core.validation.datasource import validate_datasource
+from trilogy.parsing.common import function_to_concept
+
+
+def validate_environment(
+    env: Environment,
+    exec: Executor,
+    scope: ValidationScope = ValidationScope.ALL,
+    targets: list[str] | None = None,
+    generate_only: bool = False,
+) -> list[ValidationTest]:
+    # avoid mutating the environment for validation
+    env = env.duplicate()
+    grain_check = function_to_concept(
+        parent=Function(
+            operator=FunctionType.SUM,
+            arguments=[1],
+            output_datatype=DataType.INTEGER,
+            output_purpose=Purpose.METRIC,
+        ),
+        name="grain_check",
+        environment=env,
+    )
+    env.add_concept(grain_check)
+    new_concepts = []
+    for concept in env.concepts.values():
+        concept_grain_check = function_to_concept(
+            parent=Function(
+                operator=FunctionType.COUNT_DISTINCT,
+                arguments=[concept.reference],
+                output_datatype=DataType.INTEGER,
+                output_purpose=Purpose.METRIC,
+            ),
+            name=f"grain_check_{concept.safe_address}",
+            environment=env,
+        )
+        new_concepts.append(concept_grain_check)
+    for concept in new_concepts:
+        env.add_concept(concept)
+    build_env = env.materialize_for_select()
+    results: list[ValidationTest] = []
+    if scope == ValidationScope.ALL or scope == ValidationScope.DATASOURCES:
+        for datasource in build_env.datasources.values():
+            if targets and datasource.name not in targets:
+                continue
+            results += validate_datasource(datasource, build_env, exec, generate_only)
+    if scope == ValidationScope.ALL or scope == ValidationScope.CONCEPTS:
+
+        for bconcept in build_env.concepts.values():
+            if targets and bconcept.address not in targets:
+                continue
+            results += validate_concept(bconcept, build_env, exec, generate_only)
+
+    # raise a nicely formatted union of all exceptions
+    exceptions: list[ModelValidationError] = [e.result for e in results if e.result]
+    if exceptions:
+        if not generate_only:
+            messages = "\n".join([str(e) for e in exceptions])
+            raise ModelValidationError(
+                f"Environment validation failed with the following errors:\n{messages}",
+                children=exceptions,
+            )
+    return results

trilogy/dialect/base.py

@@ -72,14 +72,16 @@ from trilogy.core.statements.author import (
     RowsetDerivationStatement,
     SelectStatement,
     ShowStatement,
+    ValidateStatement,
 )
 from trilogy.core.statements.execute import (
-    ProcessedCopyStatement,
+    PROCESSED_STATEMENT_TYPES,
     ProcessedQuery,
     ProcessedQueryPersist,
     ProcessedRawSQLStatement,
     ProcessedShowStatement,
     ProcessedStaticValueOutput,
+    ProcessedValidateStatement,
 )
 from trilogy.core.utility import safe_quote
 from trilogy.dialect.common import render_join, render_unnest
@@ -1025,21 +1027,11 @@ class BaseDialect:
             | RawSQLStatement
             | MergeStatementV2
             | CopyStatement
+            | ValidateStatement
         ],
         hooks: Optional[List[BaseHook]] = None,
-    ) -> List[
-        ProcessedQuery
-        | ProcessedQueryPersist
-        | ProcessedShowStatement
-        | ProcessedRawSQLStatement
-    ]:
-        output: List[
-            ProcessedQuery
-            | ProcessedQueryPersist
-            | ProcessedShowStatement
-            | ProcessedRawSQLStatement
-            | ProcessedCopyStatement
-        ] = []
+    ) -> List[PROCESSED_STATEMENT_TYPES]:
+        output: List[PROCESSED_STATEMENT_TYPES] = []
         for statement in statements:
             if isinstance(statement, PersistStatement):
                 if hooks:
@@ -1089,10 +1081,39 @@ class BaseDialect:
                     output.append(
                         self.create_show_output(environment, statement.content)
                     )
+                elif isinstance(statement.content, ValidateStatement):
+                    output.append(
+                        ProcessedShowStatement(
+                            output_columns=[
+                                environment.concepts[
+                                    DEFAULT_CONCEPTS["label"].address
+                                ].reference,
+                                environment.concepts[
+                                    DEFAULT_CONCEPTS["query_text"].address
+                                ].reference,
+                                environment.concepts[
+                                    DEFAULT_CONCEPTS["expected"].address
+                                ].reference,
+                            ],
+                            output_values=[
+                                ProcessedValidateStatement(
+                                    scope=statement.content.scope,
+                                    targets=statement.content.targets,
+                                )
+                            ],
+                        )
+                    )
                 else:
                     raise NotImplementedError(type(statement.content))
             elif isinstance(statement, RawSQLStatement):
                 output.append(ProcessedRawSQLStatement(text=statement.text))
+            elif isinstance(statement, ValidateStatement):
+                output.append(
+                    ProcessedValidateStatement(
+                        scope=statement.scope,
+                        targets=statement.targets,
+                    )
+                )
             elif isinstance(
                 statement,
                 (
@@ -1111,18 +1132,16 @@ class BaseDialect:
 
     def compile_statement(
         self,
-        query: (
-            ProcessedQuery
-            | ProcessedQueryPersist
-            | ProcessedShowStatement
-            | ProcessedRawSQLStatement
-        ),
+        query: PROCESSED_STATEMENT_TYPES,
     ) -> str:
         if isinstance(query, ProcessedShowStatement):
             return ";\n".join([str(x) for x in query.output_values])
         elif isinstance(query, ProcessedRawSQLStatement):
             return query.text
 
+        elif isinstance(query, ProcessedValidateStatement):
+            return "select 1;"
+
         recursive = any(isinstance(x, RecursiveCTE) for x in query.ctes)
 
         compiled_ctes = self.generate_ctes(query)
@@ -1139,7 +1158,7 @@ class BaseDialect:
         if CONFIG.strict_mode and INVALID_REFERENCE_STRING(1) in final:
             raise ValueError(
                 f"Invalid reference string found in query: {final}, this should never"
-                " occur. Please create a GitHub issue to report this."
+                " occur. Please create an issue to report this."
             )
         logger.info(f"{LOGGER_PREFIX} Compiled query: {final}")
         return final

trilogy/dialect/sql_server.py

@@ -8,6 +8,7 @@ from trilogy.core.statements.execute import (
     ProcessedQueryPersist,
     ProcessedRawSQLStatement,
     ProcessedShowStatement,
+    ProcessedValidateStatement,
 )
 from trilogy.dialect.base import BaseDialect
 from trilogy.utility import string_to_hash
@@ -90,10 +91,11 @@ class SqlServerDialect(BaseDialect):
             | ProcessedQueryPersist
             | ProcessedShowStatement
             | ProcessedRawSQLStatement
+            | ProcessedValidateStatement
         ),
     ) -> str:
         base = super().compile_statement(query)
-        if isinstance(base, (ProcessedQuery, ProcessedQueryPersist)):
+        if isinstance(query, (ProcessedQuery, ProcessedQueryPersist)):
             for cte in query.ctes:
                 if len(cte.name) > MAX_IDENTIFIER_LENGTH:
                     new_name = f"rhash_{string_to_hash(cte.name)}"

trilogy/engine.py

@@ -1,21 +1,27 @@
-from typing import Any, Protocol
+from typing import Any, Generator, List, Optional, Protocol
 
 from sqlalchemy.engine import Connection, CursorResult, Engine
 
 from trilogy.core.models.environment import Environment
 
 
-class EngineResult(Protocol):
-    pass
+class ResultProtocol(Protocol):
 
-    def fetchall(self) -> list[tuple]:
-        pass
+    def fetchall(self) -> List[Any]: ...
+
+    def keys(self) -> List[str]: ...
+
+    def fetchone(self) -> Optional[Any]: ...
+
+    def fetchmany(self, size: int) -> List[Any]: ...
+
+    def __iter__(self) -> Generator[Any, None, None]: ...
 
 
 class EngineConnection(Protocol):
     pass
 
-    def execute(self, statement: str, parameters: Any | None = None) -> EngineResult:
+    def execute(self, statement: str, parameters: Any | None = None) -> ResultProtocol:
         pass
 
     def commit(self):
@@ -39,13 +45,25 @@ class ExecutionEngine(Protocol):
 
 
 ### Begin default SQLAlchemy implementation
-class SqlAlchemyResult(EngineResult):
+class SqlAlchemyResult:
     def __init__(self, result: CursorResult):
         self.result = result
 
     def fetchall(self):
         return self.result.fetchall()
 
+    def keys(self):
+        return self.result.keys()
+
+    def fetchone(self):
+        return self.result.fetchone()
+
+    def fetchmany(self, size: int):
+        return self.result.fetchmany(size)
+
+    def __iter__(self):
+        return iter(self.result)
+
 
 class SqlAlchemyConnection(EngineConnection):
     def __init__(self, connection: Connection):