pytrilogy 0.3.138__cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

trilogy/core/table_processor.py

@@ -0,0 +1,66 @@
+from trilogy.core.enums import Modifier
+from trilogy.core.models.datasource import Address, Datasource
+from trilogy.core.models.environment import Environment
+from trilogy.core.statements.author import CreateStatement
+from trilogy.core.statements.execute import (
+    ColumnInfo,
+    CreateTableInfo,
+    ProcessedCreateStatement,
+)
+
+
+def datasource_to_create_table_info(
+    datasource: Datasource,
+) -> CreateTableInfo:
+    address_field_map: dict[str, str] = {
+        column.concept.address: column.alias  # type: ignore
+        for column in datasource.columns
+        if column.is_concrete
+    }
+    columns_info = [
+        ColumnInfo(
+            # the is_concrete restricts this
+            name=col.alias,  # type: ignore
+            type=col.concept.output_datatype,
+            description=(
+                col.concept.metadata.description if col.concept.metadata else None
+            ),
+            nullable=Modifier.OPTIONAL in col.modifiers,
+            primary_key=col.concept.address in datasource.grain.components,
+        )
+        for col in datasource.columns
+        if col.is_concrete
+    ]
+
+    return CreateTableInfo(
+        name=(
+            datasource.address.location
+            if isinstance(datasource.address, Address)
+            else datasource.address
+        ),
+        columns=columns_info,
+        partition_keys=[
+            address_field_map[c.address]
+            for c in datasource.partition_by
+            if c.address in address_field_map
+        ],
+    )
+
+
+def process_create_statement(
+    statement: CreateStatement,
+    environment: Environment,
+) -> ProcessedCreateStatement:
+    # Process the create statement to extract table info
+    targets_info = []
+    for target in statement.targets:
+        datasource: Datasource | None = environment.datasources.get(target)
+        if not datasource:
+            raise ValueError(f"Datasource {target} not found in environment.")
+
+        create_table_info = datasource_to_create_table_info(datasource)
+        targets_info.append(create_table_info)
+
+    return ProcessedCreateStatement(
+        scope=statement.scope, targets=targets_info, create_mode=statement.create_mode
+    )

trilogy/core/utility.py

@@ -0,0 +1,8 @@
+def safe_quote(string: str, quote_char: str):
+    # split dotted identifiers
+    # TODO: evaluate if we need smarter parsing for strings that could actually include .
+    if string.startswith("https://"):
+        # it's a url, no splitting
+        return f"{quote_char}{string}{quote_char}"
+    components = string.split(".")
+    return ".".join([f"{quote_char}{string}{quote_char}" for string in components])

trilogy/core/validation/README.md

@@ -0,0 +1,46 @@
+# Validation Behavior
+
+
+## Environment
+
+Runs all checks.
+
+## Datasource
+
+Runs checks by comma separated list of datasource names
+
+### Checks
+
+- Column type bindings
+- Grain
+
+## Concepts
+
+Run checks by comma separated list of concept names.
+
+### Checks
+
+- Root concepts have at least one datasource binding
+- Key concepts bound to datasources are correctly partial if they do not contan full set
+
+## Internal vs External Valid
+
+Validation requires us to query the DB to get results to compare against in some cases, and minimally have schema access.
+
+For example, validating bindings to a datasource requires getting all column types, which can be done per-engine based on information schema.
+
+Validating datasource _grain_ requires either checking an enforced PK or - more generally - querying to see duplicates.
+
+For inline evaluation in trilogy, we can internally optimize and raise errors by default.
+
+For external cases where the trilogy engine is not being used for DB access - such as for studio - we instead can only validate
+checks that do not require DB access.
+
+For those that require DB access, we can instead return the required queries and some logical condition formatting and spec.
+
+The client is responsible for then running the query and evaluating the results. This requires more work to integrate on the client side.
+
+We don't have a canonical interchange format, so this will be brittle until we define that.
+
+TODO: explore if we can offload all checks to SQL? Can we, for example, do the datasource validation by unioning multiple tables together and ensuring that the target table has the max?
+

trilogy/core/validation/common.py

@@ -0,0 +1,161 @@
+from dataclasses import dataclass
+from enum import Enum
+
+from trilogy import Environment
+from trilogy.authoring import (
+    ConceptRef,
+    DataType,
+    Ordering,
+    Purpose,
+)
+from trilogy.constants import MagicConstants
+from trilogy.core.enums import ComparisonOperator, FunctionType
+from trilogy.core.exceptions import ModelValidationError
+from trilogy.core.models.build import (
+    BuildCaseElse,
+    BuildCaseWhen,
+    BuildComparison,
+    BuildConcept,
+    BuildConditional,
+    BuildDatasource,
+    BuildFunction,
+    BuildOrderBy,
+    BuildOrderItem,
+)
+from trilogy.core.models.environment import EnvironmentConceptDict
+from trilogy.core.models.execute import (
+    CTE,
+    QueryDatasource,
+)
+from trilogy.core.statements.execute import ProcessedQuery
+
+
+class ExpectationType(Enum):
+    LOGICAL = "logical"
+    ROWCOUNT = "rowcount"
+    DATA_TYPE_LIST = "data_type_list"
+
+
+@dataclass
+class ValidationTest:
+    check_type: ExpectationType
+    raw_query: ProcessedQuery | None = None
+    generated_query: str | None = None
+    expected: str | None = None
+    result: ModelValidationError | None = None
+    ran: bool = True
+
+
+class ValidationType(Enum):
+    DATASOURCES = "datasources"
+    CONCEPTS = "concepts"
+
+
+def build_order_args(concepts: list[BuildConcept]) -> list[BuildFunction]:
+    order_args = []
+    for concept in concepts:
+        order_args.append(
+            BuildFunction(
+                operator=FunctionType.CASE,
+                arguments=[
+                    BuildCaseWhen(
+                        comparison=BuildComparison(
+                            left=concept,
+                            operator=ComparisonOperator.IS,
+                            right=MagicConstants.NULL,
+                        ),
+                        expr=1,
+                    ),
+                    BuildCaseElse(expr=0),
+                ],
+                output_data_type=DataType.INTEGER,
+                output_purpose=Purpose.PROPERTY,
+                arg_count=2,
+            )
+        )
+
+    return order_args
+
+
+def easy_query(
+    concepts: list[BuildConcept],
+    datasource: BuildDatasource,
+    env: Environment,
+    condition: BuildConditional | BuildComparison | None = None,
+    limit: int = 100,
+):
+    """
+    Build basic datasource specific queries.
+    """
+    datasource_outputs = {c.address: c for c in datasource.concepts}
+    first_qds_concepts = datasource.concepts + concepts
+    root_qds = QueryDatasource(
+        input_concepts=first_qds_concepts,
+        output_concepts=concepts,
+        datasources=[datasource],
+        joins=[],
+        source_map={
+            concept.address: (
+                set([datasource]) if concept.address in datasource_outputs else set()
+            )
+            # include all base datasource conepts for convenience
+            for concept in first_qds_concepts
+        },
+        grain=datasource.grain,
+    )
+    cte = CTE(
+        name=f"datasource_{datasource.name}_base",
+        source=root_qds,
+        output_columns=concepts,
+        source_map={
+            concept.address: (
+                [datasource.safe_identifier]
+                if concept.address in datasource_outputs
+                else []
+            )
+            for concept in first_qds_concepts
+        },
+        grain=datasource.grain,
+        group_to_grain=True,
+        base_alias_override=datasource.safe_identifier,
+    )
+    filter_cte = CTE(
+        name=f"datasource_{datasource.name}_filter",
+        source=QueryDatasource(
+            datasources=[root_qds],
+            input_concepts=cte.output_columns,
+            output_concepts=cte.output_columns,
+            joins=[],
+            source_map={concept.address: (set([root_qds])) for concept in concepts},
+            grain=cte.grain,
+        ),
+        parent_ctes=[cte],
+        output_columns=cte.output_columns,
+        source_map={
+            concept.address: [cte.identifier] for concept in cte.output_columns
+        },
+        grain=cte.grain,
+        condition=condition,
+        limit=limit,
+        order_by=BuildOrderBy(
+            items=[
+                BuildOrderItem(
+                    expr=BuildFunction(
+                        operator=FunctionType.SUM,
+                        arguments=build_order_args(concepts),
+                        output_data_type=DataType.INTEGER,
+                        output_purpose=Purpose.PROPERTY,
+                        arg_count=len(concepts),
+                    ),
+                    order=Ordering.DESCENDING,
+                )
+            ]
+        ),
+    )
+
+    return ProcessedQuery(
+        output_columns=[ConceptRef(address=concept.address) for concept in concepts],
+        ctes=[cte, filter_cte],
+        base=cte,
+        local_concepts=EnvironmentConceptDict(**{}),
+    )

trilogy/core/validation/concept.py

@@ -0,0 +1,146 @@
+from trilogy import Environment, Executor
+from trilogy.core.enums import Derivation, Modifier, Purpose
+from trilogy.core.exceptions import (
+    ConceptModelValidationError,
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
+)
+from trilogy.core.models.build import (
+    BuildConcept,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.validation.common import ExpectationType, ValidationTest, easy_query
+
+
+def validate_property_concept(
+    concept: BuildConcept, exec: Executor | None = None
+) -> list[ValidationTest]:
+    return []
+
+
+def validate_key_concept(
+    concept: BuildConcept,
+    env: Environment,
+    build_env: BuildEnvironment,
+    exec: Executor | None = None,
+):
+    results: list[ValidationTest] = []
+    seen: dict[str, int] = {}
+
+    count = 0
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            count += 1
+    # if it only has one source, it's a key
+    if count <= 1:
+        return results
+
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            assignment = [
+                x for x in datasource.columns if x.concept.address == concept.address
+            ][0]
+            # if it's not a partial, skip it
+            if not assignment.is_complete:
+                continue
+            type_query = easy_query(
+                concepts=[
+                    # build_env.concepts[concept.address],
+                    build_env.concepts[f"grain_check_{concept.safe_address}"],
+                ],
+                datasource=datasource,
+                env=env,
+                limit=1,
+            )
+            if exec:
+                type_sql = exec.generate_sql(type_query)[-1]
+
+                rows = exec.execute_raw_sql(type_sql).fetchall()
+                seen[datasource.name] = rows[0][0] if rows else 0
+            else:
+                results.append(
+                    ValidationTest(
+                        raw_query=type_query,
+                        check_type=ExpectationType.ROWCOUNT,
+                        expected=f"equal_max_{concept.safe_address}",
+                        result=None,
+                        ran=False,
+                    )
+                )
+
+    if not exec:
+        return results
+    max_seen: int = max([v for v in seen.values() if v is not None], default=0)
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            assignment = [
+                x for x in datasource.columns if x.concept.address == concept.address
+            ][0]
+            err = None
+            datasource_count: int = seen.get(datasource.name, 0)
+            if datasource_count < max_seen and assignment.is_complete:
+                err = DatasourceColumnBindingError(
+                    address=datasource.identifier,
+                    errors=[
+                        DatasourceColumnBindingData(
+                            address=concept.address,
+                            value=None,
+                            value_type=concept.datatype,
+                            value_modifiers=[Modifier.PARTIAL],
+                            actual_type=concept.datatype,
+                            actual_modifiers=concept.modifiers,
+                        )
+                    ],
+                    message=f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial.",
+                )
+            results.append(
+                ValidationTest(
+                    check_type=ExpectationType.ROWCOUNT,
+                    expected=str(max_seen),
+                    result=err,
+                    ran=True,
+                )
+            )
+
+    return results
+
+
+def validate_datasources(
+    concept: BuildConcept, build_env: BuildEnvironment
+) -> list[ValidationTest]:
+    if concept.lineage:
+        return []
+    for datasource in build_env.datasources.values():
+        if concept.address in [c.address for c in datasource.concepts]:
+            return []
+    if not concept.derivation == Derivation.ROOT:
+        return []
+    if concept.name.startswith("__") or (
+        concept.namespace and concept.namespace.startswith("__")
+    ):
+        return []
+    return [
+        ValidationTest(
+            check_type=ExpectationType.LOGICAL,
+            expected=None,
+            result=ConceptModelValidationError(
+                f"Concept {concept.address} is a root concept but has no datasources bound"
+            ),
+            ran=True,
+        )
+    ]
+
+
+def validate_concept(
+    concept: BuildConcept,
+    env: Environment,
+    build_env: BuildEnvironment,
+    exec: Executor | None = None,
+) -> list[ValidationTest]:
+    base: list[ValidationTest] = []
+    base += validate_datasources(concept, build_env)
+    if concept.purpose == Purpose.PROPERTY:
+        base += validate_property_concept(concept)
+    elif concept.purpose == Purpose.KEY:
+        base += validate_key_concept(concept, env, build_env, exec)
+    return base

trilogy/core/validation/datasource.py

@@ -0,0 +1,227 @@
+from datetime import date, datetime
+from decimal import Decimal
+from typing import Any
+
+from trilogy import Environment, Executor
+from trilogy.authoring import (
+    ArrayType,
+    DataType,
+    MapType,
+    NumericType,
+    StructType,
+    TraitDataType,
+    arg_to_datatype,
+)
+from trilogy.core.enums import ComparisonOperator, Modifier
+from trilogy.core.exceptions import (
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
+    DatasourceModelValidationError,
+)
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildDatasource,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.validation.common import ExpectationType, ValidationTest, easy_query
+from trilogy.utility import unique
+
+
+def row_to_dict(row):
+    return {key: row[key] for key in row.keys()}
+
+
+def type_check(
+    input: Any,
+    expected_type: (
+        DataType | ArrayType | StructType | MapType | NumericType | TraitDataType
+    ),
+    nullable: bool = True,
+) -> bool:
+    if input is None and nullable:
+        return True
+
+    target_type = expected_type
+    while isinstance(target_type, TraitDataType):
+        return type_check(input, target_type.data_type, nullable)
+
+    if target_type == DataType.STRING:
+        return isinstance(input, str)
+    if target_type == DataType.INTEGER:
+        return isinstance(input, int)
+    if target_type == DataType.BIGINT:
+        return isinstance(input, int)  # or check for larger int if needed
+    if target_type == DataType.FLOAT or isinstance(target_type, NumericType):
+        return (
+            isinstance(input, float)
+            or isinstance(input, int)
+            or isinstance(input, Decimal)
+        )
+    if target_type == DataType.NUMBER:
+        return isinstance(input, (int, float, Decimal))
+    if target_type == DataType.NUMERIC:
+        return isinstance(input, (int, float, Decimal))
+    if target_type == DataType.BOOL:
+        return isinstance(input, bool)
+    if target_type == DataType.DATE:
+        return isinstance(input, date) and not isinstance(input, datetime)
+    if target_type == DataType.DATETIME:
+        return isinstance(input, datetime)
+    if target_type == DataType.TIMESTAMP:
+        return isinstance(input, datetime)  # or timestamp type if you have one
+    if target_type == DataType.UNIX_SECONDS:
+        return isinstance(input, (int, float))  # Unix timestamps are numeric
+    if target_type == DataType.DATE_PART:
+        return isinstance(
+            input, str
+        )  # assuming date parts are strings like "year", "month"
+    if target_type == DataType.ARRAY or isinstance(target_type, ArrayType):
+        return isinstance(input, list)
+    if target_type == DataType.MAP or isinstance(target_type, MapType):
+        return isinstance(input, dict)
+    if target_type == DataType.STRUCT or isinstance(target_type, StructType):
+        return isinstance(input, dict)
+    if target_type == DataType.NULL:
+        return input is None
+    if target_type == DataType.UNKNOWN:
+        return True
+    return False
+
+
+def validate_datasource(
+    datasource: BuildDatasource,
+    env: Environment,
+    build_env: BuildEnvironment,
+    exec: Executor | None = None,
+    fix: bool = False,
+) -> list[ValidationTest]:
+    results: list[ValidationTest] = []
+    # we might have merged concepts, where both will map out to the same
+    unique_outputs = unique(
+        [build_env.concepts[col.concept.address] for col in datasource.columns],
+        "address",
+    )
+    type_query = easy_query(
+        concepts=unique_outputs,
+        datasource=datasource,
+        env=env,
+        limit=100,
+    )
+
+    rows = []
+    if exec:
+        type_sql = exec.generate_sql(type_query)[-1]
+        try:
+            rows = exec.execute_raw_sql(type_sql).fetchall()
+        except Exception as e:
+            results.append(
+                ValidationTest(
+                    raw_query=type_query,
+                    generated_query=type_sql,
+                    check_type=ExpectationType.LOGICAL,
+                    expected="valid_sql",
+                    result=DatasourceModelValidationError(
+                        f"Datasource {datasource.name} failed validation. Error executing type query {type_sql}: {e}"
+                    ),
+                    ran=True,
+                )
+            )
+            return results
+    else:
+
+        results.append(
+            ValidationTest(
+                raw_query=type_query,
+                check_type=ExpectationType.LOGICAL,
+                expected="datatype_match",
+                result=None,
+                ran=False,
+            )
+        )
+        return results
+    failures: list[DatasourceColumnBindingData] = []
+    cols_with_error = set()
+    for row in rows:
+        for col in datasource.columns:
+            actual_address = build_env.concepts[col.concept.address].safe_address
+            if actual_address in cols_with_error:
+                continue
+            rval = row[actual_address]
+            passed = type_check(rval, col.concept.datatype, col.is_nullable)
+            if not passed:
+                value_type = (
+                    arg_to_datatype(rval) if rval is not None else col.concept.datatype
+                )
+                traits = None
+                if isinstance(col.concept.datatype, TraitDataType):
+                    traits = col.concept.datatype.traits
+                if traits and not isinstance(value_type, TraitDataType):
+                    value_type = TraitDataType(type=value_type, traits=traits)
+                failures.append(
+                    DatasourceColumnBindingData(
+                        address=col.concept.address,
+                        value=rval,
+                        value_type=value_type,
+                        value_modifiers=[Modifier.NULLABLE] if rval is None else [],
+                        actual_type=col.concept.datatype,
+                        actual_modifiers=col.concept.modifiers,
+                    )
+                )
+                cols_with_error.add(actual_address)
+
+    if failures:
+        results.append(
+            ValidationTest(
+                check_type=ExpectationType.LOGICAL,
+                expected="datatype_match",
+                ran=True,
+                result=DatasourceColumnBindingError(
+                    address=datasource.identifier, errors=failures
+                ),
+            )
+        )
+    if not datasource.grain.components:
+        return results
+
+    # grain validation section
+    query = easy_query(
+        concepts=[build_env.concepts[name] for name in datasource.grain.components]
+        + [build_env.concepts["grain_check"]],
+        datasource=datasource,
+        env=exec.environment,
+        condition=BuildComparison(
+            left=build_env.concepts["grain_check"],
+            right=1,
+            operator=ComparisonOperator.GT,
+        ),
+    )
+    if not exec:
+        results.append(
+            ValidationTest(
+                raw_query=query,
+                check_type=ExpectationType.ROWCOUNT,
+                expected="0",
+                result=None,
+                ran=False,
+            )
+        )
+
+    else:
+        sql = exec.generate_sql(query)[-1]
+
+        rows = exec.execute_raw_sql(sql).fetchmany(10)
+        if rows:
+            results.append(
+                ValidationTest(
+                    raw_query=query,
+                    generated_query=sql,
+                    check_type=ExpectationType.ROWCOUNT,
+                    expected="0",
+                    result=DatasourceModelValidationError(
+                        f"Datasource {datasource.name} failed validation. Found rows that do not conform to grain: {[row_to_dict(r) for r in rows]}"
+                    ),
+                    ran=True,
+                )
+            )
+
+    return results