pytrilogy 0.3.138__cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

trilogy/core/validation/environment.py

@@ -0,0 +1,73 @@
+from trilogy import Environment, Executor
+from trilogy.authoring import DataType, Function
+from trilogy.core.enums import FunctionType, Purpose, ValidationScope
+from trilogy.core.exceptions import (
+    ModelValidationError,
+)
+from trilogy.core.validation.common import ValidationTest
+from trilogy.core.validation.concept import validate_concept
+from trilogy.core.validation.datasource import validate_datasource
+from trilogy.parsing.common import function_to_concept
+
+
+def validate_environment(
+    env: Environment,
+    scope: ValidationScope = ValidationScope.ALL,
+    targets: list[str] | None = None,
+    exec: Executor | None = None,
+    generate_only: bool = False,
+) -> list[ValidationTest]:
+    # avoid mutating the environment for validation
+    generate_only = exec is None or generate_only
+    env = env.duplicate()
+    grain_check = function_to_concept(
+        parent=Function(
+            operator=FunctionType.SUM,
+            arguments=[1],
+            output_datatype=DataType.INTEGER,
+            output_purpose=Purpose.METRIC,
+        ),
+        name="grain_check",
+        environment=env,
+    )
+    env.add_concept(grain_check)
+    new_concepts = []
+    for concept in env.concepts.values():
+        concept_grain_check = function_to_concept(
+            parent=Function(
+                operator=FunctionType.COUNT_DISTINCT,
+                arguments=[concept.reference],
+                output_datatype=DataType.INTEGER,
+                output_purpose=Purpose.METRIC,
+            ),
+            name=f"grain_check_{concept.safe_address}",
+            environment=env,
+        )
+        new_concepts.append(concept_grain_check)
+    for concept in new_concepts:
+        env.add_concept(concept)
+    build_env = env.materialize_for_select()
+    results: list[ValidationTest] = []
+    if scope == ValidationScope.ALL or scope == ValidationScope.DATASOURCES:
+        for datasource in build_env.datasources.values():
+            if targets and datasource.name not in targets:
+                continue
+            results += validate_datasource(datasource, env, build_env, exec)
+    if scope == ValidationScope.ALL or scope == ValidationScope.CONCEPTS:
+
+        for bconcept in build_env.concepts.values():
+            if targets and bconcept.address not in targets:
+                continue
+            results += validate_concept(bconcept, env, build_env, exec)
+
+    # raise a nicely formatted union of all exceptions
+    exceptions: list[ModelValidationError] = [e.result for e in results if e.result]
+    if exceptions:
+        if not generate_only:
+            messages = "\n".join([str(e) for e in exceptions])
+            raise ModelValidationError(
+                f"Environment validation failed with the following errors:\n{messages}",
+                children=exceptions,
+            )
+
+    return results

trilogy/core/validation/fix.py

@@ -0,0 +1,106 @@
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from trilogy import Environment, Executor
+from trilogy.authoring import ConceptDeclarationStatement, Datasource
+from trilogy.core.exceptions import (
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
+)
+from trilogy.core.validation.environment import validate_environment
+from trilogy.parsing.render import Renderer
+
+
+def rewrite_file_with_errors(
+    statements: list[Any], errors: list[DatasourceColumnBindingError]
+):
+    renderer = Renderer()
+    output = []
+    ds_error_map: dict[str, list[DatasourceColumnBindingData]] = defaultdict(list)
+    concept_error_map: dict[str, list[DatasourceColumnBindingData]] = defaultdict(list)
+    for error in errors:
+        if isinstance(error, DatasourceColumnBindingError):
+            for x in error.errors:
+                if error.dataset_address not in ds_error_map:
+                    ds_error_map[error.dataset_address] = []
+                # this is by dataset address
+                if x.is_modifier_issue():
+                    ds_error_map[error.dataset_address].append(x)
+                # this is by column
+                if x.is_type_issue():
+                    concept_error_map[x.address].append(x)
+    for statement in statements:
+        if isinstance(statement, Datasource):
+            if statement.identifier in ds_error_map:
+                error_cols = ds_error_map[statement.identifier]
+                for col in statement.columns:
+                    if col.concept.address in [x.address for x in error_cols]:
+                        error_col = [
+                            x for x in error_cols if x.address == col.concept.address
+                        ][0]
+                        col.modifiers = list(
+                            set(col.modifiers + error_col.value_modifiers)
+                        )
+        elif isinstance(statement, ConceptDeclarationStatement):
+            if statement.concept.address in concept_error_map:
+                error_cols = concept_error_map[statement.concept.address]
+                statement.concept.datatype = error_cols[0].value_type
+        output.append(statement)
+
+    return renderer.render_statement_string(output)
+
+
+DEPTH_CUTOFF = 3
+
+
+def validate_and_rewrite(
+    input: Path | str, exec: Executor | None = None, depth: int = 0
+) -> str | None:
+    if depth > DEPTH_CUTOFF:
+        print(f"Reached depth cutoff of {DEPTH_CUTOFF}, stopping.")
+        return None
+    if isinstance(input, str):
+        raw = input
+        env = Environment()
+    else:
+        with open(input, "r") as f:
+            raw = f.read()
+        env = Environment(working_path=input.parent)
+    if exec:
+        env = exec.environment
+    env, statements = env.parse(raw)
+
+    validation_results = validate_environment(env, exec=exec, generate_only=True)
+
+    errors = [
+        x.result
+        for x in validation_results
+        if isinstance(x.result, DatasourceColumnBindingError)
+    ]
+
+    if not errors:
+        print("No validation errors found")
+        return None
+    print(
+        f"Found {len(errors)} validation errors, attempting to fix, current depth: {depth}..."
+    )
+    for error in errors:
+        for item in error.errors:
+            print(f"- {item.format_failure()}")
+
+    new_text = rewrite_file_with_errors(statements, errors)
+
+    while iteration := validate_and_rewrite(new_text, exec=exec, depth=depth + 1):
+        depth = depth + 1
+        if depth >= DEPTH_CUTOFF:
+            break
+        if iteration:
+            new_text = iteration
+        depth += 1
+    if isinstance(input, Path):
+        with open(input, "w") as f:
+            f.write(new_text)
+        return None
+    else:
+        return new_text

trilogy/dialect/__init__.py

@@ -0,0 +1,32 @@
+from .bigquery import BigqueryDialect
+from .config import (
+    BigQueryConfig,
+    DialectConfig,
+    DuckDBConfig,
+    PostgresConfig,
+    PrestoConfig,
+    SnowflakeConfig,
+    SQLServerConfig,
+)
+from .duckdb import DuckDBDialect
+from .postgres import PostgresDialect
+from .presto import PrestoDialect
+from .snowflake import SnowflakeDialect
+from .sql_server import SqlServerDialect
+
+__all__ = [
+    "BigqueryDialect",
+    "PrestoDialect",
+    "DuckDBDialect",
+    "SnowflakeDialect",
+    "PostgresDialect",
+    "SqlServerDialect",
+    "SQLServerConfig",
+    "DialectConfig",
+    "DuckDBConfig",
+    "BigQueryConfig",
+    "SnowflakeConfig",
+    "PrestoConfig",
+    "PostgresConfig",
+    "DialectConfig",
+]