pytrilogy 0.0.3.95__py3-none-any.whl → 0.0.3.97__py3-none-any.whl

trilogy/core/validation/concept.py

@@ -1,8 +1,9 @@
-from trilogy import Executor
-from trilogy.core.enums import Derivation, Purpose
+from trilogy import Environment, Executor
+from trilogy.core.enums import Derivation, Modifier, Purpose
 from trilogy.core.exceptions import (
     ConceptModelValidationError,
-    DatasourceModelValidationError,
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
 )
 from trilogy.core.models.build import (
     BuildConcept,
@@ -12,64 +13,79 @@ from trilogy.core.validation.common import ExpectationType, ValidationTest, easy
 
 
 def validate_property_concept(
-    concept: BuildConcept, generate_only: bool = False
+    concept: BuildConcept, exec: Executor | None = None
 ) -> list[ValidationTest]:
     return []
 
 
 def validate_key_concept(
     concept: BuildConcept,
+    env: Environment,
     build_env: BuildEnvironment,
-    exec: Executor,
-    generate_only: bool = False,
+    exec: Executor | None = None,
 ):
     results: list[ValidationTest] = []
-    seen = {}
+    seen: dict[str, int] = {}
     for datasource in build_env.datasources.values():
         if concept.address in [c.address for c in datasource.concepts]:
             assignment = [
                 x for x in datasource.columns if x.concept.address == concept.address
             ][0]
+            # if it's not a partial, skip it
+            if not assignment.is_complete:
+                continue
             type_query = easy_query(
                 concepts=[
                     # build_env.concepts[concept.address],
                     build_env.concepts[f"grain_check_{concept.safe_address}"],
                 ],
                 datasource=datasource,
-                env=exec.environment,
+                env=env,
                 limit=1,
             )
-            type_sql = exec.generate_sql(type_query)[-1]
+            if exec:
+                type_sql = exec.generate_sql(type_query)[-1]
 
-            rows = exec.execute_raw_sql(type_sql).fetchall()
-            if generate_only and assignment.is_complete:
+                rows = exec.execute_raw_sql(type_sql).fetchall()
+                seen[datasource.name] = rows[0][0] if rows else 0
+            else:
                 results.append(
                     ValidationTest(
-                        query=type_sql,
+                        raw_query=type_query,
                         check_type=ExpectationType.ROWCOUNT,
                         expected=f"equal_max_{concept.safe_address}",
                         result=None,
                         ran=False,
                     )
                 )
-                continue
-            seen[datasource.name] = rows[0][0] if rows else None
-    if generate_only:
+
+    if not exec:
         return results
-    max_seen = max([v for v in seen.values() if v is not None], default=0)
+    max_seen: int = max([v for v in seen.values() if v is not None], default=0)
     for datasource in build_env.datasources.values():
         if concept.address in [c.address for c in datasource.concepts]:
             assignment = [
                 x for x in datasource.columns if x.concept.address == concept.address
             ][0]
             err = None
-            if (seen[datasource.name] or 0) < max_seen and assignment.is_complete:
-                err = DatasourceModelValidationError(
-                    f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial."
+            datasource_count: int = seen.get(datasource.name, 0)
+            if datasource_count < max_seen and assignment.is_complete:
+                err = DatasourceColumnBindingError(
+                    address=datasource.identifier,
+                    errors=[
+                        DatasourceColumnBindingData(
+                            address=concept.address,
+                            value=None,
+                            value_type=concept.datatype,
+                            value_modifiers=[Modifier.PARTIAL],
+                            actual_type=concept.datatype,
+                            actual_modifiers=concept.modifiers,
+                        )
+                    ],
+                    message=f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial.",
                 )
             results.append(
                 ValidationTest(
-                    query=None,
                     check_type=ExpectationType.ROWCOUNT,
                     expected=str(max_seen),
                     result=err,
@@ -96,7 +112,6 @@ def validate_datasources(
         return []
     return [
         ValidationTest(
-            query=None,
             check_type=ExpectationType.LOGICAL,
             expected=None,
             result=ConceptModelValidationError(
@@ -109,14 +124,14 @@ def validate_datasources(
 
 def validate_concept(
     concept: BuildConcept,
+    env: Environment,
     build_env: BuildEnvironment,
-    exec: Executor,
-    generate_only: bool = False,
+    exec: Executor | None = None,
 ) -> list[ValidationTest]:
     base: list[ValidationTest] = []
     base += validate_datasources(concept, build_env)
     if concept.purpose == Purpose.PROPERTY:
-        base += validate_property_concept(concept, generate_only)
+        base += validate_property_concept(concept)
     elif concept.purpose == Purpose.KEY:
-        base += validate_key_concept(concept, build_env, exec, generate_only)
+        base += validate_key_concept(concept, env, build_env, exec)
     return base

trilogy/core/validation/datasource.py

@@ -2,7 +2,7 @@ from datetime import date, datetime
 from decimal import Decimal
 from typing import Any
 
-from trilogy import Executor
+from trilogy import Environment, Executor
 from trilogy.authoring import (
     ArrayType,
     DataType,
@@ -10,9 +10,14 @@ from trilogy.authoring import (
     NumericType,
     StructType,
     TraitDataType,
+    arg_to_datatype,
+)
+from trilogy.core.enums import ComparisonOperator, Modifier
+from trilogy.core.exceptions import (
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
+    DatasourceModelValidationError,
 )
-from trilogy.core.enums import ComparisonOperator
-from trilogy.core.exceptions import DatasourceModelValidationError
 from trilogy.core.models.build import (
     BuildComparison,
     BuildDatasource,
@@ -61,12 +66,13 @@ def type_check(
 
 def validate_datasource(
     datasource: BuildDatasource,
+    env: Environment,
     build_env: BuildEnvironment,
-    exec: Executor,
-    generate_only: bool = False,
+    exec: Executor | None = None,
+    fix: bool = False,
 ) -> list[ValidationTest]:
     results: list[ValidationTest] = []
-    # we might have merged concepts, where both wil lmap out to the same
+    # we might have merged concepts, where both will map out to the same
     unique_outputs = unique(
         [build_env.concepts[col.concept.address] for col in datasource.columns],
         "address",
@@ -74,18 +80,20 @@ def validate_datasource(
     type_query = easy_query(
         concepts=unique_outputs,
         datasource=datasource,
-        env=exec.environment,
+        env=env,
         limit=100,
     )
-    type_sql = exec.generate_sql(type_query)[-1]
+
     rows = []
-    if not generate_only:
+    if exec:
+        type_sql = exec.generate_sql(type_query)[-1]
         try:
             rows = exec.execute_raw_sql(type_sql).fetchall()
         except Exception as e:
             results.append(
                 ValidationTest(
-                    query=type_sql,
+                    raw_query=type_query,
+                    generated_query=type_sql,
                     check_type=ExpectationType.LOGICAL,
                     expected="valid_sql",
                     result=DatasourceModelValidationError(
@@ -96,9 +104,10 @@ def validate_datasource(
             )
             return results
     else:
+
         results.append(
             ValidationTest(
-                query=type_sql,
+                raw_query=type_query,
                 check_type=ExpectationType.LOGICAL,
                 expected="datatype_match",
                 result=None,
@@ -106,18 +115,10 @@ def validate_datasource(
             )
         )
         return results
-    failures: list[
-        tuple[
-            str,
-            Any,
-            DataType | ArrayType | StructType | MapType | NumericType | TraitDataType,
-            bool,
-        ]
-    ] = []
+    failures: list[DatasourceColumnBindingData] = []
     cols_with_error = set()
     for row in rows:
         for col in datasource.columns:
-
             actual_address = build_env.concepts[col.concept.address].safe_address
             if actual_address in cols_with_error:
                 continue
@@ -125,27 +126,29 @@ def validate_datasource(
             passed = type_check(rval, col.concept.datatype, col.is_nullable)
             if not passed:
                 failures.append(
-                    (
-                        col.concept.address,
-                        rval,
-                        col.concept.datatype,
-                        col.is_nullable,
+                    DatasourceColumnBindingData(
+                        address=col.concept.address,
+                        value=rval,
+                        value_type=(
+                            arg_to_datatype(rval)
+                            if rval is not None
+                            else col.concept.datatype
+                        ),
+                        value_modifiers=[Modifier.NULLABLE] if rval is None else [],
+                        actual_type=col.concept.datatype,
+                        actual_modifiers=col.concept.modifiers,
                     )
                 )
                 cols_with_error.add(actual_address)
 
-    def format_failure(failure):
-        return f"Concept {failure[0]} value '{failure[1]}' does not conform to expected type {str(failure[2])} (nullable={failure[3]})"
-
     if failures:
         results.append(
             ValidationTest(
-                query=None,
                 check_type=ExpectationType.LOGICAL,
                 expected="datatype_match",
                 ran=True,
-                result=DatasourceModelValidationError(
-                    f"Datasource {datasource.name} failed validation. Found rows that do not conform to types: {[format_failure(failure) for failure in failures]}",
+                result=DatasourceColumnBindingError(
+                    address=datasource.identifier, errors=failures
                 ),
             )
         )
@@ -161,10 +164,10 @@ def validate_datasource(
             operator=ComparisonOperator.GT,
         ),
     )
-    if generate_only:
+    if not exec:
         results.append(
             ValidationTest(
-                query=exec.generate_sql(query)[-1],
+                raw_query=query,
                 check_type=ExpectationType.ROWCOUNT,
                 expected="0",
                 result=None,
@@ -179,7 +182,8 @@ def validate_datasource(
         if rows:
             results.append(
                 ValidationTest(
-                    query=sql,
+                    raw_query=query,
+                    generated_query=sql,
                     check_type=ExpectationType.ROWCOUNT,
                     expected="0",
                     result=DatasourceModelValidationError(

trilogy/core/validation/environment.py

@@ -12,12 +12,13 @@ from trilogy.parsing.common import function_to_concept
 
 def validate_environment(
     env: Environment,
-    exec: Executor,
     scope: ValidationScope = ValidationScope.ALL,
     targets: list[str] | None = None,
+    exec: Executor | None = None,
     generate_only: bool = False,
 ) -> list[ValidationTest]:
     # avoid mutating the environment for validation
+    generate_only = exec is None or generate_only
     env = env.duplicate()
     grain_check = function_to_concept(
         parent=Function(
@@ -51,13 +52,13 @@ def validate_environment(
         for datasource in build_env.datasources.values():
             if targets and datasource.name not in targets:
                 continue
-            results += validate_datasource(datasource, build_env, exec, generate_only)
+            results += validate_datasource(datasource, env, build_env, exec)
     if scope == ValidationScope.ALL or scope == ValidationScope.CONCEPTS:
 
         for bconcept in build_env.concepts.values():
             if targets and bconcept.address not in targets:
                 continue
-            results += validate_concept(bconcept, build_env, exec, generate_only)
+            results += validate_concept(bconcept, env, build_env, exec)
 
     # raise a nicely formatted union of all exceptions
     exceptions: list[ModelValidationError] = [e.result for e in results if e.result]

trilogy/core/validation/fix.py

@@ -0,0 +1,106 @@
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from trilogy import Environment, Executor
+from trilogy.authoring import ConceptDeclarationStatement, Datasource
+from trilogy.core.exceptions import (
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
+)
+from trilogy.core.validation.environment import validate_environment
+from trilogy.parsing.render import Renderer
+
+
+def rewrite_file_with_errors(
+    statements: list[Any], errors: list[DatasourceColumnBindingError]
+):
+    renderer = Renderer()
+    output = []
+    ds_error_map: dict[str, list[DatasourceColumnBindingData]] = defaultdict(list)
+    concept_error_map: dict[str, list[DatasourceColumnBindingData]] = defaultdict(list)
+    for error in errors:
+        if isinstance(error, DatasourceColumnBindingError):
+            for x in error.errors:
+                if error.dataset_address not in ds_error_map:
+                    ds_error_map[error.dataset_address] = []
+                # this is by dataset address
+                if x.is_modifier_issue():
+                    ds_error_map[error.dataset_address].append(x)
+                # this is by column
+                if x.is_type_issue():
+                    concept_error_map[x.address].append(x)
+    for statement in statements:
+        if isinstance(statement, Datasource):
+            if statement.identifier in ds_error_map:
+                error_cols = ds_error_map[statement.identifier]
+                for col in statement.columns:
+                    if col.concept.address in [x.address for x in error_cols]:
+                        error_col = [
+                            x for x in error_cols if x.address == col.concept.address
+                        ][0]
+                        col.modifiers = list(
+                            set(col.modifiers + error_col.value_modifiers)
+                        )
+        elif isinstance(statement, ConceptDeclarationStatement):
+            if statement.concept.address in concept_error_map:
+                error_cols = concept_error_map[statement.concept.address]
+                statement.concept.datatype = error_cols[0].value_type
+        output.append(statement)
+
+    return renderer.render_statement_string(output)
+
+
+DEPTH_CUTOFF = 3
+
+
+def validate_and_rewrite(
+    input: Path | str, exec: Executor | None = None, depth: int = 0
+) -> str | None:
+    if depth > DEPTH_CUTOFF:
+        print(f"Reached depth cutoff of {DEPTH_CUTOFF}, stopping.")
+        return None
+    if isinstance(input, str):
+        raw = input
+        env = Environment()
+    else:
+        with open(input, "r") as f:
+            raw = f.read()
+        env = Environment(working_path=input.parent)
+    if exec:
+        env = exec.environment
+    env, statements = env.parse(raw)
+
+    validation_results = validate_environment(env, exec=exec, generate_only=True)
+
+    errors = [
+        x.result
+        for x in validation_results
+        if isinstance(x.result, DatasourceColumnBindingError)
+    ]
+
+    if not errors:
+        print("No validation errors found")
+        return None
+    print(
+        f"Found {len(errors)} validation errors, attempting to fix, current depth: {depth}..."
+    )
+    for error in errors:
+        for item in error.errors:
+            print(f"- {item.format_failure()}")
+
+    new_text = rewrite_file_with_errors(statements, errors)
+
+    while iteration := validate_and_rewrite(new_text, exec=exec, depth=depth + 1):
+        depth = depth + 1
+        if depth >= DEPTH_CUTOFF:
+            break
+        if iteration:
+            new_text = iteration
+        depth += 1
+    if isinstance(input, Path):
+        with open(input, "w") as f:
+            f.write(new_text)
+        return None
+    else:
+        return new_text

trilogy/dialect/base.py

@@ -76,6 +76,7 @@ from trilogy.core.statements.author import (
 )
 from trilogy.core.statements.execute import (
     PROCESSED_STATEMENT_TYPES,
+    ProcessedCopyStatement,
     ProcessedQuery,
     ProcessedQueryPersist,
     ProcessedRawSQLStatement,
@@ -345,6 +346,7 @@ class BaseDialect:
     COMPLEX_DATATYPE_MAP = COMPLEX_DATATYPE_MAP
     UNNEST_MODE = UnnestMode.CROSS_APPLY
     GROUP_MODE = GroupMode.AUTO
+    EXPLAIN_KEYWORD = "EXPLAIN"
 
     def __init__(self, rendering: Rendering | None = None):
         self.rendering = rendering or CONFIG.rendering
@@ -759,6 +761,7 @@ class BaseDialect:
         elif isinstance(e, MagicConstants):
             if e == MagicConstants.NULL:
                 return "null"
+            return str(e.value)
         elif isinstance(e, date):
             return self.FUNCTION_MAP[FunctionType.DATE_LITERAL](e)
         elif isinstance(e, datetime):
@@ -1135,7 +1138,13 @@ class BaseDialect:
         query: PROCESSED_STATEMENT_TYPES,
     ) -> str:
         if isinstance(query, ProcessedShowStatement):
-            return ";\n".join([str(x) for x in query.output_values])
+            return ";\n".join(
+                [
+                    f"{self.EXPLAIN_KEYWORD} {self.compile_statement(x)}"
+                    for x in query.output_values
+                    if isinstance(x, (ProcessedQuery, ProcessedCopyStatement))
+                ]
+            )
         elif isinstance(query, ProcessedRawSQLStatement):
             return query.text