pytrilogy 0.0.3.95__py3-none-any.whl → 0.0.3.96__py3-none-any.whl

{pytrilogy-0.0.3.95.dist-info → pytrilogy-0.0.3.96.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.95
+Version: 0.0.3.96
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -136,11 +136,11 @@ Versus SQL, Trilogy aims to:
 
 | Backend | Status | Notes |
 |---------|--------|-------|
-| **BigQuery** | ✅ Core | Full support |
-| **DuckDB** | ✅ Core | Full support |
-| **Snowflake** | ✅ Core | Full support |
-| **SQL Server** | ⚠️ Experimental | Limited testing |
-| **Presto** | ⚠️ Experimental | Limited testing |
+| **BigQuery** | Core | Full support |
+| **DuckDB** | Core | Full support |
+| **Snowflake** | Core | Full support |
+| **SQL Server** | Experimental | Limited testing |
+| **Presto** | Experimental | Limited testing |
 
 ## Examples
 
@@ -311,7 +311,30 @@ trilogy fmt <path to trilogy file>
 - [Public model repository](https://github.com/trilogydata/trilogy-public-models) - Great place for modeling examples
 - [Full documentation](https://trilogydata.dev/)
 
-## Syntax Reference
+## Python API Integration
+
+### Root Imports
+
+Are stable and should be sufficient for executing code from Trilogy as text.
+
+```python
+from pytrilogy import Executor, Dialect
+```
+
+### Authoring Imports
+
+Are also stable, and should be used for cases which programatically generate Trilogy statements without a base text format 
+or need to process/transform parsed code in more complicated ways.
+
+```python
+from pytrilogy.authoring import Concept, Function, ...
+```
+
+### Other Imports
+
+Are likely to be unstable. Open an issue if you need to take dependencies on other modules outside those two paths. 
+
+## Trilogy Syntax Reference
 
 ### Import
 ```sql

{pytrilogy-0.0.3.95.dist-info → pytrilogy-0.0.3.96.dist-info}/RECORD

@@ -1,14 +1,13 @@
-pytrilogy-0.0.3.95.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=2SHgfndSm3yWw7j2feLaNCtzQoSJCcpfq9h1nEy7XF8,303
-trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pytrilogy-0.0.3.96.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=hBxtfxlbUvTLp_8FCY_-wDqJM7G2RJQ2jntfMf3a0PM,303
 trilogy/constants.py,sha256=eKb_EJvSqjN9tGbdVEViwdtwwh8fZ3-jpOEDqL71y70,1691
 trilogy/engine.py,sha256=3MiADf5MKcmxqiHBuRqiYdsXiLj7oitDfVvXvHrfjkA,2178
-trilogy/executor.py,sha256=AwzC9J2GFfipc4PupFa0mpx5GXghsr0v2djpDV0D70M,19559
+trilogy/executor.py,sha256=YfSjuJ0FVm2gHnNgmUlXijWDTUFjqq9FNakWpeEYO48,15769
 trilogy/parser.py,sha256=o4cfk3j3yhUFoiDKq9ZX_GjBF3dKhDjXEwb63rcBkBM,293
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/render.py,sha256=qQWwduymauOlB517UtM-VGbVe8Cswa4UJub5aGbSO6c,1512
 trilogy/utility.py,sha256=euQccZLKoYBz0LNg5tzLlvv2YHvXh9HArnYp1V3uXsM,763
-trilogy/authoring/__init__.py,sha256=3VQyDVexk-Lpk5oWFA26pvNkJELDiLbVbet5tv7Cmdg,2671
+trilogy/authoring/__init__.py,sha256=TABMOETSMERrWuyDLR0nK4ISlqR0yaqeXrmuOdrSvAY,3060
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=nizWYDCJQ1bigQMtkNIEMNTcN0NoEAXiIHLzpelxQ24,201
 trilogy/core/enums.py,sha256=EusAzz7o_YrWf64TLIED7MfziFOJk8EHM8se5W3nyJk,8644
@@ -17,7 +16,7 @@ trilogy/core/environment_helpers.py,sha256=TRlqVctqIRBxzfjRBmpQsAVoiCcsEKBhG1B6P
 trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
 trilogy/core/exceptions.py,sha256=0Lmc3awJYx94k6uifbHc-EIqlFGV6YrX0QIwP84D4a4,1150
 trilogy/core/functions.py,sha256=ESUWMRmwtavwCLl6z1NP9EFzWTJoXn3orTaaOSsj33Q,33093
-trilogy/core/graph_models.py,sha256=jfYjDQoMtTkSM3n16BCYfhmvAwMHMU-nuQamlYkwzdM,3356
+trilogy/core/graph_models.py,sha256=4EWFTHGfYd72zvS2HYoV6hm7nMC_VEd7vWr6txY-ig0,3400
 trilogy/core/internal.py,sha256=r9QagDB2GvpqlyD_I7VrsfbVfIk5mnok2znEbv72Aa4,2681
 trilogy/core/optimization.py,sha256=ojpn-p79lr03SSVQbbw74iPCyoYpDYBmj1dbZ3oXCjI,8860
 trilogy/core/query_processor.py,sha256=uqygDJqkjIH4vLP-lbGRgTN7rRcYEkr3KGqNimNw_80,20345
@@ -75,20 +74,21 @@ trilogy/core/statements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
 trilogy/core/statements/author.py,sha256=VFzylve72fw0tqMSP5Yiwp8--_r92b9zzX1XAdxuTYQ,15963
 trilogy/core/statements/build.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/statements/common.py,sha256=VnVLULQg1TJLNUFzJaROT1tsf2ewk3IpuhvZaP36R6A,535
-trilogy/core/statements/execute.py,sha256=2ev8nnf41MAgJJAb5gStd__d9hZNOR2pSUTYGFdHvhU,2342
+trilogy/core/statements/execute.py,sha256=kiwJcVeMa4wZR-xLfM2oYOJ9DeyJkP8An38WFyJxktM,2413
 trilogy/core/validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/validation/common.py,sha256=xMx5jF3HDydaGwp9ybBo_NVWOqsmo_mimofdhAv8Kjs,3139
-trilogy/core/validation/concept.py,sha256=HLY8zzMP15VOotvT3lVLh7h_lLJ9evZJ3qaG9huwG50,4344
-trilogy/core/validation/datasource.py,sha256=92l0T0kxzBnGFT1zW_LvZHMkvkfoWvqQf0Nk4qysWts,6346
-trilogy/core/validation/environment.py,sha256=8TuLfIE49zSEtvHPBlNC8QJ5fnS0QZqM0TFpHhy2GN0,2790
+trilogy/core/validation/common.py,sha256=cVbDSowtLf2nl0-QVmNauAeLBBNFkSE5bRZtTHIzW20,3193
+trilogy/core/validation/concept.py,sha256=23wZYw_cGmTQuFvaRM-0T7M2b5ZwqjFMucfvfzyQxlc,4425
+trilogy/core/validation/datasource.py,sha256=HIk7iEKK99k0-WXosiIhedH-U2rDGYGdUMKEt1eMl1w,6394
+trilogy/core/validation/environment.py,sha256=yjSnEH893mTiW9o6YXBtXJWbGSR2kMQWjszDuECznLs,2784
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=QwjQm2os5prVgCaWgpyk1bIL0okCMOGFQdAnKcYXmIo,49392
+trilogy/dialect/base.py,sha256=m2a8azbI3AWfQz-VtSn84H5T-BqjF5PULH6BrgwZzok,49666
 trilogy/dialect/bigquery.py,sha256=XS3hpybeowgfrOrkycAigAF3NX2YUzTzfgE6f__2fT4,4316
 trilogy/dialect/common.py,sha256=tSthIZOXXRPQ4KeMKnDDsH7KlTmf2EVqigVtLyoc4zc,6071
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
 trilogy/dialect/dataframe.py,sha256=RUbNgReEa9g3pL6H7fP9lPTrAij5pkqedpZ99D8_5AE,1522
 trilogy/dialect/duckdb.py,sha256=JoUvQ19WvgxoaJkGLM7DPXOd1H0394k3vBiblksQzOI,5631
 trilogy/dialect/enums.py,sha256=FRNYQ5-w-B6-X0yXKNU5g9GowsMlERFogTC5u2nxL_s,4740
+trilogy/dialect/metadata.py,sha256=Vt4-p82bD1ijqeoI2dagUVUbC-KgNNJ2MvDwQIa5mG8,7034
 trilogy/dialect/postgres.py,sha256=el2PKwfyvWGk5EZtLudqAH5ewLitY1sFHJiocBSyxyM,3393
 trilogy/dialect/presto.py,sha256=k1IaeilR3nzPC9Hp7jlAdzJ7TsuxB3LQTBQ28MYE7O8,3715
 trilogy/dialect/snowflake.py,sha256=T6_mKfhpDazB1xQxqFLS2AJwzwzBcPYY6_qxRnAtFBs,3326
@@ -116,8 +116,8 @@ trilogy/std/money.preql,sha256=XWwvAV3WxBsHX9zfptoYRnBigcfYwrYtBHXTME0xJuQ,2082
 trilogy/std/net.preql,sha256=WZCuvH87_rZntZiuGJMmBDMVKkdhTtxeHOkrXNwJ1EE,416
 trilogy/std/ranking.preql,sha256=LDoZrYyz4g3xsII9XwXfmstZD-_92i1Eox1UqkBIfi8,83
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.95.dist-info/METADATA,sha256=00qsW428K_K33ZhoMDKEMPViV1eCY6Gm6MJZ71fRHRo,10441
-pytrilogy-0.0.3.95.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pytrilogy-0.0.3.95.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.95.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.95.dist-info/RECORD,,
+pytrilogy-0.0.3.96.dist-info/METADATA,sha256=l4yiGzDzMYd4of8nPXBMBKyetasPk4yfIOyENWjEqcU,11023
+pytrilogy-0.0.3.96.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytrilogy-0.0.3.96.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.96.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.96.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.95"
+__version__ = "0.0.3.96"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/authoring/__init__.py

@@ -60,68 +60,82 @@ from trilogy.core.statements.author import (
     RowsetDerivationStatement,
     SelectItem,
     SelectStatement,
+    ShowCategory,
+    ShowStatement,
+    ValidateStatement,
 )
 from trilogy.parsing.common import arbitrary_to_concept, arg_to_datatype
 
 __all__ = [
-    "Concept",
-    "Function",
-    "WhereClause",
-    "Comparison",
-    "FilterItem",
-    "CaseWhen",
-    "CaseElse",
-    "AggregateWrapper",
-    "WindowItem",
-    "WindowOrder",
-    "WindowType",
-    "WindowItemOrder",
-    "WindowItemOver",
-    "DataType",
-    "StructType",
-    "ArrayType",
-    "NumericType",
-    "Grain",
-    "RowsetDerivationStatement",
-    "MapType",
-    "ListWrapper",
+    # trilogy.constants
+    "DEFAULT_NAMESPACE",
+    # trilogy.core.enums
+    "BooleanOperator",
+    "ComparisonOperator",
+    "FunctionClass",
     "FunctionType",
+    "InfiniteFunctionArgs",
+    "Ordering",
+    "Purpose",
+    # trilogy.core.functions
     "FunctionFactory",
-    "ConceptDeclarationStatement",
-    "ConceptTransform",
-    "SelectItem",
-    "SelectStatement",
-    "Environment",
+    # trilogy.core.models.author
+    "AggregateWrapper",
+    "CaseElse",
+    "CaseWhen",
+    "Comparison",
+    "Concept",
     "ConceptRef",
+    "Conditional",
+    "FilterItem",
+    "Function",
+    "FunctionCallWrapper",
     "HavingClause",
     "MagicConstants",
     "Metadata",
+    "MultiSelectLineage",
     "OrderBy",
     "OrderItem",
     "Parenthetical",
+    "RowsetItem",
     "SubselectComparison",
-    "Conditional",
-    "BooleanOperator",
-    "ComparisonOperator",
-    "FunctionClass",
-    "FunctionType",
-    "InfiniteFunctionArgs",
-    "Ordering",
-    "Purpose",
-    "DEFAULT_NAMESPACE",
-    "arbitrary_to_concept",
-    "arg_to_datatype",
-    "MultiSelectStatement",
-    "PersistStatement",
-    "RawSQLStatement",
+    "WhereClause",
+    "WindowItem",
+    "WindowItemOrder",
+    "WindowItemOver",
+    "WindowOrder",
+    "WindowType",
+    # trilogy.core.models.core
+    "ArrayType",
+    "DataType",
+    "ListWrapper",
+    "MapType",
+    "NumericType",
+    "StructType",
+    "TraitDataType",
+    # trilogy.core.models.datasource
+    "Address",
     "Datasource",
     "DatasourceMetadata",
-    "MultiSelectLineage",
-    "RowsetItem",
-    "FunctionCallWrapper",
+    # trilogy.core.models.environment
+    "Environment",
+    # trilogy.core.statements.author
+    "ConceptDeclarationStatement",
+    "ConceptTransform",
     "CopyStatement",
+    "Grain",
     "HasUUID",
     "ImportStatement",
-    "Address",
-    "TraitDataType",
+    "MultiSelectStatement",
+    "PersistStatement",
+    "RawSQLStatement",
+    "RowsetDerivationStatement",
+    "SelectItem",
+    "SelectStatement",
+    "ShowCategory",
+    "ShowStatement",
+    "ValidateStatement",
+    # trilogy.parsing.common
+    "arbitrary_to_concept",
+    "arg_to_datatype",
 ]

trilogy/core/graph_models.py

@@ -64,13 +64,13 @@ def datasource_to_node(input: BuildDatasource) -> str:
 
 
 class ReferenceGraph(nx.DiGraph):
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self.concepts: dict[str, BuildConcept] = {}
         self.datasources: dict[str, BuildDatasource] = {}
         self.pseudonyms: set[tuple[str, str]] = set()
 
-    def copy(self):
+    def copy(self) -> "ReferenceGraph":
         g = ReferenceGraph()
         g.concepts = self.concepts.copy()
         g.datasources = self.datasources.copy()
@@ -83,7 +83,7 @@ class ReferenceGraph(nx.DiGraph):
         # g.add_edges_from(self.edges(data=True))
         return g
 
-    def remove_node(self, n):
+    def remove_node(self, n) -> None:
         if n in self.concepts:
             del self.concepts[n]
         if n in self.datasources:
@@ -98,7 +98,7 @@ class ReferenceGraph(nx.DiGraph):
             self.datasources[node_name] = attr["datasource"]
         super().add_node(node_name, **attr)
 
-    def add_datasource_node(self, node_name, datasource):
+    def add_datasource_node(self, node_name, datasource) -> None:
         self.datasources[node_name] = datasource
         super().add_node(node_name, datasource=datasource)
 

trilogy/core/statements/execute.py

@@ -88,6 +88,8 @@ class ProcessedShowStatement:
             BuildConcept,
             BuildDatasource,
             ProcessedQuery,
+            ProcessedQueryPersist,
+            ProcessedCopyStatement,
             ProcessedValidateStatement,
             ProcessedStaticValueOutput,
         ]

trilogy/core/validation/common.py

@@ -27,7 +27,8 @@ class ExpectationType(Enum):
 @dataclass
 class ValidationTest:
     check_type: ExpectationType
-    query: str | None = None
+    raw_query: ProcessedQuery | None = None
+    generated_query: str | None = None
     expected: str | None = None
     result: ModelValidationError | None = None
     ran: bool = True

trilogy/core/validation/concept.py

@@ -1,4 +1,4 @@
-from trilogy import Executor
+from trilogy import Environment, Executor
 from trilogy.core.enums import Derivation, Purpose
 from trilogy.core.exceptions import (
     ConceptModelValidationError,
@@ -12,64 +12,68 @@ from trilogy.core.validation.common import ExpectationType, ValidationTest, easy
 
 
 def validate_property_concept(
-    concept: BuildConcept, generate_only: bool = False
+    concept: BuildConcept, exec: Executor | None = None
 ) -> list[ValidationTest]:
     return []
 
 
 def validate_key_concept(
     concept: BuildConcept,
+    env: Environment,
     build_env: BuildEnvironment,
-    exec: Executor,
-    generate_only: bool = False,
+    exec: Executor | None = None,
 ):
     results: list[ValidationTest] = []
-    seen = {}
+    seen: dict[str, int] = {}
     for datasource in build_env.datasources.values():
         if concept.address in [c.address for c in datasource.concepts]:
             assignment = [
                 x for x in datasource.columns if x.concept.address == concept.address
             ][0]
+            # if it's not a partial, skip it
+            if not assignment.is_complete:
+                continue
             type_query = easy_query(
                 concepts=[
                     # build_env.concepts[concept.address],
                     build_env.concepts[f"grain_check_{concept.safe_address}"],
                 ],
                 datasource=datasource,
-                env=exec.environment,
+                env=env,
                 limit=1,
             )
-            type_sql = exec.generate_sql(type_query)[-1]
+            if exec:
+                type_sql = exec.generate_sql(type_query)[-1]
 
-            rows = exec.execute_raw_sql(type_sql).fetchall()
-            if generate_only and assignment.is_complete:
+                rows = exec.execute_raw_sql(type_sql).fetchall()
+                seen[datasource.name] = rows[0][0] if rows else 0
+            else:
                 results.append(
                     ValidationTest(
-                        query=type_sql,
+                        raw_query=type_query,
                         check_type=ExpectationType.ROWCOUNT,
                         expected=f"equal_max_{concept.safe_address}",
                         result=None,
                         ran=False,
                     )
                 )
-                continue
-            seen[datasource.name] = rows[0][0] if rows else None
-    if generate_only:
+
+    if not exec:
         return results
-    max_seen = max([v for v in seen.values() if v is not None], default=0)
+    max_seen: int = max([v for v in seen.values() if v is not None], default=0)
     for datasource in build_env.datasources.values():
         if concept.address in [c.address for c in datasource.concepts]:
             assignment = [
                 x for x in datasource.columns if x.concept.address == concept.address
             ][0]
             err = None
-            if (seen[datasource.name] or 0) < max_seen and assignment.is_complete:
+            datasource_count: int = seen.get(datasource.name, 0)
+            if datasource_count < max_seen and assignment.is_complete:
                 err = DatasourceModelValidationError(
                     f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial."
                 )
             results.append(
                 ValidationTest(
-                    query=None,
                     check_type=ExpectationType.ROWCOUNT,
                     expected=str(max_seen),
                     result=err,
@@ -96,7 +100,6 @@ def validate_datasources(
         return []
     return [
         ValidationTest(
-            query=None,
             check_type=ExpectationType.LOGICAL,
             expected=None,
             result=ConceptModelValidationError(
@@ -109,14 +112,14 @@ def validate_datasources(
 
 def validate_concept(
     concept: BuildConcept,
+    env: Environment,
     build_env: BuildEnvironment,
-    exec: Executor,
-    generate_only: bool = False,
+    exec: Executor | None = None,
 ) -> list[ValidationTest]:
     base: list[ValidationTest] = []
     base += validate_datasources(concept, build_env)
     if concept.purpose == Purpose.PROPERTY:
-        base += validate_property_concept(concept, generate_only)
+        base += validate_property_concept(concept)
     elif concept.purpose == Purpose.KEY:
-        base += validate_key_concept(concept, build_env, exec, generate_only)
+        base += validate_key_concept(concept, env, build_env, exec)
     return base

trilogy/core/validation/datasource.py

@@ -2,7 +2,7 @@ from datetime import date, datetime
 from decimal import Decimal
 from typing import Any
 
-from trilogy import Executor
+from trilogy import Environment, Executor
 from trilogy.authoring import (
     ArrayType,
     DataType,
@@ -61,12 +61,12 @@ def type_check(
 
 def validate_datasource(
     datasource: BuildDatasource,
+    env: Environment,
     build_env: BuildEnvironment,
-    exec: Executor,
-    generate_only: bool = False,
+    exec: Executor | None = None,
 ) -> list[ValidationTest]:
     results: list[ValidationTest] = []
-    # we might have merged concepts, where both wil lmap out to the same
+    # we might have merged concepts, where both will map out to the same
     unique_outputs = unique(
         [build_env.concepts[col.concept.address] for col in datasource.columns],
         "address",
@@ -74,18 +74,20 @@ def validate_datasource(
     type_query = easy_query(
         concepts=unique_outputs,
         datasource=datasource,
-        env=exec.environment,
+        env=env,
         limit=100,
     )
-    type_sql = exec.generate_sql(type_query)[-1]
+
     rows = []
-    if not generate_only:
+    if exec:
+        type_sql = exec.generate_sql(type_query)[-1]
         try:
             rows = exec.execute_raw_sql(type_sql).fetchall()
         except Exception as e:
             results.append(
                 ValidationTest(
-                    query=type_sql,
+                    raw_query=type_query,
+                    generated_query=type_sql,
                     check_type=ExpectationType.LOGICAL,
                     expected="valid_sql",
                     result=DatasourceModelValidationError(
@@ -96,9 +98,10 @@ def validate_datasource(
             )
             return results
     else:
+
         results.append(
             ValidationTest(
-                query=type_sql,
+                raw_query=type_query,
                 check_type=ExpectationType.LOGICAL,
                 expected="datatype_match",
                 result=None,
@@ -117,7 +120,6 @@ def validate_datasource(
     cols_with_error = set()
     for row in rows:
         for col in datasource.columns:
-
             actual_address = build_env.concepts[col.concept.address].safe_address
             if actual_address in cols_with_error:
                 continue
@@ -140,7 +142,6 @@ def validate_datasource(
     if failures:
         results.append(
             ValidationTest(
-                query=None,
                 check_type=ExpectationType.LOGICAL,
                 expected="datatype_match",
                 ran=True,
@@ -161,10 +162,10 @@ def validate_datasource(
             operator=ComparisonOperator.GT,
         ),
     )
-    if generate_only:
+    if not exec:
         results.append(
             ValidationTest(
-                query=exec.generate_sql(query)[-1],
+                raw_query=query,
                 check_type=ExpectationType.ROWCOUNT,
                 expected="0",
                 result=None,
@@ -179,7 +180,8 @@ def validate_datasource(
         if rows:
             results.append(
                 ValidationTest(
-                    query=sql,
+                    raw_query=query,
+                    generated_query=sql,
                     check_type=ExpectationType.ROWCOUNT,
                     expected="0",
                     result=DatasourceModelValidationError(

trilogy/core/validation/environment.py

@@ -12,12 +12,12 @@ from trilogy.parsing.common import function_to_concept
 
 def validate_environment(
     env: Environment,
-    exec: Executor,
     scope: ValidationScope = ValidationScope.ALL,
     targets: list[str] | None = None,
-    generate_only: bool = False,
+    exec: Executor | None = None,
 ) -> list[ValidationTest]:
     # avoid mutating the environment for validation
+    generate_only = exec is None
     env = env.duplicate()
     grain_check = function_to_concept(
         parent=Function(
@@ -51,13 +51,13 @@ def validate_environment(
         for datasource in build_env.datasources.values():
             if targets and datasource.name not in targets:
                 continue
-            results += validate_datasource(datasource, build_env, exec, generate_only)
+            results += validate_datasource(datasource, env, build_env, exec)
     if scope == ValidationScope.ALL or scope == ValidationScope.CONCEPTS:
 
         for bconcept in build_env.concepts.values():
             if targets and bconcept.address not in targets:
                 continue
-            results += validate_concept(bconcept, build_env, exec, generate_only)
+            results += validate_concept(bconcept, env, build_env, exec)
 
     # raise a nicely formatted union of all exceptions
     exceptions: list[ModelValidationError] = [e.result for e in results if e.result]

trilogy/dialect/base.py

@@ -76,6 +76,7 @@ from trilogy.core.statements.author import (
 )
 from trilogy.core.statements.execute import (
     PROCESSED_STATEMENT_TYPES,
+    ProcessedCopyStatement,
     ProcessedQuery,
     ProcessedQueryPersist,
     ProcessedRawSQLStatement,
@@ -345,6 +346,7 @@ class BaseDialect:
     COMPLEX_DATATYPE_MAP = COMPLEX_DATATYPE_MAP
     UNNEST_MODE = UnnestMode.CROSS_APPLY
     GROUP_MODE = GroupMode.AUTO
+    EXPLAIN_KEYWORD = "EXPLAIN"
 
     def __init__(self, rendering: Rendering | None = None):
         self.rendering = rendering or CONFIG.rendering
@@ -1135,7 +1137,13 @@ class BaseDialect:
         query: PROCESSED_STATEMENT_TYPES,
     ) -> str:
         if isinstance(query, ProcessedShowStatement):
-            return ";\n".join([str(x) for x in query.output_values])
+            return ";\n".join(
+                [
+                    f'{self.EXPLAIN_KEYWORD} {self.compile_statement(x)}'
+                    for x in query.output_values
+                    if isinstance(x, (ProcessedQuery, ProcessedCopyStatement))
+                ]
+            )
         elif isinstance(query, ProcessedRawSQLStatement):
             return query.text
 

trilogy/dialect/metadata.py

@@ -0,0 +1,233 @@
+from dataclasses import dataclass
+from typing import Any, List, Optional
+
+from trilogy.core.models.author import ConceptRef
+from trilogy.core.models.datasource import Datasource
+from trilogy.core.models.environment import Environment
+from trilogy.core.statements.author import (
+    ConceptDeclarationStatement,
+    ImportStatement,
+    MergeStatementV2,
+)
+from trilogy.core.statements.execute import (
+    ProcessedShowStatement,
+    ProcessedStaticValueOutput,
+    ProcessedValidateStatement,
+)
+from trilogy.core.validation.common import ValidationTest
+from trilogy.dialect.base import BaseDialect
+from trilogy.engine import ResultProtocol
+
+
+@dataclass
+class MockResult(ResultProtocol):
+    values: list["MockResultRow"]
+    columns: list[str]
+
+    def __init__(self, values: list[Any], columns: list[str]):
+        processed: list[MockResultRow] = []
+        for x in values:
+            if isinstance(x, dict):
+                processed.append(MockResultRow(x))
+            elif isinstance(x, MockResultRow):
+                processed.append(x)
+            else:
+                raise ValueError(
+                    f"Cannot process value of type {type(x)} in MockResult"
+                )
+        self.columns = columns
+        self.values = processed
+
+    def __iter__(self):
+        while self.values:
+            yield self.values.pop(0)
+
+    def fetchall(self):
+        return self.values
+
+    def fetchone(self):
+        if self.values:
+            return self.values.pop(0)
+        return None
+
+    def fetchmany(self, size: int):
+        rval = self.values[:size]
+        self.values = self.values[size:]
+        return rval
+
+    def keys(self):
+        return self.columns
+
+    def as_dict(self):
+        return [x.as_dict() if isinstance(x, MockResultRow) else x for x in self.values]
+
+
+@dataclass
+class MockResultRow:
+    _values: dict[str, Any]
+
+    def as_dict(self):
+        return self._values
+
+    def __str__(self) -> str:
+        return str(self._values)
+
+    def __repr__(self) -> str:
+        return repr(self._values)
+
+    def __getattr__(self, name: str) -> Any:
+        if name in self._values:
+            return self._values[name]
+        return super().__getattribute__(name)
+
+    def __getitem__(self, key: str) -> Any:
+        return self._values[key]
+
+    def values(self):
+        return self._values.values()
+
+    def keys(self):
+        return self._values.keys()
+
+
+def generate_result_set(
+    columns: List[ConceptRef], output_data: list[Any]
+) -> MockResult:
+    """Generate a mock result set from columns and output data."""
+    names = [x.address.replace(".", "_") for x in columns]
+    return MockResult(
+        values=[dict(zip(names, [row])) for row in output_data], columns=names
+    )
+
+
+def handle_concept_declaration(query: ConceptDeclarationStatement) -> MockResult:
+    """Handle concept declaration statements without execution."""
+    concept = query.concept
+    return MockResult(
+        [
+            {
+                "address": concept.address,
+                "type": concept.datatype.value,
+                "purpose": concept.purpose.value,
+                "derivation": concept.derivation.value,
+            }
+        ],
+        ["address", "type", "purpose", "derivation"],
+    )
+
+
+def handle_datasource(query: Datasource) -> MockResult:
+    """Handle datasource queries without execution."""
+    return MockResult(
+        [
+            {
+                "name": query.name,
+            }
+        ],
+        ["name"],
+    )
+
+
+def handle_import_statement(query: ImportStatement) -> MockResult:
+    """Handle import statements without execution."""
+    return MockResult(
+        [
+            {
+                "path": query.path,
+                "alias": query.alias,
+            }
+        ],
+        ["path", "alias"],
+    )
+
+
+def handle_merge_statement(
+    query: MergeStatementV2, environment: Environment
+) -> MockResult:
+    """Handle merge statements by updating environment and returning result."""
+    for concept in query.sources:
+        environment.merge_concept(
+            concept, query.targets[concept.address], modifiers=query.modifiers
+        )
+
+    return MockResult(
+        [
+            {
+                "sources": ",".join([x.address for x in query.sources]),
+                "targets": ",".join([x.address for _, x in query.targets.items()]),
+            }
+        ],
+        ["source", "target"],
+    )
+
+
+def handle_processed_show_statement(
+    query: ProcessedShowStatement, compiled_statements: list[str]
+) -> MockResult:
+    """Handle processed show statements without execution."""
+
+    return generate_result_set(query.output_columns, compiled_statements)
+
+
+def raw_validation_to_result(
+    raw: list[ValidationTest], generator: Optional[BaseDialect] = None
+) -> Optional[MockResult]:
+    """Convert raw validation tests to mock result."""
+    if not raw:
+        return None
+    output = []
+    for row in raw:
+        if row.raw_query and generator and not row.generated_query:
+            try:
+                row.generated_query = generator.compile_statement(row.raw_query)
+            except Exception as e:
+                row.generated_query = f"Error generating query: {e}"
+        output.append(
+            {
+                "check_type": row.check_type.value,
+                "expected": row.expected,
+                "result": str(row.result) if row.result else None,
+                "ran": row.ran,
+                "query": row.generated_query if row.generated_query else "",
+            }
+        )
+    return MockResult(output, ["check_type", "expected", "result", "ran", "query"])
+
+
+def handle_processed_validate_statement(
+    query: ProcessedValidateStatement, dialect: BaseDialect, validate_environment_func
+) -> Optional[MockResult]:
+    """Handle processed validate statements."""
+    results = validate_environment_func(query.scope, query.targets)
+    return raw_validation_to_result(results, dialect)
+
+
+def handle_show_statement_outputs(
+    statement: ProcessedShowStatement,
+    compiled_statements: list[str],
+    environment: Environment,
+    dialect: BaseDialect,
+) -> list[MockResult]:
+    """Handle show statement outputs without execution."""
+    output = []
+    for x in statement.output_values:
+        if isinstance(x, ProcessedStaticValueOutput):
+            output.append(generate_result_set(statement.output_columns, x.values))
+        elif compiled_statements:
+
+            output.append(
+                generate_result_set(
+                    statement.output_columns,
+                    compiled_statements,
+                )
+            )
+        elif isinstance(x, ProcessedValidateStatement):
+            from trilogy.core.validation.environment import validate_environment
+
+            raw = validate_environment(environment, x.scope, x.targets)
+            results = raw_validation_to_result(raw, dialect)
+            if results:
+                output.append(results)
+        else:
+            raise NotImplementedError(f"Cannot show type {type(x)} in show statement")
+    return output

trilogy/executor.py

@@ -1,4 +1,3 @@
-from dataclasses import dataclass
 from functools import singledispatchmethod
 from pathlib import Path
 from typing import Any, Generator, List, Optional
@@ -7,7 +6,7 @@ from sqlalchemy import text
 
 from trilogy.constants import MagicConstants, Rendering, logger
 from trilogy.core.enums import FunctionType, Granularity, IOType, ValidationScope
-from trilogy.core.models.author import Concept, ConceptRef, Function
+from trilogy.core.models.author import Concept, Function
 from trilogy.core.models.build import BuildFunction
 from trilogy.core.models.core import ListWrapper, MapWrapper
 from trilogy.core.models.datasource import Datasource
@@ -31,7 +30,6 @@ from trilogy.core.statements.execute import (
     ProcessedQueryPersist,
     ProcessedRawSQLStatement,
     ProcessedShowStatement,
-    ProcessedStaticValueOutput,
     ProcessedValidateStatement,
 )
 from trilogy.core.validation.common import (
@@ -39,82 +37,22 @@ from trilogy.core.validation.common import (
 )
 from trilogy.dialect.base import BaseDialect
 from trilogy.dialect.enums import Dialects
+from trilogy.dialect.metadata import (
+    generate_result_set,
+    handle_concept_declaration,
+    handle_datasource,
+    handle_import_statement,
+    handle_merge_statement,
+    handle_processed_show_statement,
+    handle_processed_validate_statement,
+    handle_show_statement_outputs,
+)
 from trilogy.engine import ExecutionEngine, ResultProtocol
 from trilogy.hooks.base_hook import BaseHook
 from trilogy.parser import parse_text
 from trilogy.render import get_dialect_generator
 
 
-@dataclass
-class MockResult(ResultProtocol):
-    values: list[Any]
-    columns: list[str]
-
-    def __init__(self, values: list[Any], columns: list[str]):
-        processed = []
-        for x in values:
-            if isinstance(x, dict):
-                processed.append(MockResultRow(x))
-            else:
-                processed.append(x)
-        self.columns = columns
-        self.values = processed
-
-    def __iter__(self):
-        while self.values:
-            yield self.values.pop(0)
-
-    def fetchall(self):
-        return self.values
-
-    def fetchone(self):
-        if self.values:
-            return self.values.pop(0)
-        return None
-
-    def fetchmany(self, size: int):
-        rval = self.values[:size]
-        self.values = self.values[size:]
-        return rval
-
-    def keys(self):
-        return self.columns
-
-
-@dataclass
-class MockResultRow:
-    _values: dict[str, Any]
-
-    def __str__(self) -> str:
-        return str(self._values)
-
-    def __repr__(self) -> str:
-        return repr(self._values)
-
-    def __getattr__(self, name: str) -> Any:
-        if name in self._values:
-            return self._values[name]
-        return super().__getattribute__(name)
-
-    def __getitem__(self, key: str) -> Any:
-        return self._values[key]
-
-    def values(self):
-        return self._values.values()
-
-    def keys(self):
-        return self._values.keys()
-
-
-def generate_result_set(
-    columns: List[ConceptRef], output_data: list[Any]
-) -> MockResult:
-    names = [x.address.replace(".", "_") for x in columns]
-    return MockResult(
-        values=[dict(zip(names, [row])) for row in output_data], columns=names
-    )
-
-
 class Executor(object):
     def __init__(
         self,
@@ -150,29 +88,11 @@ class Executor(object):
 
     @execute_query.register
     def _(self, query: ConceptDeclarationStatement) -> ResultProtocol | None:
-        concept = query.concept
-        return MockResult(
-            [
-                {
-                    "address": concept.address,
-                    "type": concept.datatype.value,
-                    "purpose": concept.purpose.value,
-                    "derivation": concept.derivation.value,
-                }
-            ],
-            ["address", "type", "purpose", "derivation"],
-        )
+        return handle_concept_declaration(query)
 
     @execute_query.register
     def _(self, query: Datasource) -> ResultProtocol | None:
-        return MockResult(
-            [
-                {
-                    "name": query.name,
-                }
-            ],
-            ["name"],
-        )
+        return handle_datasource(query)
 
     @execute_query.register
     def _(self, query: str) -> ResultProtocol | None:
@@ -208,66 +128,28 @@ class Executor(object):
 
     @execute_query.register
     def _(self, query: ProcessedShowStatement) -> ResultProtocol | None:
-        return generate_result_set(
-            query.output_columns,
+        return handle_processed_show_statement(
+            query,
             [
                 self.generator.compile_statement(x)
                 for x in query.output_values
-                if isinstance(x, ProcessedQuery)
+                if isinstance(x, (ProcessedQuery, ProcessedQueryPersist))
             ],
         )
 
-    def _raw_validation_to_result(
-        self, raw: list[ValidationTest]
-    ) -> Optional[ResultProtocol]:
-        if not raw:
-            return None
-        output = []
-        for row in raw:
-            output.append(
-                {
-                    "check_type": row.check_type.value,
-                    "expected": row.expected,
-                    "result": str(row.result) if row.result else None,
-                    "ran": row.ran,
-                    "query": row.query if row.query else "",
-                }
-            )
-        return MockResult(output, ["check_type", "expected", "result", "ran", "query"])
-
     @execute_query.register
     def _(self, query: ProcessedValidateStatement) -> ResultProtocol | None:
-        results = self.validate_environment(query.scope, query.targets)
-        return self._raw_validation_to_result(results)
+        return handle_processed_validate_statement(
+            query, self.generator, self.validate_environment
+        )
 
     @execute_query.register
     def _(self, query: ImportStatement) -> ResultProtocol | None:
-        return MockResult(
-            [
-                {
-                    "path": query.path,
-                    "alias": query.alias,
-                }
-            ],
-            ["path", "alias"],
-        )
+        return handle_import_statement(query)
 
     @execute_query.register
     def _(self, query: MergeStatementV2) -> ResultProtocol | None:
-        for concept in query.sources:
-            self.environment.merge_concept(
-                concept, query.targets[concept.address], modifiers=query.modifiers
-            )
-
-        return MockResult(
-            [
-                {
-                    "sources": ",".join([x.address for x in query.sources]),
-                    "targets": ",".join([x.address for _, x in query.targets.items()]),
-                }
-            ],
-            ["source", "target"],
-        )
+        return handle_merge_statement(query, self.environment)
 
     @execute_query.register
     def _(self, query: ProcessedRawSQLStatement) -> ResultProtocol | None:
@@ -516,29 +398,17 @@ class Executor(object):
         # connection = self.engine.connect()
         for statement in self.parse_text_generator(command):
             if isinstance(statement, ProcessedShowStatement):
-                for x in statement.output_values:
-                    if isinstance(x, ProcessedStaticValueOutput):
-                        output.append(
-                            generate_result_set(statement.output_columns, x.values)
-                        )
-                    elif isinstance(x, ProcessedQuery):
-                        output.append(
-                            generate_result_set(
-                                statement.output_columns,
-                                [self.generator.compile_statement(x)],
-                            )
-                        )
-                    elif isinstance(x, ProcessedValidateStatement):
-                        raw = self.validate_environment(
-                            x.scope, x.targets, generate_only=True
-                        )
-                        results = self._raw_validation_to_result(raw)
-                        if results:
-                            output.append(results)
-                    else:
-                        raise NotImplementedError(
-                            f"Cannot show type {type(x)} in show statement"
-                        )
+                results = handle_show_statement_outputs(
+                    statement,
+                    [
+                        self.generator.compile_statement(x)
+                        for x in statement.output_values
+                        if isinstance(x, (ProcessedQuery, ProcessedQueryPersist))
+                    ],
+                    self.environment,
+                    self.generator,
+                )
+                output.extend(results)
                 continue
             if non_interactive:
                 if not isinstance(
@@ -567,5 +437,5 @@ class Executor(object):
         from trilogy.core.validation.environment import validate_environment
 
         return validate_environment(
-            self.environment, self, scope, targets, generate_only
+            self.environment, scope, targets, exec=None if generate_only else self
         )