pytrilogy 0.0.3.96__py3-none-any.whl → 0.0.3.97__py3-none-any.whl

{pytrilogy-0.0.3.96.dist-info → pytrilogy-0.0.3.97.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.96
+Version: 0.0.3.97
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -334,7 +334,21 @@ from pytrilogy.authoring import Concept, Function, ...
 
 Are likely to be unstable. Open an issue if you need to take dependencies on other modules outside those two paths. 
 
-## Trilogy Syntax Reference
+## MCP/Server
+
+Trilogy is straightforward to run as a server/MCP server; the former to generate SQL on demand and integrate into other tools, and MCP
+for full interactive query loops.
+
+This makes it easy to integrate Trilogy into existing tools or workflows.
+
+You can see examples of both use cases in the trilogy-studio codebase [here](https://github.com/trilogy-data/trilogy-studio-core)
+and install and run an MCP server directly with that codebase.
+
+If you're interested in a more fleshed out standalone server or MCP server, please open an issue and we'll prioritize it!
+
+## Trilogy Syntax Reference 
+
+Not exhaustive - see [documentation](https://trilogydata.dev/) for more details.
 
 ### Import
 ```sql

{pytrilogy-0.0.3.96.dist-info → pytrilogy-0.0.3.97.dist-info}/RECORD

@@ -1,6 +1,6 @@
-pytrilogy-0.0.3.96.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=hBxtfxlbUvTLp_8FCY_-wDqJM7G2RJQ2jntfMf3a0PM,303
-trilogy/constants.py,sha256=eKb_EJvSqjN9tGbdVEViwdtwwh8fZ3-jpOEDqL71y70,1691
+pytrilogy-0.0.3.97.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=cYPR5Qx9Z_vJCc8nN83bLxuX2AWSJ8GNmHHEIFoyexA,303
+trilogy/constants.py,sha256=SSsRMg9HTou259nMKAw-rJNBgzkWjQ3QIQXcrq9i5Kk,1717
 trilogy/engine.py,sha256=3MiADf5MKcmxqiHBuRqiYdsXiLj7oitDfVvXvHrfjkA,2178
 trilogy/executor.py,sha256=YfSjuJ0FVm2gHnNgmUlXijWDTUFjqq9FNakWpeEYO48,15769
 trilogy/parser.py,sha256=o4cfk3j3yhUFoiDKq9ZX_GjBF3dKhDjXEwb63rcBkBM,293
@@ -10,11 +10,11 @@ trilogy/utility.py,sha256=euQccZLKoYBz0LNg5tzLlvv2YHvXh9HArnYp1V3uXsM,763
 trilogy/authoring/__init__.py,sha256=TABMOETSMERrWuyDLR0nK4ISlqR0yaqeXrmuOdrSvAY,3060
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=nizWYDCJQ1bigQMtkNIEMNTcN0NoEAXiIHLzpelxQ24,201
-trilogy/core/enums.py,sha256=EusAzz7o_YrWf64TLIED7MfziFOJk8EHM8se5W3nyJk,8644
+trilogy/core/enums.py,sha256=H8I2Dz4POHZ4ixYCGzNs4c3KDqxLQklGLVfmje1DSMo,8877
 trilogy/core/env_processor.py,sha256=H-rr2ALj31l5oh3FqeI47Qju6OOfiXBacXNJGNZ92zQ,4521
 trilogy/core/environment_helpers.py,sha256=TRlqVctqIRBxzfjRBmpQsAVoiCcsEKBhG1B6PUE0l1M,12743
 trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
-trilogy/core/exceptions.py,sha256=0Lmc3awJYx94k6uifbHc-EIqlFGV6YrX0QIwP84D4a4,1150
+trilogy/core/exceptions.py,sha256=fI16oTNCVMMAJFSn2AFzZVapzsF5M9WbdN5e5UixwXc,2807
 trilogy/core/functions.py,sha256=ESUWMRmwtavwCLl6z1NP9EFzWTJoXn3orTaaOSsj33Q,33093
 trilogy/core/graph_models.py,sha256=4EWFTHGfYd72zvS2HYoV6hm7nMC_VEd7vWr6txY-ig0,3400
 trilogy/core/internal.py,sha256=r9QagDB2GvpqlyD_I7VrsfbVfIk5mnok2znEbv72Aa4,2681
@@ -76,12 +76,13 @@ trilogy/core/statements/build.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 trilogy/core/statements/common.py,sha256=VnVLULQg1TJLNUFzJaROT1tsf2ewk3IpuhvZaP36R6A,535
 trilogy/core/statements/execute.py,sha256=kiwJcVeMa4wZR-xLfM2oYOJ9DeyJkP8An38WFyJxktM,2413
 trilogy/core/validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/validation/common.py,sha256=cVbDSowtLf2nl0-QVmNauAeLBBNFkSE5bRZtTHIzW20,3193
-trilogy/core/validation/concept.py,sha256=23wZYw_cGmTQuFvaRM-0T7M2b5ZwqjFMucfvfzyQxlc,4425
-trilogy/core/validation/datasource.py,sha256=HIk7iEKK99k0-WXosiIhedH-U2rDGYGdUMKEt1eMl1w,6394
-trilogy/core/validation/environment.py,sha256=yjSnEH893mTiW9o6YXBtXJWbGSR2kMQWjszDuECznLs,2784
+trilogy/core/validation/common.py,sha256=Sd-towAX1uSDe3dK51FcVtIwVrMhayEwdHqhzeJHro0,4776
+trilogy/core/validation/concept.py,sha256=Jr8H9wn0-OMJeSFD5GydiB-LvCSj4CUe72evHjRq85E,4991
+trilogy/core/validation/datasource.py,sha256=d9AQNcukIRgN2spItPsXFiNtlZva-lDnfei3i06yQCE,6489
+trilogy/core/validation/environment.py,sha256=waBPMB6rxa-9SKUAdhfT8UUDqh4a76n27pKtRDre560,2834
+trilogy/core/validation/fix.py,sha256=Z818UFNLxndMTLiyhB3doLxIfnOZ-16QGvVFWuD7UsA,3750
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=m2a8azbI3AWfQz-VtSn84H5T-BqjF5PULH6BrgwZzok,49666
+trilogy/dialect/base.py,sha256=0QVHv4F0t3_gRQrZ0woFoUNKu7vaXGo-BG1l47CZUKc,49698
 trilogy/dialect/bigquery.py,sha256=XS3hpybeowgfrOrkycAigAF3NX2YUzTzfgE6f__2fT4,4316
 trilogy/dialect/common.py,sha256=tSthIZOXXRPQ4KeMKnDDsH7KlTmf2EVqigVtLyoc4zc,6071
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
@@ -103,9 +104,9 @@ trilogy/parsing/common.py,sha256=550-L0444GUuBFdiDWkOg_DxnMXtcJFUMES2R5zlwik,310
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=Xwwsv2C9kSNv2q-HrrKC1f60JNHShXcCMzstTSEbiCw,154
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=fZGFNN7PEVJ7_tk0GxBHJQerIJTifbrjoMcHfTR_TYk,81647
-trilogy/parsing/render.py,sha256=HSNntD82GiiwHT-TWPLXAaIMWLYVV5B5zQEsgwrHIBE,19605
-trilogy/parsing/trilogy.lark,sha256=9-SMrKFGZLdBTNheK1szif0VYOIt5m0xhVd8pOFCByU,16267
+trilogy/parsing/parse_engine.py,sha256=Zd4Zpj8k_Q95onXlThV_AWSwjUAGDsD57uTyKKmXxNI,81784
+trilogy/parsing/render.py,sha256=OirN76I8z9xDNGvydO4DhsTsN-NS_1SC5_77_V23CkI,20515
+trilogy/parsing/trilogy.lark,sha256=rM4WleeyGhoRgU-FOGcaeHOzZcYVxN4f13e_3B4OeLQ,16389
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -116,8 +117,8 @@ trilogy/std/money.preql,sha256=XWwvAV3WxBsHX9zfptoYRnBigcfYwrYtBHXTME0xJuQ,2082
 trilogy/std/net.preql,sha256=WZCuvH87_rZntZiuGJMmBDMVKkdhTtxeHOkrXNwJ1EE,416
 trilogy/std/ranking.preql,sha256=LDoZrYyz4g3xsII9XwXfmstZD-_92i1Eox1UqkBIfi8,83
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.96.dist-info/METADATA,sha256=l4yiGzDzMYd4of8nPXBMBKyetasPk4yfIOyENWjEqcU,11023
-pytrilogy-0.0.3.96.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pytrilogy-0.0.3.96.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.96.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.96.dist-info/RECORD,,
+pytrilogy-0.0.3.97.dist-info/METADATA,sha256=t-pwxx9XVd6oSuAMdPfTDHFaY7AevCESieu_mK6YrcU,11683
+pytrilogy-0.0.3.97.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytrilogy-0.0.3.97.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.97.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.97.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.96"
+__version__ = "0.0.3.97"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -16,6 +16,7 @@ ENV_CACHE_NAME = ".preql_cache.json"
 
 class MagicConstants(Enum):
     NULL = "null"
+    LINE_SEPARATOR = "\n"
 
 
 NULL_VALUE = MagicConstants.NULL

trilogy/core/enums.py

@@ -82,6 +82,15 @@ class Modifier(Enum):
             return Modifier.NULLABLE
         return super()._missing_(value=strval.capitalize())
 
+    def __lt__(self, other):
+        order = [
+            Modifier.HIDDEN,
+            Modifier.PARTIAL,
+            Modifier.NULLABLE,
+            Modifier.OPTIONAL,
+        ]
+        return order.index(self) < order.index(other)
+
 
 class JoinType(Enum):
     INNER = "inner"

trilogy/core/exceptions.py

@@ -1,4 +1,15 @@
-from typing import List, Sequence
+from dataclasses import dataclass
+from typing import Any, List, Sequence
+
+from trilogy.core.enums import Modifier
+from trilogy.core.models.core import (
+    ArrayType,
+    DataType,
+    MapType,
+    NumericType,
+    StructType,
+    TraitDataType,
+)
 
 
 class UndefinedConceptException(Exception):
@@ -29,7 +40,7 @@ class ModelValidationError(Exception):
         self,
         message,
         children: Sequence["ModelValidationError"] | None = None,
-        **kwargs
+        **kwargs,
     ):
         super().__init__(self, message, **kwargs)
         self.message = message
@@ -40,6 +51,49 @@ class DatasourceModelValidationError(ModelValidationError):
     pass
 
 
+class DatasourceGrainValidationError(DatasourceModelValidationError):
+    pass
+
+
+@dataclass
+class DatasourceColumnBindingData:
+    address: str
+    value: Any
+    value_type: (
+        DataType | ArrayType | StructType | MapType | NumericType | TraitDataType
+    )
+    value_modifiers: List[Modifier]
+    actual_type: (
+        DataType | ArrayType | StructType | MapType | NumericType | TraitDataType
+    )
+    actual_modifiers: List[Modifier]
+
+    def format_failure(self):
+        return f"Concept {self.address} value '{self.value}' with type {self.value_modifiers} does not conform to expected type {str(self.actual_type)} with modifiers {self.actual_modifiers}"
+
+    def is_modifier_issue(self) -> bool:
+        return len(self.value_modifiers) > 0 and any(
+            [x not in self.actual_modifiers for x in self.value_modifiers]
+        )
+
+    def is_type_issue(self) -> bool:
+        return self.value_type != self.actual_type
+
+
+class DatasourceColumnBindingError(DatasourceModelValidationError):
+    def __init__(
+        self,
+        address: str,
+        errors: list[DatasourceColumnBindingData],
+        message: str | None = None,
+    ):
+        if not message:
+            message = f"Datasource {address} failed validation. Found rows that do not conform to types: {[failure.format_failure() for failure in errors]}"
+        super().__init__(message)
+        self.errors = errors
+        self.dataset_address = address
+
+
 class ConceptModelValidationError(ModelValidationError):
     pass
 

trilogy/core/validation/common.py

@@ -2,13 +2,25 @@ from dataclasses import dataclass
 from enum import Enum
 
 from trilogy import Environment
-from trilogy.authoring import ConceptRef
+from trilogy.authoring import (
+    ConceptRef,
+    DataType,
+    Ordering,
+    Purpose,
+)
+from trilogy.constants import MagicConstants
+from trilogy.core.enums import ComparisonOperator, FunctionType
 from trilogy.core.exceptions import ModelValidationError
 from trilogy.core.models.build import (
+    BuildCaseElse,
+    BuildCaseWhen,
     BuildComparison,
     BuildConcept,
     BuildConditional,
     BuildDatasource,
+    BuildFunction,
+    BuildOrderBy,
+    BuildOrderItem,
 )
 from trilogy.core.models.environment import EnvironmentConceptDict
 from trilogy.core.models.execute import (
@@ -39,6 +51,32 @@ class ValidationType(Enum):
     CONCEPTS = "concepts"
 
 
+def build_order_args(concepts: list[BuildConcept]) -> list[BuildFunction]:
+    order_args = []
+    for concept in concepts:
+        order_args.append(
+            BuildFunction(
+                operator=FunctionType.CASE,
+                arguments=[
+                    BuildCaseWhen(
+                        comparison=BuildComparison(
+                            left=concept,
+                            operator=ComparisonOperator.IS,
+                            right=MagicConstants.NULL,
+                        ),
+                        expr=1,
+                    ),
+                    BuildCaseElse(expr=0),
+                ],
+                output_data_type=DataType.INTEGER,
+                output_purpose=Purpose.PROPERTY,
+                arg_count=2,
+            )
+        )
+
+    return order_args
+
+
 def easy_query(
     concepts: list[BuildConcept],
     datasource: BuildDatasource,
@@ -81,7 +119,6 @@ def easy_query(
         group_to_grain=True,
         base_alias_override=datasource.safe_identifier,
     )
-
     filter_cte = CTE(
         name=f"datasource_{datasource.name}_filter",
         source=QueryDatasource(
@@ -100,6 +137,20 @@ def easy_query(
         grain=cte.grain,
         condition=condition,
         limit=limit,
+        order_by=BuildOrderBy(
+            items=[
+                BuildOrderItem(
+                    expr=BuildFunction(
+                        operator=FunctionType.SUM,
+                        arguments=build_order_args(concepts),
+                        output_data_type=DataType.INTEGER,
+                        output_purpose=Purpose.PROPERTY,
+                        arg_count=len(concepts),
+                    ),
+                    order=Ordering.DESCENDING,
+                )
+            ]
+        ),
     )
 
     return ProcessedQuery(

trilogy/core/validation/concept.py

@@ -1,8 +1,9 @@
 from trilogy import Environment, Executor
-from trilogy.core.enums import Derivation, Purpose
+from trilogy.core.enums import Derivation, Modifier, Purpose
 from trilogy.core.exceptions import (
     ConceptModelValidationError,
-    DatasourceModelValidationError,
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
 )
 from trilogy.core.models.build import (
     BuildConcept,
@@ -69,8 +70,19 @@ def validate_key_concept(
             err = None
             datasource_count: int = seen.get(datasource.name, 0)
             if datasource_count < max_seen and assignment.is_complete:
-                err = DatasourceModelValidationError(
-                    f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial."
+                err = DatasourceColumnBindingError(
+                    address=datasource.identifier,
+                    errors=[
+                        DatasourceColumnBindingData(
+                            address=concept.address,
+                            value=None,
+                            value_type=concept.datatype,
+                            value_modifiers=[Modifier.PARTIAL],
+                            actual_type=concept.datatype,
+                            actual_modifiers=concept.modifiers,
+                        )
+                    ],
+                    message=f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial.",
                 )
             results.append(
                 ValidationTest(

trilogy/core/validation/datasource.py

@@ -10,9 +10,14 @@ from trilogy.authoring import (
     NumericType,
     StructType,
     TraitDataType,
+    arg_to_datatype,
+)
+from trilogy.core.enums import ComparisonOperator, Modifier
+from trilogy.core.exceptions import (
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
+    DatasourceModelValidationError,
 )
-from trilogy.core.enums import ComparisonOperator
-from trilogy.core.exceptions import DatasourceModelValidationError
 from trilogy.core.models.build import (
     BuildComparison,
     BuildDatasource,
@@ -64,6 +69,7 @@ def validate_datasource(
     env: Environment,
     build_env: BuildEnvironment,
     exec: Executor | None = None,
+    fix: bool = False,
 ) -> list[ValidationTest]:
     results: list[ValidationTest] = []
     # we might have merged concepts, where both will map out to the same
@@ -109,14 +115,7 @@ def validate_datasource(
             )
         )
         return results
-    failures: list[
-        tuple[
-            str,
-            Any,
-            DataType | ArrayType | StructType | MapType | NumericType | TraitDataType,
-            bool,
-        ]
-    ] = []
+    failures: list[DatasourceColumnBindingData] = []
     cols_with_error = set()
     for row in rows:
         for col in datasource.columns:
@@ -127,26 +126,29 @@ def validate_datasource(
             passed = type_check(rval, col.concept.datatype, col.is_nullable)
             if not passed:
                 failures.append(
-                    (
-                        col.concept.address,
-                        rval,
-                        col.concept.datatype,
-                        col.is_nullable,
+                    DatasourceColumnBindingData(
+                        address=col.concept.address,
+                        value=rval,
+                        value_type=(
+                            arg_to_datatype(rval)
+                            if rval is not None
+                            else col.concept.datatype
+                        ),
+                        value_modifiers=[Modifier.NULLABLE] if rval is None else [],
+                        actual_type=col.concept.datatype,
+                        actual_modifiers=col.concept.modifiers,
                     )
                 )
                 cols_with_error.add(actual_address)
 
-    def format_failure(failure):
-        return f"Concept {failure[0]} value '{failure[1]}' does not conform to expected type {str(failure[2])} (nullable={failure[3]})"
-
     if failures:
         results.append(
             ValidationTest(
                 check_type=ExpectationType.LOGICAL,
                 expected="datatype_match",
                 ran=True,
-                result=DatasourceModelValidationError(
-                    f"Datasource {datasource.name} failed validation. Found rows that do not conform to types: {[format_failure(failure) for failure in failures]}",
+                result=DatasourceColumnBindingError(
+                    address=datasource.identifier, errors=failures
                 ),
             )
         )

trilogy/core/validation/environment.py

@@ -15,9 +15,10 @@ def validate_environment(
     scope: ValidationScope = ValidationScope.ALL,
     targets: list[str] | None = None,
     exec: Executor | None = None,
+    generate_only: bool = False,
 ) -> list[ValidationTest]:
     # avoid mutating the environment for validation
-    generate_only = exec is None
+    generate_only = exec is None or generate_only
     env = env.duplicate()
     grain_check = function_to_concept(
         parent=Function(

trilogy/core/validation/fix.py

@@ -0,0 +1,106 @@
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from trilogy import Environment, Executor
+from trilogy.authoring import ConceptDeclarationStatement, Datasource
+from trilogy.core.exceptions import (
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
+)
+from trilogy.core.validation.environment import validate_environment
+from trilogy.parsing.render import Renderer
+
+
+def rewrite_file_with_errors(
+    statements: list[Any], errors: list[DatasourceColumnBindingError]
+):
+    renderer = Renderer()
+    output = []
+    ds_error_map: dict[str, list[DatasourceColumnBindingData]] = defaultdict(list)
+    concept_error_map: dict[str, list[DatasourceColumnBindingData]] = defaultdict(list)
+    for error in errors:
+        if isinstance(error, DatasourceColumnBindingError):
+            for x in error.errors:
+                if error.dataset_address not in ds_error_map:
+                    ds_error_map[error.dataset_address] = []
+                # this is by dataset address
+                if x.is_modifier_issue():
+                    ds_error_map[error.dataset_address].append(x)
+                # this is by column
+                if x.is_type_issue():
+                    concept_error_map[x.address].append(x)
+    for statement in statements:
+        if isinstance(statement, Datasource):
+            if statement.identifier in ds_error_map:
+                error_cols = ds_error_map[statement.identifier]
+                for col in statement.columns:
+                    if col.concept.address in [x.address for x in error_cols]:
+                        error_col = [
+                            x for x in error_cols if x.address == col.concept.address
+                        ][0]
+                        col.modifiers = list(
+                            set(col.modifiers + error_col.value_modifiers)
+                        )
+        elif isinstance(statement, ConceptDeclarationStatement):
+            if statement.concept.address in concept_error_map:
+                error_cols = concept_error_map[statement.concept.address]
+                statement.concept.datatype = error_cols[0].value_type
+        output.append(statement)
+
+    return renderer.render_statement_string(output)
+
+
+DEPTH_CUTOFF = 3
+
+
+def validate_and_rewrite(
+    input: Path | str, exec: Executor | None = None, depth: int = 0
+) -> str | None:
+    if depth > DEPTH_CUTOFF:
+        print(f"Reached depth cutoff of {DEPTH_CUTOFF}, stopping.")
+        return None
+    if isinstance(input, str):
+        raw = input
+        env = Environment()
+    else:
+        with open(input, "r") as f:
+            raw = f.read()
+        env = Environment(working_path=input.parent)
+    if exec:
+        env = exec.environment
+    env, statements = env.parse(raw)
+
+    validation_results = validate_environment(env, exec=exec, generate_only=True)
+
+    errors = [
+        x.result
+        for x in validation_results
+        if isinstance(x.result, DatasourceColumnBindingError)
+    ]
+
+    if not errors:
+        print("No validation errors found")
+        return None
+    print(
+        f"Found {len(errors)} validation errors, attempting to fix, current depth: {depth}..."
+    )
+    for error in errors:
+        for item in error.errors:
+            print(f"- {item.format_failure()}")
+
+    new_text = rewrite_file_with_errors(statements, errors)
+
+    while iteration := validate_and_rewrite(new_text, exec=exec, depth=depth + 1):
+        depth = depth + 1
+        if depth >= DEPTH_CUTOFF:
+            break
+        if iteration:
+            new_text = iteration
+        depth += 1
+    if isinstance(input, Path):
+        with open(input, "w") as f:
+            f.write(new_text)
+        return None
+    else:
+        return new_text

trilogy/dialect/base.py

@@ -761,6 +761,7 @@ class BaseDialect:
         elif isinstance(e, MagicConstants):
             if e == MagicConstants.NULL:
                 return "null"
+            return str(e.value)
         elif isinstance(e, date):
             return self.FUNCTION_MAP[FunctionType.DATE_LITERAL](e)
         elif isinstance(e, datetime):
@@ -1139,7 +1140,7 @@ class BaseDialect:
         if isinstance(query, ProcessedShowStatement):
             return ";\n".join(
                 [
-                    f'{self.EXPLAIN_KEYWORD} {self.compile_statement(x)}'
+                    f"{self.EXPLAIN_KEYWORD} {self.compile_statement(x)}"
                     for x in query.output_values
                     if isinstance(x, (ProcessedQuery, ProcessedCopyStatement))
                 ]

trilogy/parsing/parse_engine.py

@@ -379,14 +379,16 @@ class ParseToObjects(Transformer):
     def start(self, args):
         return args
 
+    def LINE_SEPARATOR(self, args):
+        return MagicConstants.LINE_SEPARATOR
+
     def block(self, args):
         output = args[0]
         if isinstance(output, ConceptDeclarationStatement):
-            if len(args) > 1 and isinstance(args[1], Comment):
-                output.concept.metadata.description = (
-                    output.concept.metadata.description
-                    or args[1].text.split("#")[1].strip()
-                )
+            if len(args) > 1 and args[1] != MagicConstants.LINE_SEPARATOR:
+                comments = [x for x in args[1:] if isinstance(x, Comment)]
+                merged = "\n".join([x.text.split("#")[1].rstrip() for x in comments])
+                output.concept.metadata.description = merged
         # this is a bad plan for now;
         # because a comment after an import statement is very common
         # and it's not intuitive that it modifies the import description
@@ -913,7 +915,7 @@ class ParseToObjects(Transformer):
         return Comment(text=args[0].value)
 
     def PARSE_COMMENT(self, args):
-        return Comment(text=args.value)
+        return Comment(text=args.value.rstrip())
 
     @v_args(meta=True)
     def select_transform(self, meta: Meta, args) -> ConceptTransform:

trilogy/parsing/render.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 from datetime import date, datetime
 from functools import singledispatchmethod
+from typing import Any
 
 from jinja2 import Template
 
@@ -12,6 +13,7 @@ from trilogy.core.models.author import (
     AlignItem,
     CaseElse,
     CaseWhen,
+    Comment,
     Comparison,
     Concept,
     ConceptRef,
@@ -83,6 +85,23 @@ class Renderer:
     def __init__(self, environment: Environment | None = None):
         self.environment = environment
 
+    def render_statement_string(self, list_of_statements: list[Any]) -> str:
+        new = []
+        last_statement_type = None
+        for stmt in list_of_statements:
+            stmt_type = type(stmt)
+            if last_statement_type is None:
+                pass
+            elif last_statement_type == Comment:
+                new.append("\n")
+            elif stmt_type != last_statement_type:
+                new.append("\n\n")
+            else:
+                new.append("\n")
+            new.append(Renderer().to_string(stmt))
+            last_statement_type = stmt_type
+        return "".join(new)
+
     @singledispatchmethod
     def to_string(self, arg):
         raise NotImplementedError("Cannot render type {}".format(type(arg)))
@@ -269,6 +288,8 @@ class Renderer:
     @to_string.register
     def _(self, arg: "Address"):
         if arg.is_query:
+            if arg.location.startswith("("):
+                return f"query '''{arg.location[1:-1]}'''"
             return f"query '''{arg.location}'''"
         return f"address {arg.location}"
 
@@ -286,7 +307,7 @@ class Renderer:
     def _(self, arg: "ColumnAssignment"):
         if arg.modifiers:
             modifiers = "".join(
-                [self.to_string(modifier) for modifier in arg.modifiers]
+                [self.to_string(modifier) for modifier in sorted(arg.modifiers)]
             )
         else:
             modifiers = ""
@@ -328,7 +349,7 @@ class Renderer:
         else:
             output = f"{concept.purpose.value} {namespace}{concept.name} <- {self.to_string(concept.lineage)};"
         if base_description:
-            output += f" # {base_description}"
+            output += f" #{base_description}"
         return output
 
     @to_string.register
@@ -428,6 +449,10 @@ class Renderer:
     def _(self, arg: "Comparison"):
         return f"{self.to_string(arg.left)} {arg.operator.value} {self.to_string(arg.right)}"
 
+    @to_string.register
+    def _(self, arg: "Comment"):
+        return f"{arg.text}"
+
     @to_string.register
     def _(self, arg: "WindowItem"):
         over = ",".join(self.to_string(c) for c in arg.over)
@@ -551,8 +576,10 @@ class Renderer:
     def _(self, arg: Modifier):
         if arg == Modifier.PARTIAL:
             return "~"
-        if arg == Modifier.HIDDEN:
+        elif arg == Modifier.HIDDEN:
             return "--"
+        elif arg == Modifier.NULLABLE:
+            return "?"
         return arg.value
 
     @to_string.register

trilogy/parsing/trilogy.lark

@@ -1,5 +1,5 @@
- !start: ( block | show_statement )*
-    block: statement _TERMINATOR PARSE_COMMENT?
+ !start: ( block | show_statement | PARSE_COMMENT )*
+    block: statement _TERMINATOR LINE_SEPARATOR? PARSE_COMMENT*
     ?statement: concept
     | datasource
     | function
@@ -14,9 +14,12 @@
     | rawsql_statement
     | validate_statement
     
-    _TERMINATOR:  ";"i /\s*/
+    _TERMINATOR:  ";"i
     
-    PARSE_COMMENT.1:   /#.*(\n|$)/ |  /\/\/.*\n/  
+    PARSE_COMMENT.1:   /#.*(\n|$)/ |  /\/\/.*(\n|$)/  
+
+    // when whitespace matters - comment placement
+    LINE_SEPARATOR.1: /[ \t\r\f\v]*\n+/
     
     // property display_name string
     concept_declaration: PURPOSE IDENTIFIER data_type concept_nullable_modifier? metadata?