pytrilogy 0.0.3.96__tar.gz → 0.0.3.97__tar.gz

{pytrilogy-0.0.3.96/pytrilogy.egg-info → pytrilogy-0.0.3.97}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.96
+Version: 0.0.3.97
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -334,7 +334,21 @@ from pytrilogy.authoring import Concept, Function, ...
 
 Are likely to be unstable. Open an issue if you need to take dependencies on other modules outside those two paths. 
 
-## Trilogy Syntax Reference
+## MCP/Server
+
+Trilogy is straightforward to run as a server/MCP server; the former to generate SQL on demand and integrate into other tools, and MCP
+for full interactive query loops.
+
+This makes it easy to integrate Trilogy into existing tools or workflows.
+
+You can see examples of both use cases in the trilogy-studio codebase [here](https://github.com/trilogy-data/trilogy-studio-core)
+and install and run an MCP server directly with that codebase.
+
+If you're interested in a more fleshed out standalone server or MCP server, please open an issue and we'll prioritize it!
+
+## Trilogy Syntax Reference 
+
+Not exhaustive - see [documentation](https://trilogydata.dev/) for more details.
 
 ### Import
 ```sql

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/README.md

@@ -296,7 +296,21 @@ from pytrilogy.authoring import Concept, Function, ...
 
 Are likely to be unstable. Open an issue if you need to take dependencies on other modules outside those two paths. 
 
-## Trilogy Syntax Reference
+## MCP/Server
+
+Trilogy is straightforward to run as a server/MCP server; the former to generate SQL on demand and integrate into other tools, and MCP
+for full interactive query loops.
+
+This makes it easy to integrate Trilogy into existing tools or workflows.
+
+You can see examples of both use cases in the trilogy-studio codebase [here](https://github.com/trilogy-data/trilogy-studio-core)
+and install and run an MCP server directly with that codebase.
+
+If you're interested in a more fleshed out standalone server or MCP server, please open an issue and we'll prioritize it!
+
+## Trilogy Syntax Reference 
+
+Not exhaustive - see [documentation](https://trilogydata.dev/) for more details.
 
 ### Import
 ```sql

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.96
+Version: 0.0.3.97
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -334,7 +334,21 @@ from pytrilogy.authoring import Concept, Function, ...
 
 Are likely to be unstable. Open an issue if you need to take dependencies on other modules outside those two paths. 
 
-## Trilogy Syntax Reference
+## MCP/Server
+
+Trilogy is straightforward to run as a server/MCP server; the former to generate SQL on demand and integrate into other tools, and MCP
+for full interactive query loops.
+
+This makes it easy to integrate Trilogy into existing tools or workflows.
+
+You can see examples of both use cases in the trilogy-studio codebase [here](https://github.com/trilogy-data/trilogy-studio-core)
+and install and run an MCP server directly with that codebase.
+
+If you're interested in a more fleshed out standalone server or MCP server, please open an issue and we'll prioritize it!
+
+## Trilogy Syntax Reference 
+
+Not exhaustive - see [documentation](https://trilogydata.dev/) for more details.
 
 ### Import
 ```sql

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/pytrilogy.egg-info/SOURCES.txt

@@ -116,6 +116,7 @@ trilogy/core/validation/common.py
 trilogy/core/validation/concept.py
 trilogy/core/validation/datasource.py
 trilogy/core/validation/environment.py
+trilogy/core/validation/fix.py
 trilogy/dialect/__init__.py
 trilogy/dialect/base.py
 trilogy/dialect/bigquery.py

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/tests/test_parsing.py

@@ -321,6 +321,55 @@ def test_the_comments():
     assert rendered == "null"
 
 
+def test_the_comment_multiline():
+    env, parsed = parse_text(
+        """const
+         # comment here?
+           order_id <- 4;  # this is the order id
+        # order ids are important
+           
+           SELECT 
+        # TOOD - add in more columns?
+        order_id  
+        WHERE 
+        # order_id should not be null
+        order_id
+        # in this comp
+          is not 
+        null; # nulls are the worst
+        
+        """
+    )
+    parsed[-1]
+    assert env.concepts["order_id"].metadata.description is not None
+    assert " this is the order id" in env.concepts["order_id"].metadata.description
+    assert " order ids are important" in env.concepts["order_id"].metadata.description
+
+
+def test_the_comment_multiline_enter():
+    # we should not associate it as a description if there is a newline
+    env, parsed = parse_text(
+        """const
+         # comment here?
+           order_id <- 4;  
+        # this is the order id
+        # order ids are important
+           
+           SELECT 
+        # TOOD - add in more columns?
+        order_id  
+        WHERE 
+        # order_id should not be null
+        order_id
+        # in this comp
+          is not 
+        null; # nulls are the worst
+        
+        """
+    )
+    assert env.concepts["order_id"].metadata.description is None
+
+
 def test_purpose_nesting():
     env, parsed = parse_text(
         """key year int;

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.96"
+__version__ = "0.0.3.97"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/trilogy/constants.py

@@ -16,6 +16,7 @@ ENV_CACHE_NAME = ".preql_cache.json"
 
 class MagicConstants(Enum):
     NULL = "null"
+    LINE_SEPARATOR = "\n"
 
 
 NULL_VALUE = MagicConstants.NULL

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/trilogy/core/enums.py

@@ -82,6 +82,15 @@ class Modifier(Enum):
             return Modifier.NULLABLE
         return super()._missing_(value=strval.capitalize())
 
+    def __lt__(self, other):
+        order = [
+            Modifier.HIDDEN,
+            Modifier.PARTIAL,
+            Modifier.NULLABLE,
+            Modifier.OPTIONAL,
+        ]
+        return order.index(self) < order.index(other)
+
 
 class JoinType(Enum):
     INNER = "inner"

pytrilogy-0.0.3.97/trilogy/core/exceptions.py

@@ -0,0 +1,105 @@
+from dataclasses import dataclass
+from typing import Any, List, Sequence
+
+from trilogy.core.enums import Modifier
+from trilogy.core.models.core import (
+    ArrayType,
+    DataType,
+    MapType,
+    NumericType,
+    StructType,
+    TraitDataType,
+)
+
+
+class UndefinedConceptException(Exception):
+    def __init__(self, message, suggestions: List[str]):
+        super().__init__(self, message)
+        self.message = message
+        self.suggestions = suggestions
+
+
+class FrozenEnvironmentException(Exception):
+    pass
+
+
+class InvalidSyntaxException(Exception):
+    pass
+
+
+class UnresolvableQueryException(Exception):
+    pass
+
+
+class NoDatasourceException(UnresolvableQueryException):
+    pass
+
+
+class ModelValidationError(Exception):
+    def __init__(
+        self,
+        message,
+        children: Sequence["ModelValidationError"] | None = None,
+        **kwargs,
+    ):
+        super().__init__(self, message, **kwargs)
+        self.message = message
+        self.children = children
+
+
+class DatasourceModelValidationError(ModelValidationError):
+    pass
+
+
+class DatasourceGrainValidationError(DatasourceModelValidationError):
+    pass
+
+
+@dataclass
+class DatasourceColumnBindingData:
+    address: str
+    value: Any
+    value_type: (
+        DataType | ArrayType | StructType | MapType | NumericType | TraitDataType
+    )
+    value_modifiers: List[Modifier]
+    actual_type: (
+        DataType | ArrayType | StructType | MapType | NumericType | TraitDataType
+    )
+    actual_modifiers: List[Modifier]
+
+    def format_failure(self):
+        return f"Concept {self.address} value '{self.value}' with type {self.value_modifiers} does not conform to expected type {str(self.actual_type)} with modifiers {self.actual_modifiers}"
+
+    def is_modifier_issue(self) -> bool:
+        return len(self.value_modifiers) > 0 and any(
+            [x not in self.actual_modifiers for x in self.value_modifiers]
+        )
+
+    def is_type_issue(self) -> bool:
+        return self.value_type != self.actual_type
+
+
+class DatasourceColumnBindingError(DatasourceModelValidationError):
+    def __init__(
+        self,
+        address: str,
+        errors: list[DatasourceColumnBindingData],
+        message: str | None = None,
+    ):
+        if not message:
+            message = f"Datasource {address} failed validation. Found rows that do not conform to types: {[failure.format_failure() for failure in errors]}"
+        super().__init__(message)
+        self.errors = errors
+        self.dataset_address = address
+
+
+class ConceptModelValidationError(ModelValidationError):
+    pass
+
+
+class AmbiguousRelationshipResolutionException(UnresolvableQueryException):
+    def __init__(self, message, parents: List[set[str]]):
+        super().__init__(self, message)
+        self.message = message
+        self.parents = parents

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/trilogy/core/validation/common.py

@@ -2,13 +2,25 @@ from dataclasses import dataclass
 from enum import Enum
 
 from trilogy import Environment
-from trilogy.authoring import ConceptRef
+from trilogy.authoring import (
+    ConceptRef,
+    DataType,
+    Ordering,
+    Purpose,
+)
+from trilogy.constants import MagicConstants
+from trilogy.core.enums import ComparisonOperator, FunctionType
 from trilogy.core.exceptions import ModelValidationError
 from trilogy.core.models.build import (
+    BuildCaseElse,
+    BuildCaseWhen,
     BuildComparison,
     BuildConcept,
     BuildConditional,
     BuildDatasource,
+    BuildFunction,
+    BuildOrderBy,
+    BuildOrderItem,
 )
 from trilogy.core.models.environment import EnvironmentConceptDict
 from trilogy.core.models.execute import (
@@ -39,6 +51,32 @@ class ValidationType(Enum):
     CONCEPTS = "concepts"
 
 
+def build_order_args(concepts: list[BuildConcept]) -> list[BuildFunction]:
+    order_args = []
+    for concept in concepts:
+        order_args.append(
+            BuildFunction(
+                operator=FunctionType.CASE,
+                arguments=[
+                    BuildCaseWhen(
+                        comparison=BuildComparison(
+                            left=concept,
+                            operator=ComparisonOperator.IS,
+                            right=MagicConstants.NULL,
+                        ),
+                        expr=1,
+                    ),
+                    BuildCaseElse(expr=0),
+                ],
+                output_data_type=DataType.INTEGER,
+                output_purpose=Purpose.PROPERTY,
+                arg_count=2,
+            )
+        )
+
+    return order_args
+
+
 def easy_query(
     concepts: list[BuildConcept],
     datasource: BuildDatasource,
@@ -81,7 +119,6 @@ def easy_query(
         group_to_grain=True,
         base_alias_override=datasource.safe_identifier,
     )
-
     filter_cte = CTE(
         name=f"datasource_{datasource.name}_filter",
         source=QueryDatasource(
@@ -100,6 +137,20 @@ def easy_query(
         grain=cte.grain,
         condition=condition,
         limit=limit,
+        order_by=BuildOrderBy(
+            items=[
+                BuildOrderItem(
+                    expr=BuildFunction(
+                        operator=FunctionType.SUM,
+                        arguments=build_order_args(concepts),
+                        output_data_type=DataType.INTEGER,
+                        output_purpose=Purpose.PROPERTY,
+                        arg_count=len(concepts),
+                    ),
+                    order=Ordering.DESCENDING,
+                )
+            ]
+        ),
     )
 
     return ProcessedQuery(

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/trilogy/core/validation/concept.py

@@ -1,8 +1,9 @@
 from trilogy import Environment, Executor
-from trilogy.core.enums import Derivation, Purpose
+from trilogy.core.enums import Derivation, Modifier, Purpose
 from trilogy.core.exceptions import (
     ConceptModelValidationError,
-    DatasourceModelValidationError,
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
 )
 from trilogy.core.models.build import (
     BuildConcept,
@@ -69,8 +70,19 @@ def validate_key_concept(
             err = None
             datasource_count: int = seen.get(datasource.name, 0)
             if datasource_count < max_seen and assignment.is_complete:
-                err = DatasourceModelValidationError(
-                    f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial."
+                err = DatasourceColumnBindingError(
+                    address=datasource.identifier,
+                    errors=[
+                        DatasourceColumnBindingData(
+                            address=concept.address,
+                            value=None,
+                            value_type=concept.datatype,
+                            value_modifiers=[Modifier.PARTIAL],
+                            actual_type=concept.datatype,
+                            actual_modifiers=concept.modifiers,
+                        )
+                    ],
+                    message=f"Key concept {concept.address} is missing values in datasource {datasource.name} (max cardinality in data {max_seen}, datasource has {seen[datasource.name]} values) but is not marked as partial.",
                 )
             results.append(
                 ValidationTest(

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/trilogy/core/validation/datasource.py

@@ -10,9 +10,14 @@ from trilogy.authoring import (
     NumericType,
     StructType,
     TraitDataType,
+    arg_to_datatype,
+)
+from trilogy.core.enums import ComparisonOperator, Modifier
+from trilogy.core.exceptions import (
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
+    DatasourceModelValidationError,
 )
-from trilogy.core.enums import ComparisonOperator
-from trilogy.core.exceptions import DatasourceModelValidationError
 from trilogy.core.models.build import (
     BuildComparison,
     BuildDatasource,
@@ -64,6 +69,7 @@ def validate_datasource(
     env: Environment,
     build_env: BuildEnvironment,
     exec: Executor | None = None,
+    fix: bool = False,
 ) -> list[ValidationTest]:
     results: list[ValidationTest] = []
     # we might have merged concepts, where both will map out to the same
@@ -109,14 +115,7 @@ def validate_datasource(
             )
         )
         return results
-    failures: list[
-        tuple[
-            str,
-            Any,
-            DataType | ArrayType | StructType | MapType | NumericType | TraitDataType,
-            bool,
-        ]
-    ] = []
+    failures: list[DatasourceColumnBindingData] = []
     cols_with_error = set()
     for row in rows:
         for col in datasource.columns:
@@ -127,26 +126,29 @@ def validate_datasource(
             passed = type_check(rval, col.concept.datatype, col.is_nullable)
             if not passed:
                 failures.append(
-                    (
-                        col.concept.address,
-                        rval,
-                        col.concept.datatype,
-                        col.is_nullable,
+                    DatasourceColumnBindingData(
+                        address=col.concept.address,
+                        value=rval,
+                        value_type=(
+                            arg_to_datatype(rval)
+                            if rval is not None
+                            else col.concept.datatype
+                        ),
+                        value_modifiers=[Modifier.NULLABLE] if rval is None else [],
+                        actual_type=col.concept.datatype,
+                        actual_modifiers=col.concept.modifiers,
                     )
                 )
                 cols_with_error.add(actual_address)
 
-    def format_failure(failure):
-        return f"Concept {failure[0]} value '{failure[1]}' does not conform to expected type {str(failure[2])} (nullable={failure[3]})"
-
     if failures:
         results.append(
             ValidationTest(
                 check_type=ExpectationType.LOGICAL,
                 expected="datatype_match",
                 ran=True,
-                result=DatasourceModelValidationError(
-                    f"Datasource {datasource.name} failed validation. Found rows that do not conform to types: {[format_failure(failure) for failure in failures]}",
+                result=DatasourceColumnBindingError(
+                    address=datasource.identifier, errors=failures
                 ),
             )
         )

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/trilogy/core/validation/environment.py

@@ -15,9 +15,10 @@ def validate_environment(
     scope: ValidationScope = ValidationScope.ALL,
     targets: list[str] | None = None,
     exec: Executor | None = None,
+    generate_only: bool = False,
 ) -> list[ValidationTest]:
     # avoid mutating the environment for validation
-    generate_only = exec is None
+    generate_only = exec is None or generate_only
     env = env.duplicate()
     grain_check = function_to_concept(
         parent=Function(

pytrilogy-0.0.3.97/trilogy/core/validation/fix.py

@@ -0,0 +1,106 @@
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from trilogy import Environment, Executor
+from trilogy.authoring import ConceptDeclarationStatement, Datasource
+from trilogy.core.exceptions import (
+    DatasourceColumnBindingData,
+    DatasourceColumnBindingError,
+)
+from trilogy.core.validation.environment import validate_environment
+from trilogy.parsing.render import Renderer
+
+
+def rewrite_file_with_errors(
+    statements: list[Any], errors: list[DatasourceColumnBindingError]
+):
+    renderer = Renderer()
+    output = []
+    ds_error_map: dict[str, list[DatasourceColumnBindingData]] = defaultdict(list)
+    concept_error_map: dict[str, list[DatasourceColumnBindingData]] = defaultdict(list)
+    for error in errors:
+        if isinstance(error, DatasourceColumnBindingError):
+            for x in error.errors:
+                if error.dataset_address not in ds_error_map:
+                    ds_error_map[error.dataset_address] = []
+                # this is by dataset address
+                if x.is_modifier_issue():
+                    ds_error_map[error.dataset_address].append(x)
+                # this is by column
+                if x.is_type_issue():
+                    concept_error_map[x.address].append(x)
+    for statement in statements:
+        if isinstance(statement, Datasource):
+            if statement.identifier in ds_error_map:
+                error_cols = ds_error_map[statement.identifier]
+                for col in statement.columns:
+                    if col.concept.address in [x.address for x in error_cols]:
+                        error_col = [
+                            x for x in error_cols if x.address == col.concept.address
+                        ][0]
+                        col.modifiers = list(
+                            set(col.modifiers + error_col.value_modifiers)
+                        )
+        elif isinstance(statement, ConceptDeclarationStatement):
+            if statement.concept.address in concept_error_map:
+                error_cols = concept_error_map[statement.concept.address]
+                statement.concept.datatype = error_cols[0].value_type
+        output.append(statement)
+
+    return renderer.render_statement_string(output)
+
+
+DEPTH_CUTOFF = 3
+
+
+def validate_and_rewrite(
+    input: Path | str, exec: Executor | None = None, depth: int = 0
+) -> str | None:
+    if depth > DEPTH_CUTOFF:
+        print(f"Reached depth cutoff of {DEPTH_CUTOFF}, stopping.")
+        return None
+    if isinstance(input, str):
+        raw = input
+        env = Environment()
+    else:
+        with open(input, "r") as f:
+            raw = f.read()
+        env = Environment(working_path=input.parent)
+    if exec:
+        env = exec.environment
+    env, statements = env.parse(raw)
+
+    validation_results = validate_environment(env, exec=exec, generate_only=True)
+
+    errors = [
+        x.result
+        for x in validation_results
+        if isinstance(x.result, DatasourceColumnBindingError)
+    ]
+
+    if not errors:
+        print("No validation errors found")
+        return None
+    print(
+        f"Found {len(errors)} validation errors, attempting to fix, current depth: {depth}..."
+    )
+    for error in errors:
+        for item in error.errors:
+            print(f"- {item.format_failure()}")
+
+    new_text = rewrite_file_with_errors(statements, errors)
+
+    while iteration := validate_and_rewrite(new_text, exec=exec, depth=depth + 1):
+        depth = depth + 1
+        if depth >= DEPTH_CUTOFF:
+            break
+        if iteration:
+            new_text = iteration
+        depth += 1
+    if isinstance(input, Path):
+        with open(input, "w") as f:
+            f.write(new_text)
+        return None
+    else:
+        return new_text

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/trilogy/dialect/base.py

@@ -761,6 +761,7 @@ class BaseDialect:
         elif isinstance(e, MagicConstants):
             if e == MagicConstants.NULL:
                 return "null"
+            return str(e.value)
         elif isinstance(e, date):
             return self.FUNCTION_MAP[FunctionType.DATE_LITERAL](e)
         elif isinstance(e, datetime):
@@ -1139,7 +1140,7 @@ class BaseDialect:
         if isinstance(query, ProcessedShowStatement):
             return ";\n".join(
                 [
-                    f'{self.EXPLAIN_KEYWORD} {self.compile_statement(x)}'
+                    f"{self.EXPLAIN_KEYWORD} {self.compile_statement(x)}"
                     for x in query.output_values
                     if isinstance(x, (ProcessedQuery, ProcessedCopyStatement))
                 ]

{pytrilogy-0.0.3.96 → pytrilogy-0.0.3.97}/trilogy/parsing/parse_engine.py

@@ -379,14 +379,16 @@ class ParseToObjects(Transformer):
     def start(self, args):
         return args
 
+    def LINE_SEPARATOR(self, args):
+        return MagicConstants.LINE_SEPARATOR
+
     def block(self, args):
         output = args[0]
         if isinstance(output, ConceptDeclarationStatement):
-            if len(args) > 1 and isinstance(args[1], Comment):
-                output.concept.metadata.description = (
-                    output.concept.metadata.description
-                    or args[1].text.split("#")[1].strip()
-                )
+            if len(args) > 1 and args[1] != MagicConstants.LINE_SEPARATOR:
+                comments = [x for x in args[1:] if isinstance(x, Comment)]
+                merged = "\n".join([x.text.split("#")[1].rstrip() for x in comments])
+                output.concept.metadata.description = merged
         # this is a bad plan for now;
         # because a comment after an import statement is very common
         # and it's not intuitive that it modifies the import description
@@ -913,7 +915,7 @@ class ParseToObjects(Transformer):
         return Comment(text=args[0].value)
 
     def PARSE_COMMENT(self, args):
-        return Comment(text=args.value)
+        return Comment(text=args.value.rstrip())
 
     @v_args(meta=True)
     def select_transform(self, meta: Meta, args) -> ConceptTransform: