pytrilogy 0.0.3.95__py3-none-any.whl → 0.0.3.97__py3-none-any.whl

trilogy/dialect/metadata.py

@@ -0,0 +1,233 @@
+from dataclasses import dataclass
+from typing import Any, List, Optional
+
+from trilogy.core.models.author import ConceptRef
+from trilogy.core.models.datasource import Datasource
+from trilogy.core.models.environment import Environment
+from trilogy.core.statements.author import (
+    ConceptDeclarationStatement,
+    ImportStatement,
+    MergeStatementV2,
+)
+from trilogy.core.statements.execute import (
+    ProcessedShowStatement,
+    ProcessedStaticValueOutput,
+    ProcessedValidateStatement,
+)
+from trilogy.core.validation.common import ValidationTest
+from trilogy.dialect.base import BaseDialect
+from trilogy.engine import ResultProtocol
+
+
+@dataclass
+class MockResult(ResultProtocol):
+    values: list["MockResultRow"]
+    columns: list[str]
+
+    def __init__(self, values: list[Any], columns: list[str]):
+        processed: list[MockResultRow] = []
+        for x in values:
+            if isinstance(x, dict):
+                processed.append(MockResultRow(x))
+            elif isinstance(x, MockResultRow):
+                processed.append(x)
+            else:
+                raise ValueError(
+                    f"Cannot process value of type {type(x)} in MockResult"
+                )
+        self.columns = columns
+        self.values = processed
+
+    def __iter__(self):
+        while self.values:
+            yield self.values.pop(0)
+
+    def fetchall(self):
+        return self.values
+
+    def fetchone(self):
+        if self.values:
+            return self.values.pop(0)
+        return None
+
+    def fetchmany(self, size: int):
+        rval = self.values[:size]
+        self.values = self.values[size:]
+        return rval
+
+    def keys(self):
+        return self.columns
+
+    def as_dict(self):
+        return [x.as_dict() if isinstance(x, MockResultRow) else x for x in self.values]
+
+
+@dataclass
+class MockResultRow:
+    _values: dict[str, Any]
+
+    def as_dict(self):
+        return self._values
+
+    def __str__(self) -> str:
+        return str(self._values)
+
+    def __repr__(self) -> str:
+        return repr(self._values)
+
+    def __getattr__(self, name: str) -> Any:
+        if name in self._values:
+            return self._values[name]
+        return super().__getattribute__(name)
+
+    def __getitem__(self, key: str) -> Any:
+        return self._values[key]
+
+    def values(self):
+        return self._values.values()
+
+    def keys(self):
+        return self._values.keys()
+
+
+def generate_result_set(
+    columns: List[ConceptRef], output_data: list[Any]
+) -> MockResult:
+    """Generate a mock result set from columns and output data."""
+    names = [x.address.replace(".", "_") for x in columns]
+    return MockResult(
+        values=[dict(zip(names, [row])) for row in output_data], columns=names
+    )
+
+
+def handle_concept_declaration(query: ConceptDeclarationStatement) -> MockResult:
+    """Handle concept declaration statements without execution."""
+    concept = query.concept
+    return MockResult(
+        [
+            {
+                "address": concept.address,
+                "type": concept.datatype.value,
+                "purpose": concept.purpose.value,
+                "derivation": concept.derivation.value,
+            }
+        ],
+        ["address", "type", "purpose", "derivation"],
+    )
+
+
+def handle_datasource(query: Datasource) -> MockResult:
+    """Handle datasource queries without execution."""
+    return MockResult(
+        [
+            {
+                "name": query.name,
+            }
+        ],
+        ["name"],
+    )
+
+
+def handle_import_statement(query: ImportStatement) -> MockResult:
+    """Handle import statements without execution."""
+    return MockResult(
+        [
+            {
+                "path": query.path,
+                "alias": query.alias,
+            }
+        ],
+        ["path", "alias"],
+    )
+
+
+def handle_merge_statement(
+    query: MergeStatementV2, environment: Environment
+) -> MockResult:
+    """Handle merge statements by updating environment and returning result."""
+    for concept in query.sources:
+        environment.merge_concept(
+            concept, query.targets[concept.address], modifiers=query.modifiers
+        )
+
+    return MockResult(
+        [
+            {
+                "sources": ",".join([x.address for x in query.sources]),
+                "targets": ",".join([x.address for _, x in query.targets.items()]),
+            }
+        ],
+        ["source", "target"],
+    )
+
+
+def handle_processed_show_statement(
+    query: ProcessedShowStatement, compiled_statements: list[str]
+) -> MockResult:
+    """Handle processed show statements without execution."""
+
+    return generate_result_set(query.output_columns, compiled_statements)
+
+
+def raw_validation_to_result(
+    raw: list[ValidationTest], generator: Optional[BaseDialect] = None
+) -> Optional[MockResult]:
+    """Convert raw validation tests to mock result."""
+    if not raw:
+        return None
+    output = []
+    for row in raw:
+        if row.raw_query and generator and not row.generated_query:
+            try:
+                row.generated_query = generator.compile_statement(row.raw_query)
+            except Exception as e:
+                row.generated_query = f"Error generating query: {e}"
+        output.append(
+            {
+                "check_type": row.check_type.value,
+                "expected": row.expected,
+                "result": str(row.result) if row.result else None,
+                "ran": row.ran,
+                "query": row.generated_query if row.generated_query else "",
+            }
+        )
+    return MockResult(output, ["check_type", "expected", "result", "ran", "query"])
+
+
+def handle_processed_validate_statement(
+    query: ProcessedValidateStatement, dialect: BaseDialect, validate_environment_func
+) -> Optional[MockResult]:
+    """Handle processed validate statements."""
+    results = validate_environment_func(query.scope, query.targets)
+    return raw_validation_to_result(results, dialect)
+
+
+def handle_show_statement_outputs(
+    statement: ProcessedShowStatement,
+    compiled_statements: list[str],
+    environment: Environment,
+    dialect: BaseDialect,
+) -> list[MockResult]:
+    """Handle show statement outputs without execution."""
+    output = []
+    for x in statement.output_values:
+        if isinstance(x, ProcessedStaticValueOutput):
+            output.append(generate_result_set(statement.output_columns, x.values))
+        elif compiled_statements:
+
+            output.append(
+                generate_result_set(
+                    statement.output_columns,
+                    compiled_statements,
+                )
+            )
+        elif isinstance(x, ProcessedValidateStatement):
+            from trilogy.core.validation.environment import validate_environment
+
+            raw = validate_environment(environment, x.scope, x.targets)
+            results = raw_validation_to_result(raw, dialect)
+            if results:
+                output.append(results)
+        else:
+            raise NotImplementedError(f"Cannot show type {type(x)} in show statement")
+    return output

trilogy/executor.py

@@ -1,4 +1,3 @@
-from dataclasses import dataclass
 from functools import singledispatchmethod
 from pathlib import Path
 from typing import Any, Generator, List, Optional
@@ -7,7 +6,7 @@ from sqlalchemy import text
 
 from trilogy.constants import MagicConstants, Rendering, logger
 from trilogy.core.enums import FunctionType, Granularity, IOType, ValidationScope
-from trilogy.core.models.author import Concept, ConceptRef, Function
+from trilogy.core.models.author import Concept, Function
 from trilogy.core.models.build import BuildFunction
 from trilogy.core.models.core import ListWrapper, MapWrapper
 from trilogy.core.models.datasource import Datasource
@@ -31,7 +30,6 @@ from trilogy.core.statements.execute import (
     ProcessedQueryPersist,
     ProcessedRawSQLStatement,
     ProcessedShowStatement,
-    ProcessedStaticValueOutput,
     ProcessedValidateStatement,
 )
 from trilogy.core.validation.common import (
@@ -39,82 +37,22 @@ from trilogy.core.validation.common import (
 )
 from trilogy.dialect.base import BaseDialect
 from trilogy.dialect.enums import Dialects
+from trilogy.dialect.metadata import (
+    generate_result_set,
+    handle_concept_declaration,
+    handle_datasource,
+    handle_import_statement,
+    handle_merge_statement,
+    handle_processed_show_statement,
+    handle_processed_validate_statement,
+    handle_show_statement_outputs,
+)
 from trilogy.engine import ExecutionEngine, ResultProtocol
 from trilogy.hooks.base_hook import BaseHook
 from trilogy.parser import parse_text
 from trilogy.render import get_dialect_generator
 
 
-@dataclass
-class MockResult(ResultProtocol):
-    values: list[Any]
-    columns: list[str]
-
-    def __init__(self, values: list[Any], columns: list[str]):
-        processed = []
-        for x in values:
-            if isinstance(x, dict):
-                processed.append(MockResultRow(x))
-            else:
-                processed.append(x)
-        self.columns = columns
-        self.values = processed
-
-    def __iter__(self):
-        while self.values:
-            yield self.values.pop(0)
-
-    def fetchall(self):
-        return self.values
-
-    def fetchone(self):
-        if self.values:
-            return self.values.pop(0)
-        return None
-
-    def fetchmany(self, size: int):
-        rval = self.values[:size]
-        self.values = self.values[size:]
-        return rval
-
-    def keys(self):
-        return self.columns
-
-
-@dataclass
-class MockResultRow:
-    _values: dict[str, Any]
-
-    def __str__(self) -> str:
-        return str(self._values)
-
-    def __repr__(self) -> str:
-        return repr(self._values)
-
-    def __getattr__(self, name: str) -> Any:
-        if name in self._values:
-            return self._values[name]
-        return super().__getattribute__(name)
-
-    def __getitem__(self, key: str) -> Any:
-        return self._values[key]
-
-    def values(self):
-        return self._values.values()
-
-    def keys(self):
-        return self._values.keys()
-
-
-def generate_result_set(
-    columns: List[ConceptRef], output_data: list[Any]
-) -> MockResult:
-    names = [x.address.replace(".", "_") for x in columns]
-    return MockResult(
-        values=[dict(zip(names, [row])) for row in output_data], columns=names
-    )
-
-
 class Executor(object):
     def __init__(
         self,
@@ -150,29 +88,11 @@ class Executor(object):
 
     @execute_query.register
     def _(self, query: ConceptDeclarationStatement) -> ResultProtocol | None:
-        concept = query.concept
-        return MockResult(
-            [
-                {
-                    "address": concept.address,
-                    "type": concept.datatype.value,
-                    "purpose": concept.purpose.value,
-                    "derivation": concept.derivation.value,
-                }
-            ],
-            ["address", "type", "purpose", "derivation"],
-        )
+        return handle_concept_declaration(query)
 
     @execute_query.register
     def _(self, query: Datasource) -> ResultProtocol | None:
-        return MockResult(
-            [
-                {
-                    "name": query.name,
-                }
-            ],
-            ["name"],
-        )
+        return handle_datasource(query)
 
     @execute_query.register
     def _(self, query: str) -> ResultProtocol | None:
@@ -208,66 +128,28 @@ class Executor(object):
 
     @execute_query.register
     def _(self, query: ProcessedShowStatement) -> ResultProtocol | None:
-        return generate_result_set(
-            query.output_columns,
+        return handle_processed_show_statement(
+            query,
             [
                 self.generator.compile_statement(x)
                 for x in query.output_values
-                if isinstance(x, ProcessedQuery)
+                if isinstance(x, (ProcessedQuery, ProcessedQueryPersist))
             ],
         )
 
-    def _raw_validation_to_result(
-        self, raw: list[ValidationTest]
-    ) -> Optional[ResultProtocol]:
-        if not raw:
-            return None
-        output = []
-        for row in raw:
-            output.append(
-                {
-                    "check_type": row.check_type.value,
-                    "expected": row.expected,
-                    "result": str(row.result) if row.result else None,
-                    "ran": row.ran,
-                    "query": row.query if row.query else "",
-                }
-            )
-        return MockResult(output, ["check_type", "expected", "result", "ran", "query"])
-
     @execute_query.register
     def _(self, query: ProcessedValidateStatement) -> ResultProtocol | None:
-        results = self.validate_environment(query.scope, query.targets)
-        return self._raw_validation_to_result(results)
+        return handle_processed_validate_statement(
+            query, self.generator, self.validate_environment
+        )
 
     @execute_query.register
     def _(self, query: ImportStatement) -> ResultProtocol | None:
-        return MockResult(
-            [
-                {
-                    "path": query.path,
-                    "alias": query.alias,
-                }
-            ],
-            ["path", "alias"],
-        )
+        return handle_import_statement(query)
 
     @execute_query.register
     def _(self, query: MergeStatementV2) -> ResultProtocol | None:
-        for concept in query.sources:
-            self.environment.merge_concept(
-                concept, query.targets[concept.address], modifiers=query.modifiers
-            )
-
-        return MockResult(
-            [
-                {
-                    "sources": ",".join([x.address for x in query.sources]),
-                    "targets": ",".join([x.address for _, x in query.targets.items()]),
-                }
-            ],
-            ["source", "target"],
-        )
+        return handle_merge_statement(query, self.environment)
 
     @execute_query.register
     def _(self, query: ProcessedRawSQLStatement) -> ResultProtocol | None:
@@ -516,29 +398,17 @@ class Executor(object):
         # connection = self.engine.connect()
         for statement in self.parse_text_generator(command):
             if isinstance(statement, ProcessedShowStatement):
-                for x in statement.output_values:
-                    if isinstance(x, ProcessedStaticValueOutput):
-                        output.append(
-                            generate_result_set(statement.output_columns, x.values)
-                        )
-                    elif isinstance(x, ProcessedQuery):
-                        output.append(
-                            generate_result_set(
-                                statement.output_columns,
-                                [self.generator.compile_statement(x)],
-                            )
-                        )
-                    elif isinstance(x, ProcessedValidateStatement):
-                        raw = self.validate_environment(
-                            x.scope, x.targets, generate_only=True
-                        )
-                        results = self._raw_validation_to_result(raw)
-                        if results:
-                            output.append(results)
-                    else:
-                        raise NotImplementedError(
-                            f"Cannot show type {type(x)} in show statement"
-                        )
+                results = handle_show_statement_outputs(
+                    statement,
+                    [
+                        self.generator.compile_statement(x)
+                        for x in statement.output_values
+                        if isinstance(x, (ProcessedQuery, ProcessedQueryPersist))
+                    ],
+                    self.environment,
+                    self.generator,
+                )
+                output.extend(results)
                 continue
             if non_interactive:
                 if not isinstance(
@@ -567,5 +437,5 @@ class Executor(object):
         from trilogy.core.validation.environment import validate_environment
 
         return validate_environment(
-            self.environment, self, scope, targets, generate_only
+            self.environment, scope, targets, exec=None if generate_only else self
         )

trilogy/parsing/parse_engine.py

@@ -379,14 +379,16 @@ class ParseToObjects(Transformer):
     def start(self, args):
         return args
 
+    def LINE_SEPARATOR(self, args):
+        return MagicConstants.LINE_SEPARATOR
+
     def block(self, args):
         output = args[0]
         if isinstance(output, ConceptDeclarationStatement):
-            if len(args) > 1 and isinstance(args[1], Comment):
-                output.concept.metadata.description = (
-                    output.concept.metadata.description
-                    or args[1].text.split("#")[1].strip()
-                )
+            if len(args) > 1 and args[1] != MagicConstants.LINE_SEPARATOR:
+                comments = [x for x in args[1:] if isinstance(x, Comment)]
+                merged = "\n".join([x.text.split("#")[1].rstrip() for x in comments])
+                output.concept.metadata.description = merged
         # this is a bad plan for now;
         # because a comment after an import statement is very common
         # and it's not intuitive that it modifies the import description
@@ -913,7 +915,7 @@ class ParseToObjects(Transformer):
         return Comment(text=args[0].value)
 
     def PARSE_COMMENT(self, args):
-        return Comment(text=args.value)
+        return Comment(text=args.value.rstrip())
 
     @v_args(meta=True)
     def select_transform(self, meta: Meta, args) -> ConceptTransform:

trilogy/parsing/render.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 from datetime import date, datetime
 from functools import singledispatchmethod
+from typing import Any
 
 from jinja2 import Template
 
@@ -12,6 +13,7 @@ from trilogy.core.models.author import (
     AlignItem,
     CaseElse,
     CaseWhen,
+    Comment,
     Comparison,
     Concept,
     ConceptRef,
@@ -83,6 +85,23 @@ class Renderer:
     def __init__(self, environment: Environment | None = None):
         self.environment = environment
 
+    def render_statement_string(self, list_of_statements: list[Any]) -> str:
+        new = []
+        last_statement_type = None
+        for stmt in list_of_statements:
+            stmt_type = type(stmt)
+            if last_statement_type is None:
+                pass
+            elif last_statement_type == Comment:
+                new.append("\n")
+            elif stmt_type != last_statement_type:
+                new.append("\n\n")
+            else:
+                new.append("\n")
+            new.append(Renderer().to_string(stmt))
+            last_statement_type = stmt_type
+        return "".join(new)
+
     @singledispatchmethod
     def to_string(self, arg):
         raise NotImplementedError("Cannot render type {}".format(type(arg)))
@@ -269,6 +288,8 @@ class Renderer:
     @to_string.register
     def _(self, arg: "Address"):
         if arg.is_query:
+            if arg.location.startswith("("):
+                return f"query '''{arg.location[1:-1]}'''"
             return f"query '''{arg.location}'''"
         return f"address {arg.location}"
 
@@ -286,7 +307,7 @@ class Renderer:
     def _(self, arg: "ColumnAssignment"):
         if arg.modifiers:
             modifiers = "".join(
-                [self.to_string(modifier) for modifier in arg.modifiers]
+                [self.to_string(modifier) for modifier in sorted(arg.modifiers)]
             )
         else:
             modifiers = ""
@@ -328,7 +349,7 @@ class Renderer:
         else:
             output = f"{concept.purpose.value} {namespace}{concept.name} <- {self.to_string(concept.lineage)};"
         if base_description:
-            output += f" # {base_description}"
+            output += f" #{base_description}"
         return output
 
     @to_string.register
@@ -428,6 +449,10 @@ class Renderer:
     def _(self, arg: "Comparison"):
         return f"{self.to_string(arg.left)} {arg.operator.value} {self.to_string(arg.right)}"
 
+    @to_string.register
+    def _(self, arg: "Comment"):
+        return f"{arg.text}"
+
     @to_string.register
     def _(self, arg: "WindowItem"):
         over = ",".join(self.to_string(c) for c in arg.over)
@@ -551,8 +576,10 @@ class Renderer:
     def _(self, arg: Modifier):
         if arg == Modifier.PARTIAL:
             return "~"
-        if arg == Modifier.HIDDEN:
+        elif arg == Modifier.HIDDEN:
             return "--"
+        elif arg == Modifier.NULLABLE:
+            return "?"
         return arg.value
 
     @to_string.register

trilogy/parsing/trilogy.lark

@@ -1,5 +1,5 @@
- !start: ( block | show_statement )*
-    block: statement _TERMINATOR PARSE_COMMENT?
+ !start: ( block | show_statement | PARSE_COMMENT )*
+    block: statement _TERMINATOR LINE_SEPARATOR? PARSE_COMMENT*
     ?statement: concept
     | datasource
     | function
@@ -14,9 +14,12 @@
     | rawsql_statement
     | validate_statement
     
-    _TERMINATOR:  ";"i /\s*/
+    _TERMINATOR:  ";"i
     
-    PARSE_COMMENT.1:   /#.*(\n|$)/ |  /\/\/.*\n/  
+    PARSE_COMMENT.1:   /#.*(\n|$)/ |  /\/\/.*(\n|$)/  
+
+    // when whitespace matters - comment placement
+    LINE_SEPARATOR.1: /[ \t\r\f\v]*\n+/
     
     // property display_name string
     concept_declaration: PURPOSE IDENTIFIER data_type concept_nullable_modifier? metadata?