pytrilogy 0.0.3.94__py3-none-any.whl → 0.0.3.96__py3-none-any.whl

trilogy/executor.py

@@ -1,14 +1,12 @@
-from dataclasses import dataclass
 from functools import singledispatchmethod
 from pathlib import Path
-from typing import Any, Generator, List, Optional, Protocol
+from typing import Any, Generator, List, Optional
 
 from sqlalchemy import text
-from sqlalchemy.engine import CursorResult
 
 from trilogy.constants import MagicConstants, Rendering, logger
-from trilogy.core.enums import FunctionType, Granularity, IOType
-from trilogy.core.models.author import Concept, ConceptRef, Function
+from trilogy.core.enums import FunctionType, Granularity, IOType, ValidationScope
+from trilogy.core.models.author import Concept, Function
 from trilogy.core.models.build import BuildFunction
 from trilogy.core.models.core import ListWrapper, MapWrapper
 from trilogy.core.models.datasource import Datasource
@@ -23,53 +21,38 @@ from trilogy.core.statements.author import (
     RawSQLStatement,
     SelectStatement,
     ShowStatement,
+    ValidateStatement,
 )
 from trilogy.core.statements.execute import (
+    PROCESSED_STATEMENT_TYPES,
     ProcessedCopyStatement,
     ProcessedQuery,
     ProcessedQueryPersist,
     ProcessedRawSQLStatement,
     ProcessedShowStatement,
-    ProcessedStaticValueOutput,
+    ProcessedValidateStatement,
+)
+from trilogy.core.validation.common import (
+    ValidationTest,
 )
 from trilogy.dialect.base import BaseDialect
 from trilogy.dialect.enums import Dialects
-from trilogy.engine import ExecutionEngine
+from trilogy.dialect.metadata import (
+    generate_result_set,
+    handle_concept_declaration,
+    handle_datasource,
+    handle_import_statement,
+    handle_merge_statement,
+    handle_processed_show_statement,
+    handle_processed_validate_statement,
+    handle_show_statement_outputs,
+)
+from trilogy.engine import ExecutionEngine, ResultProtocol
 from trilogy.hooks.base_hook import BaseHook
 from trilogy.parser import parse_text
 from trilogy.render import get_dialect_generator
 
 
-class ResultProtocol(Protocol):
-    values: List[Any]
-    columns: List[str]
-
-    def fetchall(self) -> List[Any]: ...
-
-    def keys(self) -> List[str]: ...
-
-
-@dataclass
-class MockResult:
-    values: list[Any]
-    columns: list[str]
-
-    def fetchall(self):
-        return self.values
-
-    def keys(self):
-        return self.columns
-
-
-def generate_result_set(
-    columns: List[ConceptRef], output_data: list[Any]
-) -> MockResult:
-    names = [x.address.replace(".", "_") for x in columns]
-    return MockResult(
-        values=[dict(zip(names, [row])) for row in output_data], columns=names
-    )
-
-
 class Executor(object):
     def __init__(
         self,
@@ -93,141 +76,93 @@ class Executor(object):
 
     def execute_statement(
         self,
-        statement: (
-            ProcessedQuery
-            | ProcessedCopyStatement
-            | ProcessedRawSQLStatement
-            | ProcessedQueryPersist
-            | ProcessedShowStatement
-        ),
-    ) -> Optional[CursorResult]:
-        if not isinstance(
-            statement,
-            (
-                ProcessedQuery,
-                ProcessedShowStatement,
-                ProcessedQueryPersist,
-                ProcessedCopyStatement,
-                ProcessedRawSQLStatement,
-            ),
-        ):
+        statement: PROCESSED_STATEMENT_TYPES,
+    ) -> Optional[ResultProtocol]:
+        if not isinstance(statement, PROCESSED_STATEMENT_TYPES):
             return None
         return self.execute_query(statement)
 
     @singledispatchmethod
-    def execute_query(self, query) -> CursorResult:
+    def execute_query(self, query) -> ResultProtocol | None:
         raise NotImplementedError("Cannot execute type {}".format(type(query)))
 
     @execute_query.register
-    def _(self, query: ConceptDeclarationStatement) -> CursorResult:
-        concept = query.concept
-        return MockResult(
-            [
-                {
-                    "address": concept.address,
-                    "type": concept.datatype.value,
-                    "purpose": concept.purpose.value,
-                    "derivation": concept.derivation.value,
-                }
-            ],
-            ["address", "type", "purpose", "derivation"],
-        )
+    def _(self, query: ConceptDeclarationStatement) -> ResultProtocol | None:
+        return handle_concept_declaration(query)
 
     @execute_query.register
-    def _(self, query: Datasource) -> CursorResult:
-        return MockResult(
-            [
-                {
-                    "name": query.name,
-                }
-            ],
-            ["name"],
-        )
+    def _(self, query: Datasource) -> ResultProtocol | None:
+        return handle_datasource(query)
 
     @execute_query.register
-    def _(self, query: str) -> CursorResult | None:
+    def _(self, query: str) -> ResultProtocol | None:
         results = self.execute_text(query)
         if results:
             return results[-1]
         return None
 
     @execute_query.register
-    def _(self, query: SelectStatement) -> CursorResult:
+    def _(self, query: SelectStatement) -> ResultProtocol | None:
         sql = self.generator.generate_queries(
             self.environment, [query], hooks=self.hooks
         )
         return self.execute_query(sql[0])
 
     @execute_query.register
-    def _(self, query: PersistStatement) -> CursorResult:
+    def _(self, query: PersistStatement) -> ResultProtocol | None:
         sql = self.generator.generate_queries(
             self.environment, [query], hooks=self.hooks
         )
         return self.execute_query(sql[0])
 
     @execute_query.register
-    def _(self, query: RawSQLStatement) -> CursorResult:
+    def _(self, query: RawSQLStatement) -> ResultProtocol | None:
         return self.execute_raw_sql(query.text)
 
     @execute_query.register
-    def _(self, query: ShowStatement) -> CursorResult:
+    def _(self, query: ShowStatement) -> ResultProtocol | None:
         sql = self.generator.generate_queries(
             self.environment, [query], hooks=self.hooks
         )
         return self.execute_query(sql[0])
 
     @execute_query.register
-    def _(self, query: ProcessedShowStatement) -> CursorResult:
-        return generate_result_set(
-            query.output_columns,
+    def _(self, query: ProcessedShowStatement) -> ResultProtocol | None:
+        return handle_processed_show_statement(
+            query,
             [
                 self.generator.compile_statement(x)
                 for x in query.output_values
-                if isinstance(x, ProcessedQuery)
+                if isinstance(x, (ProcessedQuery, ProcessedQueryPersist))
             ],
         )
 
     @execute_query.register
-    def _(self, query: ImportStatement) -> CursorResult:
-        return MockResult(
-            [
-                {
-                    "path": query.path,
-                    "alias": query.alias,
-                }
-            ],
-            ["path", "alias"],
+    def _(self, query: ProcessedValidateStatement) -> ResultProtocol | None:
+        return handle_processed_validate_statement(
+            query, self.generator, self.validate_environment
         )
 
     @execute_query.register
-    def _(self, query: MergeStatementV2) -> CursorResult:
-        for concept in query.sources:
-            self.environment.merge_concept(
-                concept, query.targets[concept.address], modifiers=query.modifiers
-            )
+    def _(self, query: ImportStatement) -> ResultProtocol | None:
+        return handle_import_statement(query)
 
-        return MockResult(
-            [
-                {
-                    "sources": ",".join([x.address for x in query.sources]),
-                    "targets": ",".join([x.address for _, x in query.targets.items()]),
-                }
-            ],
-            ["source", "target"],
-        )
+    @execute_query.register
+    def _(self, query: MergeStatementV2) -> ResultProtocol | None:
+        return handle_merge_statement(query, self.environment)
 
     @execute_query.register
-    def _(self, query: ProcessedRawSQLStatement) -> CursorResult:
+    def _(self, query: ProcessedRawSQLStatement) -> ResultProtocol | None:
         return self.execute_raw_sql(query.text)
 
     @execute_query.register
-    def _(self, query: ProcessedQuery) -> CursorResult:
+    def _(self, query: ProcessedQuery) -> ResultProtocol | None:
         sql = self.generator.compile_statement(query)
         output = self.execute_raw_sql(sql, local_concepts=query.local_concepts)
         return output
 
     @execute_query.register
-    def _(self, query: ProcessedQueryPersist) -> CursorResult:
+    def _(self, query: ProcessedQueryPersist) -> ResultProtocol | None:
         sql = self.generator.compile_statement(query)
 
         output = self.execute_raw_sql(sql, local_concepts=query.local_concepts)
@@ -235,9 +170,9 @@ class Executor(object):
         return output
 
     @execute_query.register
-    def _(self, query: ProcessedCopyStatement) -> CursorResult:
+    def _(self, query: ProcessedCopyStatement) -> ResultProtocol | None:
         sql = self.generator.compile_statement(query)
-        output: CursorResult = self.execute_raw_sql(
+        output: ResultProtocol = self.execute_raw_sql(
             sql, local_concepts=query.local_concepts
         )
         if query.target_type == IOType.CSV:
@@ -278,8 +213,6 @@ class Executor(object):
         for statement in sql:
             compiled_sql = self.generator.compile_statement(statement)
             output.append(compiled_sql)
-
-        output.append(compiled_sql)
         return output
 
     @generate_sql.register  # type: ignore
@@ -313,23 +246,13 @@ class Executor(object):
 
     def parse_file(
         self, file: str | Path, persist: bool = False
-    ) -> list[
-        ProcessedQuery
-        | ProcessedQueryPersist
-        | ProcessedShowStatement
-        | ProcessedRawSQLStatement
-        | ProcessedCopyStatement,
-    ]:
+    ) -> list[PROCESSED_STATEMENT_TYPES]:
         return list(self.parse_file_generator(file, persist=persist))
 
     def parse_file_generator(
         self, file: str | Path, persist: bool = False
     ) -> Generator[
-        ProcessedQuery
-        | ProcessedQueryPersist
-        | ProcessedShowStatement
-        | ProcessedRawSQLStatement
-        | ProcessedCopyStatement,
+        PROCESSED_STATEMENT_TYPES,
         None,
         None,
     ]:
@@ -340,23 +263,13 @@ class Executor(object):
 
     def parse_text(
         self, command: str, persist: bool = False, root: Path | None = None
-    ) -> List[
-        ProcessedQuery
-        | ProcessedQueryPersist
-        | ProcessedShowStatement
-        | ProcessedRawSQLStatement
-        | ProcessedCopyStatement
-    ]:
+    ) -> List[PROCESSED_STATEMENT_TYPES]:
         return list(self.parse_text_generator(command, persist=persist, root=root))
 
     def parse_text_generator(
         self, command: str, persist: bool = False, root: Path | None = None
     ) -> Generator[
-        ProcessedQuery
-        | ProcessedQueryPersist
-        | ProcessedShowStatement
-        | ProcessedRawSQLStatement
-        | ProcessedCopyStatement,
+        PROCESSED_STATEMENT_TYPES,
         None,
         None,
     ]:
@@ -374,6 +287,7 @@ class Executor(object):
                     ShowStatement,
                     RawSQLStatement,
                     CopyStatement,
+                    ValidateStatement,
                 ),
             )
         ]
@@ -420,10 +334,12 @@ class Executor(object):
             #     return self._concept_to_value(self.environment.concepts[rval.address], local_concepts=local_concepts)
             return rval
         else:
-            results = self.execute_query(f"select {concept.name} limit 1;").fetchone()
-        if not results:
+            results = self.execute_query(f"select {concept.name} limit 1;")
+            if results:
+                fetcher = results.fetchone()
+                if fetcher:
+                    return fetcher[0]
             return None
-        return results[0]
 
     def _hydrate_param(
         self, param: str, local_concepts: dict[str, Concept] | None = None
@@ -447,13 +363,16 @@ class Executor(object):
 
     def execute_raw_sql(
         self,
-        command: str,
+        command: str | Path,
         variables: dict | None = None,
         local_concepts: dict[str, Concept] | None = None,
-    ) -> CursorResult:
+    ) -> ResultProtocol:
         """Run a command against the raw underlying
         execution engine."""
         final_params = None
+        if isinstance(command, Path):
+            with open(command, "r") as f:
+                command = f.read()
         q = text(command)
         if variables:
             final_params = variables
@@ -473,37 +392,50 @@ class Executor(object):
 
     def execute_text(
         self, command: str, non_interactive: bool = False
-    ) -> List[CursorResult]:
+    ) -> List[ResultProtocol]:
         """Run a trilogy query expressed as text."""
-        output = []
+        output: list[ResultProtocol] = []
         # connection = self.engine.connect()
         for statement in self.parse_text_generator(command):
             if isinstance(statement, ProcessedShowStatement):
-                for x in statement.output_values:
-                    if isinstance(x, ProcessedStaticValueOutput):
-                        output.append(
-                            generate_result_set(statement.output_columns, x.values)
-                        )
-                    elif isinstance(x, ProcessedQuery):
-                        output.append(
-                            generate_result_set(
-                                statement.output_columns,
-                                [self.generator.compile_statement(x)],
-                            )
-                        )
+                results = handle_show_statement_outputs(
+                    statement,
+                    [
+                        self.generator.compile_statement(x)
+                        for x in statement.output_values
+                        if isinstance(x, (ProcessedQuery, ProcessedQueryPersist))
+                    ],
+                    self.environment,
+                    self.generator,
+                )
+                output.extend(results)
                 continue
             if non_interactive:
                 if not isinstance(
                     statement, (ProcessedCopyStatement, ProcessedQueryPersist)
                 ):
                     continue
-            output.append(self.execute_query(statement))
+            result = self.execute_statement(statement)
+            if result:
+                output.append(result)
         return output
 
     def execute_file(
         self, file: str | Path, non_interactive: bool = False
-    ) -> List[CursorResult]:
+    ) -> List[ResultProtocol]:
         file = Path(file)
         with open(file, "r") as f:
             command = f.read()
         return self.execute_text(command, non_interactive=non_interactive)
+
+    def validate_environment(
+        self,
+        scope: ValidationScope = ValidationScope.ALL,
+        targets: Optional[List[str]] = None,
+        generate_only: bool = False,
+    ) -> list[ValidationTest]:
+        from trilogy.core.validation.environment import validate_environment
+
+        return validate_environment(
+            self.environment, scope, targets, exec=None if generate_only else self
+        )

trilogy/parsing/parse_engine.py

@@ -38,13 +38,13 @@ from trilogy.core.enums import (
     Ordering,
     Purpose,
     ShowCategory,
+    ValidationScope,
     WindowOrder,
     WindowType,
 )
 from trilogy.core.exceptions import InvalidSyntaxException, UndefinedConceptException
 from trilogy.core.functions import (
     CurrentDate,
-    CurrentDatetime,
     FunctionFactory,
 )
 from trilogy.core.internal import ALL_ROWS_CONCEPT, INTERNAL_NAMESPACE
@@ -129,6 +129,7 @@ from trilogy.core.statements.author import (
     SelectStatement,
     ShowStatement,
     TypeDeclaration,
+    ValidateStatement,
 )
 from trilogy.parsing.common import (
     align_item_to_concept,
@@ -819,7 +820,9 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def aggregate_by(self, meta: Meta, args):
-        args = [self.environment.concepts[a] for a in args]
+        base = args[0]
+        b_concept = base.value.split(" ")[-1]
+        args = [self.environment.concepts[a] for a in [b_concept] + args[1:]]
         return self.function_factory.create_function(args, FunctionType.GROUP, meta)
 
     def whole_grain_clause(self, args) -> WholeGrainWrapper:
@@ -990,6 +993,25 @@ class ParseToObjects(Transformer):
     def over_list(self, args):
         return [x for x in args]
 
+    def VALIDATION_SCOPE(self, args) -> ValidationScope:
+        return ValidationScope(args.lower())
+
+    @v_args(meta=True)
+    def validate_statement(self, meta: Meta, args) -> ValidateStatement:
+        if len(args) == 2:
+            scope = args[0]
+            targets = args[1]
+        elif len(args) == 0:
+            scope = ValidationScope.ALL
+            targets = None
+        else:
+            scope = args[0]
+            targets = None
+        return ValidateStatement(
+            scope=scope,
+            targets=targets,
+        )
+
     @v_args(meta=True)
     def merge_statement(self, meta: Meta, args) -> MergeStatementV2 | None:
         modifiers = []
@@ -2055,7 +2077,15 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def fcurrent_datetime(self, meta, args):
-        return CurrentDatetime([])
+        return self.function_factory.create_function(
+            args=[], operator=FunctionType.CURRENT_DATETIME, meta=meta
+        )
+
+    @v_args(meta=True)
+    def fcurrent_timestamp(self, meta, args):
+        return self.function_factory.create_function(
+            args=[], operator=FunctionType.CURRENT_TIMESTAMP, meta=meta
+        )
 
     @v_args(meta=True)
     def fnot(self, meta, args):
@@ -2232,6 +2262,7 @@ def parse_text(
         | PersistStatement
         | ShowStatement
         | RawSQLStatement
+        | ValidateStatement
         | None
     ],
 ]:

trilogy/parsing/trilogy.lark

@@ -12,6 +12,7 @@
     | copy_statement
     | merge_statement
     | rawsql_statement
+    | validate_statement
     
     _TERMINATOR:  ";"i /\s*/
     
@@ -81,8 +82,11 @@
     // raw sql statement
     rawsql_statement: "raw_sql"i "(" MULTILINE_STRING ")"
 
+    // validate_statement
 
+    VALIDATE_SCOPE: "concepts"i | "datasources"i 
 
+    validate_statement: ("validate"i "all"i) | ( "validate"i VALIDATE_SCOPE (IDENTIFIER ("," IDENTIFIER)* ","? )? )
     // copy statement
 
     COPY_TYPE: "csv"i
@@ -262,10 +266,12 @@
     //constant
     CURRENT_DATE.1: /current_date\(\)/
     CURRENT_DATETIME.1: /current_datetime\(\)/
+    CURRENT_TIMESTAMP.1: /current_timestamp\(\)/
     fcurrent_date: CURRENT_DATE
     fcurrent_datetime: CURRENT_DATETIME
+    fcurrent_timestamp: CURRENT_TIMESTAMP
 
-    _constant_functions: fcurrent_date | fcurrent_datetime
+    _constant_functions: fcurrent_date | fcurrent_datetime | fcurrent_timestamp
     
     //string
     _LIKE.1: "like("i
@@ -325,8 +331,8 @@
     _GROUP.1: "group("i
     fgroup: _GROUP expr ")" aggregate_over?
 
-    //by:
-    aggregate_by: "group" IDENTIFIER "BY"i (IDENTIFIER ",")* IDENTIFIER
+    //by:  
+    aggregate_by: /(group)\s+([a-zA-Z\_][a-zA-Z0-9\_\.]*)/i "BY"i (IDENTIFIER ",")* IDENTIFIER
 
     //aggregates
     _COUNT.1: "count("i
@@ -424,7 +430,7 @@
     map_lit: "{" (literal ":" literal ",")* literal ":" literal ","? "}"
 
     _STRUCT.1: "struct("i
-    struct_lit: _STRUCT (IDENTIFIER "=" literal ",")* IDENTIFIER "=" literal ","? ")"
+    struct_lit: _STRUCT (IDENTIFIER "->" expr ",")* IDENTIFIER "->" expr ","? ")"
     
     !bool_lit: "True"i | "False"i
 
@@ -455,7 +461,7 @@
     DATASOURCES: "DATASOURCES"i
     show_category: CONCEPTS | DATASOURCES
 
-    show_statement: "show"i ( show_category | select_statement | persist_statement) _TERMINATOR
+    show_statement: "show"i ( show_category | validate_statement | select_statement | persist_statement) _TERMINATOR
     COMMENT: /#.*(\n|$)/ |  /\/\/.*\n/  
     %import common.WS
     %ignore WS