pytrilogy 0.0.2.36__tar.gz → 0.0.2.38__tar.gz

{pytrilogy-0.0.2.36/pytrilogy.egg-info → pytrilogy-0.0.2.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.36
+Version: 0.0.2.38
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.36 → pytrilogy-0.0.2.38/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.36
+Version: 0.0.2.38
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.36 → pytrilogy-0.0.2.38}/tests/test_select.py

@@ -126,7 +126,7 @@ def test_modifiers():
     generator = BigqueryDialect()
 
     text = generator.compile_statement(query)
-    assert "2 = 2" in text
+    assert ":b = 2" in text
     assert "as `b`" not in text
 
 

{pytrilogy-0.0.2.36 → pytrilogy-0.0.2.38}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.36"
+__version__ = "0.0.2.38"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.36 → pytrilogy-0.0.2.38}/trilogy/constants.py

@@ -39,6 +39,13 @@ class Comments:
     partial: bool = True
 
 
+@dataclass
+class Rendering:
+    """Control how the SQL is rendered"""
+
+    parameters: bool = True
+
+
 # TODO: support loading from environments
 @dataclass
 class Config:
@@ -48,6 +55,7 @@ class Config:
     validate_missing: bool = True
     comments: Comments = field(default_factory=Comments)
     optimizations: Optimizations = field(default_factory=Optimizations)
+    rendering: Rendering = field(default_factory=Rendering)
 
     @property
     def show_comments(self) -> bool:

{pytrilogy-0.0.2.36 → pytrilogy-0.0.2.38}/trilogy/core/models.py

@@ -2085,14 +2085,16 @@ class DatasourceMetadata(BaseModel):
 
 
 class MergeStatementV2(HasUUID, Namespaced, BaseModel):
-    source: Concept
-    target: Concept
+    sources: list[Concept]
+    targets: dict[str, Concept]
+    source_wildcard: str | None = None
+    target_wildcard: str | None = None
     modifiers: List[Modifier] = Field(default_factory=list)
 
     def with_namespace(self, namespace: str) -> "MergeStatementV2":
         new = MergeStatementV2(
-            source=self.source.with_namespace(namespace),
-            target=self.target.with_namespace(namespace),
+            sources=[x.with_namespace(namespace) for x in self.sources],
+            targets={k: v.with_namespace(namespace) for k, v in self.targets.items()},
             modifiers=self.modifiers,
         )
         return new
@@ -3381,6 +3383,21 @@ class Environment(BaseModel):
     materialized_concepts: List[Concept] = Field(default_factory=list)
     alias_origin_lookup: Dict[str, Concept] = Field(default_factory=dict)
 
+    def __init__(self, **data):
+        super().__init__(**data)
+        self.concepts["_env_working_path"] = Concept(
+            name="_env_working_path",
+            namespace=self.namespace,
+            lineage=Function(
+                operator=FunctionType.CONSTANT,
+                arguments=[str(self.working_path)],
+                output_datatype=DataType.STRING,
+                output_purpose=Purpose.CONSTANT,
+            ),
+            datatype=DataType.STRING,
+            purpose=Purpose.CONSTANT,
+        )
+
     @classmethod
     def from_file(cls, path: str | Path) -> "Environment":
         with open(path, "r") as f:
@@ -3554,16 +3571,7 @@ class Environment(BaseModel):
             target = target.with_suffix(".preql")
         else:
             target = path
-        if alias in self.imports:
-            imports = self.imports[alias]
-            for x in imports:
-                if x.path == target:
-                    return imports
-        if env:
-            self.imports[alias].append(
-                ImportStatement(alias=alias, path=target, environment=env)
-            )
-        else:
+        if not env:
             parse_address = gen_cache_lookup(str(target), alias, str(self.working_path))
             try:
                 with open(target, "r", encoding="utf-8") as f:
@@ -3587,13 +3595,8 @@ class Environment(BaseModel):
                     f"Unable to import file {target.parent}, parsing error: {e}"
                 )
             env = nparser.environment
-        for _, concept in env.concepts.items():
-            self.add_concept(concept.with_namespace(alias))
-
-        for _, datasource in env.datasources.items():
-            self.add_datasource(datasource.with_namespace(alias))
         imps = ImportStatement(alias=alias, path=target, environment=env)
-        self.imports[alias].append(imps)
+        self.add_import(alias, source=env, imp_stm=imps)
         return imps
 
     def parse(

{pytrilogy-0.0.2.36 → pytrilogy-0.0.2.38}/trilogy/dialect/base.py

@@ -337,6 +337,13 @@ class BaseDialect:
                         " target grain"
                     )
                     rval = f"{self.FUNCTION_GRAIN_MATCH_MAP[c.lineage.function.operator](args)}"
+            elif (
+                isinstance(c.lineage, Function)
+                and c.lineage.operator == FunctionType.CONSTANT
+                and CONFIG.rendering.parameters is True
+                and c.datatype.data_type != DataType.MAP
+            ):
+                rval = f":{c.safe_address}"
             else:
                 args = [
                     self.render_expr(
@@ -541,7 +548,7 @@ class BaseDialect:
         else:
             raise ValueError(f"Unable to render type {type(e)} {e}")
 
-    def render_cte(self, cte: CTE, auto_sort: bool = True):
+    def render_cte(self, cte: CTE, auto_sort: bool = True) -> CompiledCTE:
         if self.UNNEST_MODE in (
             UnnestMode.CROSS_APPLY,
             UnnestMode.CROSS_JOIN,

{pytrilogy-0.0.2.36 → pytrilogy-0.0.2.38}/trilogy/executor.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Any, Generator
+from typing import List, Optional, Any, Generator, Protocol
 from functools import singledispatchmethod
 from sqlalchemy import text
 from sqlalchemy.engine import Engine, CursorResult
@@ -22,16 +22,29 @@ from trilogy.core.models import (
     CopyStatement,
     ImportStatement,
     MergeStatementV2,
+    Function,
+    FunctionType,
+    MapWrapper,
+    ListWrapper,
 )
 from trilogy.dialect.base import BaseDialect
 from trilogy.dialect.enums import Dialects
-from trilogy.core.enums import IOType
+from trilogy.core.enums import IOType, Granularity
 from trilogy.parser import parse_text
 from trilogy.hooks.base_hook import BaseHook
 from pathlib import Path
 from dataclasses import dataclass
 
 
+class ResultProtocol(Protocol):
+    values: List[Any]
+    columns: List[str]
+
+    def fetchall(self) -> List[Any]: ...
+
+    def keys(self) -> List[str]: ...
+
+
 @dataclass
 class MockResult:
     values: list[Any]
@@ -185,7 +198,6 @@ class Executor(object):
 
     @execute_query.register
     def _(self, query: ImportStatement) -> CursorResult:
-        self.environment.add_file_import(query.path, query.alias)
         return MockResult(
             [
                 {
@@ -198,15 +210,16 @@ class Executor(object):
 
     @execute_query.register
     def _(self, query: MergeStatementV2) -> CursorResult:
+        for concept in query.sources:
+            self.environment.merge_concept(
+                concept, query.targets[concept.address], modifiers=query.modifiers
+            )
 
-        self.environment.merge_concept(
-            query.source, query.target, modifiers=query.modifiers
-        )
         return MockResult(
             [
                 {
-                    "source": query.source.address,
-                    "target": query.target.address,
+                    "sources": ",".join([x.address for x in query.sources]),
+                    "targets": ",".join([x.address for _, x in query.targets.items()]),
                 }
             ],
             ["source", "target"],
@@ -219,8 +232,7 @@ class Executor(object):
     @execute_query.register
     def _(self, query: ProcessedQuery) -> CursorResult:
         sql = self.generator.compile_statement(query)
-        # connection = self.engine.connect()
-        output = self.connection.execute(text(sql))
+        output = self.execute_raw_sql(sql)
         return output
 
     @execute_query.register
@@ -228,14 +240,14 @@ class Executor(object):
 
         sql = self.generator.compile_statement(query)
 
-        output = self.connection.execute(text(sql))
+        output = self.execute_raw_sql(sql)
         self.environment.add_datasource(query.datasource)
         return output
 
     @execute_query.register
     def _(self, query: ProcessedCopyStatement) -> CursorResult:
         sql = self.generator.compile_statement(query)
-        output: CursorResult = self.connection.execute(text(sql))
+        output: CursorResult = self.execute_raw_sql(sql)
         if query.target_type == IOType.CSV:
             import csv
 
@@ -244,7 +256,7 @@ class Executor(object):
                 outcsv.writerow(output.keys())
                 outcsv.writerows(output)
         else:
-            raise NotImplementedError(f"Unsupported IOType {query.target_type}")
+            raise NotImplementedError(f"Unsupported IO Type {query.target_type}")
         # now return the query we ran through IO
         # TODO: instead return how many rows were written?
         return generate_result_set(
@@ -307,7 +319,20 @@ class Executor(object):
             output.append(compiled_sql)
         return output
 
-    def parse_file(self, file: str | Path, persist: bool = False) -> Generator[
+    def parse_file(
+        self, file: str | Path, persist: bool = False
+    ) -> list[
+        ProcessedQuery
+        | ProcessedQueryPersist
+        | ProcessedShowStatement
+        | ProcessedRawSQLStatement
+        | ProcessedCopyStatement,
+    ]:
+        return list(self.parse_file_generator(file, persist=persist))
+
+    def parse_file_generator(
+        self, file: str | Path, persist: bool = False
+    ) -> Generator[
         ProcessedQuery
         | ProcessedQueryPersist
         | ProcessedShowStatement
@@ -370,13 +395,50 @@ class Executor(object):
             if persist and isinstance(x, ProcessedQueryPersist):
                 self.environment.add_datasource(x.datasource)
 
+    def _hydrate_param(self, param: str) -> Any:
+        matched = [
+            v
+            for v in self.environment.concepts.values()
+            if v.safe_address == param or v.address == param
+        ]
+        if not matched:
+            raise SyntaxError(f"No concept found for parameter {param}")
+
+        concept: Concept = matched.pop()
+        if not concept.granularity == Granularity.SINGLE_ROW:
+            raise SyntaxError(f"Cannot bind non-singleton concept {concept.address}")
+        if (
+            isinstance(concept.lineage, Function)
+            and concept.lineage.operator == FunctionType.CONSTANT
+        ):
+            rval = concept.lineage.arguments[0]
+            if isinstance(rval, ListWrapper):
+                return [x for x in rval]
+            if isinstance(rval, MapWrapper):
+                return {k: v for k, v in rval.items()}
+            return rval
+        else:
+            results = self.execute_query(f"select {concept.name} limit 1;").fetchone()
+        if not results:
+            return None
+        return results[0]
+
     def execute_raw_sql(
         self, command: str, variables: dict | None = None
     ) -> CursorResult:
         """Run a command against the raw underlying
         execution engine"""
+        final_params = None
+        q = text(command)
         if variables:
-            return self.connection.execute(text(command), variables)
+            final_params = variables
+        else:
+            params = q.compile().params
+            if params:
+                final_params = {x: self._hydrate_param(x) for x in params}
+
+        if final_params:
+            return self.connection.execute(text(command), final_params)
         return self.connection.execute(
             text(command),
         )

{pytrilogy-0.0.2.36 → pytrilogy-0.0.2.38}/trilogy/parsing/parse_engine.py

@@ -302,6 +302,9 @@ class ParseToObjects(Transformer):
     def IDENTIFIER(self, args) -> str:
         return args.value
 
+    def WILDCARD_IDENTIFIER(self, args) -> str:
+        return args.value
+
     def QUOTED_IDENTIFIER(self, args) -> str:
         return args.value[1:-1]
 
@@ -786,25 +789,54 @@ class ParseToObjects(Transformer):
         return [x for x in args]
 
     @v_args(meta=True)
-    def merge_statement_v2(self, meta: Meta, args) -> MergeStatementV2:
+    def merge_statement(self, meta: Meta, args) -> MergeStatementV2:
         modifiers = []
-        concepts = []
+        cargs: list[str] = []
+        source_wildcard = None
+        target_wildcard = None
         for arg in args:
             if isinstance(arg, Modifier):
                 modifiers.append(arg)
             else:
-                concepts.append(self.environment.concepts[arg])
+                cargs.append(arg)
+        source, target = cargs
+        if source.endswith(".*"):
+            if not target.endswith(".*"):
+                raise ValueError("Invalid merge, source is wildcard, target is not")
+            source_wildcard = source[:-2]
+            target_wildcard = target[:-2]
+            sources = [
+                v
+                for k, v in self.environment.concepts.items()
+                if v.namespace == source_wildcard
+            ]
+            targets = {}
+            for x in sources:
+                target = target_wildcard + "." + x.name
+                if target in self.environment.concepts:
+                    targets[x.address] = self.environment.concepts[target]
+            sources = [x for x in sources if x.address in targets]
+        else:
+            sources = [self.environment.concepts[source]]
+            targets = {sources[0].address: self.environment.concepts[target]}
         new = MergeStatementV2(
-            source=concepts[0], target=concepts[1], modifiers=modifiers
+            sources=sources,
+            targets=targets,
+            modifiers=modifiers,
+            source_wildcard=source_wildcard,
+            target_wildcard=target_wildcard,
         )
-
-        self.environment.merge_concept(new.source, new.target, modifiers)
+        for source_c in new.sources:
+            self.environment.merge_concept(
+                source_c, targets[source_c.address], modifiers
+            )
 
         return new
 
     @v_args(meta=True)
     def rawsql_statement(self, meta: Meta, args) -> RawSQLStatement:
-        return RawSQLStatement(meta=Metadata(line_number=meta.line), text=args[0])
+        statement = RawSQLStatement(meta=Metadata(line_number=meta.line), text=args[0])
+        return statement
 
     def COPY_TYPE(self, args) -> IOType:
         return IOType(args.value)
@@ -1744,7 +1776,7 @@ class ParseToObjects(Transformer):
             arguments=args,
             output_datatype=output_datatype,
             output_purpose=function_args_to_output_purpose(args),
-            # valid_inputs={DataType.DATE, DataType.TIMESTAMP, DataType.DATETIME},
+            valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
             arg_count=2,
         )
 
@@ -1757,7 +1789,7 @@ class ParseToObjects(Transformer):
             arguments=args,
             output_datatype=output_datatype,
             output_purpose=function_args_to_output_purpose(args),
-            # valid_inputs={DataType.DATE, DataType.TIMESTAMP, DataType.DATETIME},
+            valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
             arg_count=2,
         )
 
@@ -1770,20 +1802,21 @@ class ParseToObjects(Transformer):
             arguments=args,
             output_datatype=output_datatype,
             output_purpose=function_args_to_output_purpose(args),
-            # valid_inputs={DataType.DATE, DataType.TIMESTAMP, DataType.DATETIME},
+            valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
             arg_count=2,
         )
 
     @v_args(meta=True)
     def fdiv(self, meta: Meta, args):
         args = process_function_args(args, meta=meta, environment=self.environment)
-        output_datatype = merge_datatypes([arg_to_datatype(x) for x in args])
+        # 2024-11-18 - this is a bit of a hack, but division always returns a float
+        # output_datatype = merge_datatypes([arg_to_datatype(x) for x in args])
         return Function(
             operator=FunctionType.DIVIDE,
             arguments=args,
-            output_datatype=output_datatype,
+            output_datatype=DataType.FLOAT,  # division always returns a float
             output_purpose=function_args_to_output_purpose(args),
-            # valid_inputs={DataType.DATE, DataType.TIMESTAMP, DataType.DATETIME},
+            valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
             arg_count=2,
         )
 

{pytrilogy-0.0.2.36 → pytrilogy-0.0.2.38}/trilogy/parsing/render.py

@@ -432,7 +432,9 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: MergeStatementV2):
-        return f"MERGE {self.to_string(arg.source)} into {''.join([self.to_string(modifier) for modifier in arg.modifiers])}{self.to_string(arg.target)};"
+        if len(arg.sources) == 1:
+            return f"MERGE {self.to_string(arg.sources[0])} into {''.join([self.to_string(modifier) for modifier in arg.modifiers])}{self.to_string(arg.targets[arg.sources[0].address])};"
+        return f"MERGE {arg.source_wildcard}.* into {''.join([self.to_string(modifier) for modifier in arg.modifiers])}{arg.target_wildcard}.*;"
 
     @to_string.register
     def _(self, arg: Modifier):

{pytrilogy-0.0.2.36 → pytrilogy-0.0.2.38}/trilogy/parsing/trilogy.lark

@@ -9,7 +9,7 @@
     | rowset_derivation_statement
     | import_statement
     | copy_statement
-    | merge_statement_v2
+    | merge_statement
     | rawsql_statement
     
     _TERMINATOR:  ";"i /\s*/
@@ -74,8 +74,7 @@
 
     align_clause: align_item ("AND"i align_item)*  "AND"i?
 
-    // merge_v2
-    merge_statement_v2: "merge"i  IDENTIFIER "into"i SHORTHAND_MODIFIER? IDENTIFIER 
+    merge_statement: "merge"i  WILDCARD_IDENTIFIER "into"i SHORTHAND_MODIFIER? WILDCARD_IDENTIFIER
 
     // raw sql statement
     rawsql_statement: "raw_sql"i "(" MULTILINE_STRING ")"
@@ -292,6 +291,7 @@
     // base language constructs
     concept_lit: IDENTIFIER
     IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\-\.]*/
+    WILDCARD_IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\-\.\*]*/
     QUOTED_IDENTIFIER:  /`[a-zA-Z\_][a-zA-Z0-9\_\.\-\*\:\s]*`/
     QUOTED_ADDRESS:  /`[a-zA-Z\_][a-zA-Z0-9\_\.\-\*\:]*`/
     ADDRESS: IDENTIFIER
@@ -302,7 +302,7 @@
     SINGLE_STRING_CHARS: /(?:(?!\${)([^'\\]|\\.))+/+ // any character except '
     _single_quote: "'" ( SINGLE_STRING_CHARS )* "'" 
     _double_quote: "\"" ( DOUBLE_STRING_CHARS )* "\"" 
-    string_lit: _single_quote | _double_quote
+    string_lit: _single_quote | _double_quote | MULTILINE_STRING
 
     MINUS: "-"