pytrilogy 0.0.3.19__tar.gz → 0.0.3.21__tar.gz

{pytrilogy-0.0.3.19/pytrilogy.egg-info → pytrilogy-0.0.3.21}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.19
+Version: 0.0.3.21
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -31,6 +31,7 @@ Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
+Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: summary
@@ -44,7 +45,7 @@ pytrilogy is an experimental implementation of the Trilogy language, a higher-le
 Trilogy looks like SQL, but simpler. It's a modern SQL refresh targeted at SQL lovers who want more reusability and composability without losing the expressiveness and iterative value of SQL. It compiles to SQL - making it easy to debug or integrate into existing workflows - and can be run against any supported SQL backend.  
 
 > [!TIP]
-> To get an overview of the language and run interactive examples, head to the [documentation](https://trilogydata.dev/).
+> Try it online in a hosted [open-source studio](https://trilogydata.dev/trilogy-studio-core/). To get an overview of the language and run interactive examples, head to the [documentation](https://trilogydata.dev/).
 
 Installation: `pip install pytrilogy`
 

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/README.md

@@ -7,7 +7,7 @@ pytrilogy is an experimental implementation of the Trilogy language, a higher-le
 Trilogy looks like SQL, but simpler. It's a modern SQL refresh targeted at SQL lovers who want more reusability and composability without losing the expressiveness and iterative value of SQL. It compiles to SQL - making it easy to debug or integrate into existing workflows - and can be run against any supported SQL backend.  
 
 > [!TIP]
-> To get an overview of the language and run interactive examples, head to the [documentation](https://trilogydata.dev/).
+> Try it online in a hosted [open-source studio](https://trilogydata.dev/trilogy-studio-core/). To get an overview of the language and run interactive examples, head to the [documentation](https://trilogydata.dev/).
 
 Installation: `pip install pytrilogy`
 
@@ -353,4 +353,4 @@ Return generated SQL without executing.
 
 ```sql
 show <select>;
-```
+```

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.19
+Version: 0.0.3.21
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -31,6 +31,7 @@ Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
+Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: summary
@@ -44,7 +45,7 @@ pytrilogy is an experimental implementation of the Trilogy language, a higher-le
 Trilogy looks like SQL, but simpler. It's a modern SQL refresh targeted at SQL lovers who want more reusability and composability without losing the expressiveness and iterative value of SQL. It compiles to SQL - making it easy to debug or integrate into existing workflows - and can be run against any supported SQL backend.  
 
 > [!TIP]
-> To get an overview of the language and run interactive examples, head to the [documentation](https://trilogydata.dev/).
+> Try it online in a hosted [open-source studio](https://trilogydata.dev/trilogy-studio-core/). To get an overview of the language and run interactive examples, head to the [documentation](https://trilogydata.dev/).
 
 Installation: `pip install pytrilogy`
 

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/tests/test_parsing.py

@@ -1,5 +1,7 @@
+from pytest import raises
+
 from trilogy import Dialects
-from trilogy.constants import MagicConstants
+from trilogy.constants import MagicConstants, Parsing
 from trilogy.core.enums import BooleanOperator, ComparisonOperator, Purpose
 from trilogy.core.functions import argument_to_purpose, function_args_to_output_purpose
 from trilogy.core.models.author import Comparison
@@ -17,6 +19,7 @@ from trilogy.core.statements.author import SelectStatement, ShowStatement
 from trilogy.core.statements.execute import ProcessedQuery
 from trilogy.dialect.base import BaseDialect
 from trilogy.parsing.parse_engine import (
+    ParseError,
     arg_to_datatype,
     parse_text,
 )
@@ -661,3 +664,73 @@ select x % 10 -> x_mod_10;
             
 """
     )
+
+
+def test_import_shows_source():
+
+    env = Environment(
+        config=EnvironmentOptions(
+            import_resolver=DictImportResolver(
+                content={
+                    "test": """
+import test_dep as test_dep;
+key x int;
+datasource test (
+x: x)
+grain(x)
+query '''
+select 1 as x
+union all
+select 11 as x
+''' TYPO
+""",
+                    "test_dep": """
+key x int;
+""",
+                }
+            )
+        )
+    )
+    assert isinstance(env.config.import_resolver, DictImportResolver)
+
+    with raises(Exception, match="Unable to import 'test', parsing error") as e:
+        env.parse(
+            """
+        import test;
+                
+    select x % 10 -> x_mod_10;
+                
+                
+    """
+        )
+        assert "TYPO" in str(e.value)
+        assert 1 == 0
+
+
+def test_concept_shadow_warning():
+    x = """
+key scalar int;    
+property scalar.int_array list<int>;
+
+key split <- unnest(int_array);
+
+datasource avalues (
+    int_array: int_array,
+	scalar: scalar
+    ) 
+grain (scalar) 
+query '''(
+select [1,2,3,4] as int_array, 2 as scalar
+union all
+select [5,6,7,8] as int_array, 4 as scalar
+)''';
+
+SELECT
+    int_array,
+    sum(scalar)->scalar
+;
+"""
+    with raises(ParseError):
+        env, parsed = parse_text(
+            x, parse_config=Parsing(strict_name_shadow_enforcement=True)
+        )

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.19"
+__version__ = "0.0.3.21"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/constants.py

@@ -47,6 +47,14 @@ class Rendering:
     concise: bool = False
 
 
+@dataclass
+class Parsing:
+    """Control Parsing"""
+
+    strict_name_shadow_enforcement: bool = False
+    select_as_definition: bool = True
+
+
 # TODO: support loading from environments
 @dataclass
 class Config:
@@ -57,7 +65,7 @@ class Config:
     comments: Comments = field(default_factory=Comments)
     optimizations: Optimizations = field(default_factory=Optimizations)
     rendering: Rendering = field(default_factory=Rendering)
-    select_as_definition: bool = True
+    parsing: Parsing = field(default_factory=Parsing)
 
     @property
     def show_comments(self) -> bool:

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/core/enums.py

@@ -8,6 +8,7 @@ class UnnestMode(Enum):
     CROSS_APPLY = "cross_apply"
     CROSS_JOIN = "cross_join"
     CROSS_JOIN_ALIAS = "cross_join_alias"
+    SNOWFLAKE = "snowflake"
 
 
 class ConceptSource(Enum):

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/core/models/execute.py

@@ -59,6 +59,14 @@ class CTE(BaseModel):
     base_name_override: Optional[str] = None
     base_alias_override: Optional[str] = None
 
+    @field_validator("join_derived_concepts")
+    def validate_join_derived_concepts(cls, v):
+        if len(v) > 1:
+            raise NotImplementedError(
+                "Multiple join derived concepts not yet supported."
+            )
+        return unique(v, "address")
+
     @property
     def identifier(self):
         return self.name

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/core/statements/author.py

@@ -142,7 +142,7 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
                 if isinstance(x.content.output, UndefinedConcept):
                     continue
                 if (
-                    CONFIG.select_as_definition
+                    CONFIG.parsing.select_as_definition
                     and not environment.frozen
                     and x.concept.address not in environment.concepts
                 ):
@@ -156,7 +156,7 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
             elif isinstance(x.content, ConceptRef):
                 output.local_concepts[x.content.address] = environment.concepts[
                     x.content.address
-                ]  # .set_select_grain(output.grain, environment)
+                ]
         output.validate_syntax(environment)
         return output
 

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/dialect/base.py

@@ -3,7 +3,7 @@ from typing import Any, Callable, Dict, List, Optional, Sequence, Union
 
 from jinja2 import Template
 
-from trilogy.constants import CONFIG, MagicConstants, logger
+from trilogy.constants import CONFIG, MagicConstants, Rendering, logger
 from trilogy.core.enums import (
     DatePart,
     FunctionType,
@@ -283,6 +283,9 @@ class BaseDialect:
     DATATYPE_MAP = DATATYPE_MAP
     UNNEST_MODE = UnnestMode.CROSS_APPLY
 
+    def __init__(self, rendering: Rendering | None = None):
+        self.rendering = rendering or CONFIG.rendering
+
     def render_order_item(
         self,
         order_item: BuildOrderItem,
@@ -416,7 +419,7 @@ class BaseDialect:
             elif (
                 isinstance(c.lineage, FUNCTION_ITEMS)
                 and c.lineage.operator == FunctionType.CONSTANT
-                and CONFIG.rendering.parameters is True
+                and self.rendering.parameters is True
                 and c.datatype.data_type != DataType.MAP
             ):
                 rval = f":{c.safe_address}"
@@ -633,7 +636,7 @@ class BaseDialect:
             if (
                 isinstance(e.lineage, FUNCTION_ITEMS)
                 and e.lineage.operator == FunctionType.CONSTANT
-                and CONFIG.rendering.parameters is True
+                and self.rendering.parameters is True
                 and e.datatype.data_type != DataType.MAP
             ):
                 return f":{e.safe_address}"
@@ -694,6 +697,7 @@ class BaseDialect:
             UnnestMode.CROSS_APPLY,
             UnnestMode.CROSS_JOIN,
             UnnestMode.CROSS_JOIN_ALIAS,
+            UnnestMode.SNOWFLAKE,
         ):
             # for a cross apply, derivation happens in the join
             # so we only use the alias to select
@@ -723,6 +727,12 @@ class BaseDialect:
                     UnnestMode.CROSS_JOIN,
                     UnnestMode.CROSS_APPLY,
                 ):
+
+                    source = f"{render_unnest(self.UNNEST_MODE, self.QUOTE_CHARACTER, cte.join_derived_concepts[0], self.render_concept_sql, cte)}"
+                elif (
+                    cte.join_derived_concepts
+                    and self.UNNEST_MODE == UnnestMode.SNOWFLAKE
+                ):
                     source = f"{render_unnest(self.UNNEST_MODE, self.QUOTE_CHARACTER, cte.join_derived_concepts[0], self.render_concept_sql, cte)}"
                 # direct - eg DUCK DB - can be directly selected inline
                 elif (

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/dialect/common.py

@@ -25,7 +25,15 @@ def render_unnest(
 ):
     if unnest_mode == UnnestMode.CROSS_JOIN:
         return f"{render_func(concept, cte, False)} as {quote_character}{concept.safe_address}{quote_character}"
-    return f"{render_func(concept, cte, False)} as unnest_wrapper ({quote_character}{concept.safe_address}{quote_character})"
+    elif unnest_mode == UnnestMode.CROSS_JOIN_ALIAS:
+        return f"{render_func(concept, cte, False)} as unnest_wrapper ({quote_character}{concept.safe_address}{quote_character})"
+    elif unnest_mode == UnnestMode.SNOWFLAKE:
+        # if we don't actually have a join, we're directly unnesting a concept, and we can skip the flatten
+        if not cte.render_from_clause:
+            return f"{render_func(concept, cte, False)} as unnest_wrapper ( unnest1, unnest2, unnest3, unnest4, {quote_character}{cte.join_derived_concepts[0].safe_address}{quote_character})"
+        # otherwise, flatten the concept for the join
+        return f"flatten({render_func(concept, cte, False)}) as unnest_wrapper ( unnest1, unnest2, unnest3, unnest4, {quote_character}{cte.join_derived_concepts[0].safe_address}{quote_character})"
+    return f"{render_func(concept, cte, False)} as {quote_character}{concept.safe_address}{quote_character}"
 
 
 def render_join_concept(
@@ -67,6 +75,8 @@ def render_join(
             return f"CROSS JOIN {render_unnest(unnest_mode, quote_character, join.concept_to_unnest, render_func, cte)}"
         if unnest_mode == UnnestMode.CROSS_JOIN_ALIAS:
             return f"CROSS JOIN {render_unnest(unnest_mode, quote_character, join.concept_to_unnest, render_func, cte)}"
+        if unnest_mode == UnnestMode.SNOWFLAKE:
+            return f"LEFT JOIN LATERAL {render_unnest(unnest_mode, quote_character, join.concept_to_unnest, render_func, cte)}"
         return f"FULL JOIN {render_unnest(unnest_mode, quote_character, join.concept_to_unnest, render_func, cte)}"
     # left_name = join.left_name
     right_name = join.right_name

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/dialect/config.py

@@ -76,12 +76,22 @@ class SnowflakeConfig(DialectConfig):
         account: str,
         username: str,
         password: str,
+        database: str | None = None,
+        schema: str | None = None,
     ):
         self.account = account
         self.username = username
         self.password = password
+        self.database = database
+        self.schema = schema
+        if self.schema and not self.database:
+            raise ValueError("Setting snowflake schema also requires setting database")
 
     def connection_string(self) -> str:
+        if self.schema:
+            return f"snowflake://{self.username}:{self.password}@{self.account}/{self.database}/{self.schema}"
+        if self.database:
+            return f"snowflake://{self.username}:{self.password}@{self.account}/{self.database}"
         return f"snowflake://{self.username}:{self.password}@{self.account}"
 
 

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/dialect/enums.py

@@ -7,7 +7,7 @@ if TYPE_CHECKING:
     from trilogy import Executor
     from trilogy.hooks.base_hook import BaseHook
 
-from trilogy.constants import logger
+from trilogy.constants import Rendering, logger
 from trilogy.dialect.config import DialectConfig
 
 
@@ -114,6 +114,7 @@ class Dialects(Enum):
         environment: Optional["Environment"] = None,
         hooks: List["BaseHook"] | None = None,
         conf: DialectConfig | None = None,
+        rendering: Rendering | None = None,
         _engine_factory: Callable | None = None,
     ) -> "Executor":
         from trilogy import Executor
@@ -123,6 +124,7 @@ class Dialects(Enum):
                 engine=self.default_engine(conf=conf, _engine_factory=_engine_factory),
                 environment=environment or Environment(),
                 dialect=self,
+                rendering=rendering,
                 hooks=hooks,
             )
 
@@ -130,5 +132,6 @@ class Dialects(Enum):
             engine=self.default_engine(conf=conf),
             environment=environment or Environment(),
             dialect=self,
+            rendering=rendering,
             hooks=hooks,
         )

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/dialect/snowflake.py

@@ -30,6 +30,8 @@ FUNCTION_MAP = {
     FunctionType.QUARTER: lambda x: f"EXTRACT(QUARTER from {x[0]})",
     # math
     FunctionType.DIVIDE: lambda x: f"DIV0({x[0]},{x[1]})",
+    FunctionType.UNNEST: lambda x: f"table(flatten({x[0]}))",
+    FunctionType.ARRAY: lambda x: f"ARRAY_CONSTRUCT({', '.join(x)})",
 }
 
 FUNCTION_GRAIN_MATCH_MAP = {
@@ -83,4 +85,4 @@ class SnowflakeDialect(BaseDialect):
     }
     QUOTE_CHARACTER = '"'
     SQL_TEMPLATE = BQ_SQL_TEMPLATE
-    UNNEST_MODE = UnnestMode.CROSS_JOIN
+    UNNEST_MODE = UnnestMode.SNOWFLAKE

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/executor.py

@@ -6,7 +6,7 @@ from typing import Any, Generator, List, Optional, Protocol
 from sqlalchemy import text
 from sqlalchemy.engine import CursorResult
 
-from trilogy.constants import logger
+from trilogy.constants import Rendering, logger
 from trilogy.core.enums import FunctionType, Granularity, IOType
 from trilogy.core.models.author import Concept, Function
 from trilogy.core.models.build import BuildConcept, BuildFunction
@@ -75,6 +75,7 @@ class Executor(object):
         dialect: Dialects,
         engine: ExecutionEngine,
         environment: Optional[Environment] = None,
+        rendering: Rendering | None = None,
         hooks: List[BaseHook] | None = None,
     ):
         self.dialect: Dialects = dialect
@@ -83,7 +84,7 @@ class Executor(object):
         self.generator: BaseDialect
         self.logger = logger
         self.hooks = hooks
-        self.generator = get_dialect_generator(self.dialect)
+        self.generator = get_dialect_generator(self.dialect, rendering)
         self.connection = self.engine.connect()
         # TODO: make generic
         if self.dialect == Dialects.DATAFRAME:

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/parsing/parse_engine.py

@@ -18,9 +18,11 @@ from lark.tree import Meta
 from pydantic import ValidationError
 
 from trilogy.constants import (
+    CONFIG,
     DEFAULT_NAMESPACE,
     NULL_VALUE,
     MagicConstants,
+    Parsing,
 )
 from trilogy.core.enums import (
     BooleanOperator,
@@ -282,6 +284,7 @@ class ParseToObjects(Transformer):
         text_lookup: dict[Path | str, str] | None = None,
         environment_lookup: dict[str, Environment] | None = None,
         import_keys: list[str] | None = None,
+        parse_config: Parsing | None = None,
     ):
         Transformer.__init__(self, True)
         self.environment: Environment = environment
@@ -298,6 +301,7 @@ class ParseToObjects(Transformer):
         self.parse_pass = ParsePass.INITIAL
         self.function_factory = FunctionFactory(self.environment)
         self.import_keys: list[str] = import_keys or ["root"]
+        self.parse_config: Parsing = parse_config or CONFIG.parsing
 
     def set_text(self, text: str):
         self.text_lookup[self.token_address] = text
@@ -983,7 +987,12 @@ class ParseToObjects(Transformer):
             text = self.resolve_import_address(target)
             self.text_lookup[token_lookup] = text
 
-            raw_tokens = PARSER.parse(text)
+            try:
+                raw_tokens = PARSER.parse(text)
+            except Exception as e:
+                raise ImportError(
+                    f"Unable to import '{target}', parsing error: {e}"
+                ) from e
             self.tokens[token_lookup] = raw_tokens
 
         if cache_lookup in self.parsed:
@@ -1009,6 +1018,7 @@ class ParseToObjects(Transformer):
                     tokens=self.tokens,
                     text_lookup=self.text_lookup,
                     import_keys=self.import_keys + [cache_key],
+                    parse_config=self.parse_config,
                 )
                 nparser.transform(raw_tokens)
                 self.parsed[cache_lookup] = nparser
@@ -1143,8 +1153,8 @@ class ParseToObjects(Transformer):
             elif isinstance(arg, HavingClause):
                 having = arg
         if not select_items:
-            raise ValueError("Malformed select, missing select items")
-
+            raise ParseError("Malformed select, missing select items")
+        pre_keys = set(self.environment.concepts.keys())
         base = SelectStatement.from_inputs(
             environment=self.environment,
             selection=select_items,
@@ -1154,6 +1164,15 @@ class ParseToObjects(Transformer):
             limit=limit,
             meta=Metadata(line_number=meta.line),
         )
+        if (
+            self.parse_pass == ParsePass.INITIAL
+            and self.parse_config.strict_name_shadow_enforcement
+        ):
+            intersection = base.locally_derived.intersection(pre_keys)
+            if intersection:
+                raise ParseError(
+                    f"Select statement {base} has derived concepts {list(intersection)} that shadow existing environment concepts, which may cause unexpected behavior. Rename these."
+                )
         return base
 
     @v_args(meta=True)
@@ -1735,7 +1754,10 @@ def parse_text_raw(text: str, environment: Optional[Environment] = None):
 
 
 def parse_text(
-    text: str, environment: Optional[Environment] = None, root: Path | None = None
+    text: str,
+    environment: Optional[Environment] = None,
+    root: Path | None = None,
+    parse_config: Parsing | None = None,
 ) -> Tuple[
     Environment,
     List[
@@ -1751,7 +1773,9 @@ def parse_text(
     environment = environment or (
         Environment(working_path=root) if root else Environment()
     )
-    parser = ParseToObjects(environment=environment, import_keys=["root"])
+    parser = ParseToObjects(
+        environment=environment, import_keys=["root"], parse_config=parse_config
+    )
 
     try:
         parser.set_text(text)

{pytrilogy-0.0.3.19 → pytrilogy-0.0.3.21}/trilogy/render.py

@@ -1,38 +1,42 @@
+from trilogy.constants import Rendering
+from trilogy.dialect.base import BaseDialect
 from trilogy.dialect.enums import Dialects
 
 
-def get_dialect_generator(dialect: Dialects):
+def get_dialect_generator(
+    dialect: Dialects, rendering: Rendering | None = None
+) -> BaseDialect:
     if dialect == Dialects.BIGQUERY:
         from trilogy.dialect.bigquery import BigqueryDialect
 
-        return BigqueryDialect()
+        return BigqueryDialect(rendering=rendering)
     elif dialect == Dialects.SQL_SERVER:
         from trilogy.dialect.sql_server import SqlServerDialect
 
-        return SqlServerDialect()
+        return SqlServerDialect(rendering=rendering)
     elif dialect == Dialects.DUCK_DB:
         from trilogy.dialect.duckdb import DuckDBDialect
 
-        return DuckDBDialect()
+        return DuckDBDialect(rendering=rendering)
     elif dialect == Dialects.PRESTO:
         from trilogy.dialect.presto import PrestoDialect
 
-        return PrestoDialect()
+        return PrestoDialect(rendering=rendering)
     elif dialect == Dialects.TRINO:
         from trilogy.dialect.presto import TrinoDialect
 
-        return TrinoDialect()
+        return TrinoDialect(rendering=rendering)
     elif dialect == Dialects.POSTGRES:
         from trilogy.dialect.postgres import PostgresDialect
 
-        return PostgresDialect()
+        return PostgresDialect(rendering=rendering)
     elif dialect == Dialects.SNOWFLAKE:
         from trilogy.dialect.snowflake import SnowflakeDialect
 
-        return SnowflakeDialect()
+        return SnowflakeDialect(rendering=rendering)
     elif dialect == Dialects.DATAFRAME:
         from trilogy.dialect.dataframe import DataframeDialect
 
-        return DataframeDialect()
+        return DataframeDialect(rendering=rendering)
     else:
         raise ValueError(f"Unsupported dialect {dialect}")