pytrilogy 0.0.2.27__tar.gz → 0.0.2.29__tar.gz

{pytrilogy-0.0.2.27/pytrilogy.egg-info → pytrilogy-0.0.2.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.27
+Version: 0.0.2.29
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.27
+Version: 0.0.2.29
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/pytrilogy.egg-info/SOURCES.txt

@@ -13,6 +13,7 @@ tests/test_declarations.py
 tests/test_derived_concepts.py
 tests/test_discovery_nodes.py
 tests/test_environment.py
+tests/test_executor.py
 tests/test_functions.py
 tests/test_imports.py
 tests/test_metadata.py

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/tests/test_environment.py

@@ -20,7 +20,7 @@ def test_environment_from_path():
 
     env = Environment.from_file(Path(__file__).parent / "test_env.preql")
 
-    assert "id" in env.concepts
+    assert "local.id" in env.concepts
 
 
 def test_environment_merge():

pytrilogy-0.0.2.29/tests/test_executor.py

@@ -0,0 +1,8 @@
+from trilogy import Dialects
+from pathlib import Path
+
+
+def test_file_parsing():
+    target = Path(__file__).parent / "test_env.preql"
+    parsed = Dialects.DUCK_DB.default_executor().parse_file(target)
+    assert len(list(parsed)) == 1

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/tests/test_parsing.py

@@ -7,6 +7,7 @@ from trilogy.core.models import (
     Environment,
     Comparison,
     TupleWrapper,
+    Datasource,
 )
 from trilogy.core.functions import argument_to_purpose, function_args_to_output_purpose
 from trilogy.parsing.parse_engine import (
@@ -178,7 +179,7 @@ select
     )
 
     for name in ["name_alphabetical", "name_alphabetical_2"]:
-        assert name in env.concepts
+        assert f"local.{name}" in env.concepts
         assert env.concepts[name].purpose == Purpose.PROPERTY
         assert env.concepts[name].keys == (env.concepts["id"],)
 
@@ -197,7 +198,6 @@ select
     )
 
     for name in ["join_id"]:
-        assert name in env.concepts
         assert env.concepts[name].purpose == Purpose.PROPERTY
         assert env.concepts[name].keys == (
             env.concepts["id"],
@@ -225,7 +225,6 @@ select
     )
     # assert output_purpose == Purpose.METRIC
     for name in ["test_name_count"]:
-        assert name in env.concepts
         assert env.concepts[name].purpose == Purpose.METRIC
 
 
@@ -513,6 +512,35 @@ address `abc:def`
     assert ds.non_partial_for.conditional.right == 10
 
 
+def test_datasource_from_persist():
+
+    text = """
+key x int;
+key y int;
+
+datasource test (
+x:x,
+y:y)
+grain(x)
+address `abc:def`
+;
+
+persist alias into tbl_alias from
+select
+x,
+y
+where y>10;
+
+
+
+"""
+    env, parsed = parse_text(text)
+
+    ds: Datasource = parsed[-1].datasource
+    assert ds.non_partial_for.conditional.right == 10
+    assert not ds.where
+
+
 def test_filter_concise():
 
     text = """

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/tests/test_undefined_concept.py

@@ -32,6 +32,6 @@ def test_undefined_concept_dict():
     try:
         env["orid"]
     except UndefinedConceptException as e:
-        assert e.suggestions == ["order_id"]
+        assert e.suggestions == ["local.order_id"]
         assert "suggestions" in e.message.lower()
         assert "order_id" in e.message.lower()

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.27"
+__version__ = "0.0.2.29"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/trilogy/core/models.py

@@ -606,6 +606,8 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
         return self.grain.components_copy if self.grain else []
 
     def with_namespace(self, namespace: str) -> "Concept":
+        if namespace == self.namespace:
+            return self
         return self.__class__(
             name=self.name,
             datatype=self.datatype,
@@ -1758,7 +1760,6 @@ class SelectStatement(HasUUID, Mergeable, Namespaced, SelectTypeMixin, BaseModel
             grain=grain or self.grain,
             columns=columns,
             namespace=namespace,
-            where=WhereClause(conditional=condition) if condition else None,
             non_partial_for=WhereClause(conditional=condition) if condition else None,
         )
         for column in columns:
@@ -3253,7 +3254,6 @@ class EnvironmentConceptDict(dict):
                 )
                 self.undefined[key] = undefined
                 return undefined
-
             matches = self._find_similar_concepts(key)
             message = f"Undefined concept: {key}."
             if matches:
@@ -3263,8 +3263,15 @@ class EnvironmentConceptDict(dict):
                 raise UndefinedConceptException(f"line: {line_no}: " + message, matches)
             raise UndefinedConceptException(message, matches)
 
-    def _find_similar_concepts(self, concept_name):
-        matches = difflib.get_close_matches(concept_name, self.keys())
+    def _find_similar_concepts(self, concept_name: str):
+        def strip_local(input: str):
+            if input.startswith(f"{DEFAULT_NAMESPACE}."):
+                return input[len(DEFAULT_NAMESPACE) + 1 :]
+            return input
+
+        matches = difflib.get_close_matches(
+            strip_local(concept_name), [strip_local(x) for x in self.keys()]
+        )
         return matches
 
     def items(self) -> ItemsView[str, Concept]:  # type: ignore
@@ -3325,7 +3332,6 @@ class Environment(BaseModel):
 
     materialized_concepts: List[Concept] = Field(default_factory=list)
     alias_origin_lookup: Dict[str, Concept] = Field(default_factory=dict)
-    canonical_map: Dict[str, str] = Field(default_factory=dict)
     _parse_count: int = 0
 
     @classmethod
@@ -3444,14 +3450,38 @@ class Environment(BaseModel):
                 exists = True
             imp_stm = ImportStatement(alias=alias, path=Path(source.working_path))
 
+        same_namespace = alias == self.namespace
+
         if not exists:
             self.imports[alias].append(imp_stm)
 
-        for _, concept in source.concepts.items():
-            self.add_concept(concept.with_namespace(alias), _ignore_cache=True)
+        for k, concept in source.concepts.items():
+            if same_namespace:
+                new = self.add_concept(concept, _ignore_cache=True)
+            else:
+                new = self.add_concept(
+                    concept.with_namespace(alias), _ignore_cache=True
+                )
+
+                k = address_with_namespace(k, alias)
+            # set this explicitly, to handle aliasing
+            self.concepts[k] = new
 
         for _, datasource in source.datasources.items():
-            self.add_datasource(datasource.with_namespace(alias), _ignore_cache=True)
+            if same_namespace:
+                self.add_datasource(datasource, _ignore_cache=True)
+            else:
+                self.add_datasource(
+                    datasource.with_namespace(alias), _ignore_cache=True
+                )
+        for key, val in source.alias_origin_lookup.items():
+            if same_namespace:
+                self.alias_origin_lookup[key] = val
+            else:
+                self.alias_origin_lookup[address_with_namespace(key, alias)] = (
+                    val.with_namespace(alias)
+                )
+
         self.gen_concept_list_caches()
         return self
 
@@ -3542,8 +3572,6 @@ class Environment(BaseModel):
             existing = self.validate_concept(concept, meta=meta)
             if existing:
                 concept = existing
-        if concept.namespace == DEFAULT_NAMESPACE:
-            self.concepts[concept.name] = concept
         self.concepts[concept.address] = concept
         from trilogy.core.environment_helpers import generate_related_concepts
 
@@ -3631,7 +3659,6 @@ class Environment(BaseModel):
                 v.pseudonyms.add(source.address)
             if v.address == source.address:
                 replacements[k] = target
-                self.canonical_map[k] = target.address
                 v.pseudonyms.add(target.address)
             # we need to update keys and grains of all concepts
             else:

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/trilogy/core/optimization.py

@@ -13,7 +13,7 @@ from trilogy.core.optimizations import (
     PredicatePushdownRemove,
     InlineDatasource,
 )
-
+from trilogy.core.processing.utility import sort_select_output
 
 MAX_OPTIMIZATION_LOOPS = 100
 
@@ -154,20 +154,6 @@ def is_direct_return_eligible(cte: CTE) -> CTE | None:
     return direct_parent
 
 
-def sort_select_output(cte: CTE, query: SelectStatement | MultiSelectStatement):
-    hidden_addresses = [c.address for c in query.hidden_components]
-    output_addresses = [
-        c.address for c in query.output_components if c.address not in hidden_addresses
-    ]
-
-    mapping = {x.address: x for x in cte.output_columns}
-
-    new_output = []
-    for x in output_addresses:
-        new_output.append(mapping[x])
-    cte.output_columns = new_output
-
-
 def optimize_ctes(
     input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
 ) -> list[CTE]:

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/trilogy/core/processing/concept_strategies_v3.py

@@ -838,9 +838,8 @@ def _search_concepts(
             f" {accept_partial} (complete: {complete}), have {found} from {[n for n in stack]} (missing {missing} partial {partial} virtual {virtual}), attempted {attempted}, mandatory w/ filter {mandatory_completion}"
         )
         if complete == ValidationResult.INCOMPLETE_CONDITION:
-            raise SyntaxError(
-                {str(node): node.preexisting_conditions for node in stack}
-            )
+            cond_dict = {str(node): node.preexisting_conditions for node in stack}
+            raise SyntaxError(f"Have {cond_dict} and need {str(conditions)}")
         # early exit if we have a complete stack with one node
         # we can only early exit if we have a complete stack
         # and we are not looking for more non-partial sources

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/trilogy/core/processing/node_generators/select_merge_node.py

@@ -233,7 +233,6 @@ def create_select_node(
     ]
     nullable_lcl = LooseConceptList(concepts=nullable_concepts)
     partial_is_full = conditions and (conditions == datasource.non_partial_for)
-
     bcandidate: StrategyNode = SelectNode(
         input_concepts=[c.concept for c in datasource.columns],
         output_concepts=all_concepts,
@@ -249,7 +248,9 @@ def create_select_node(
         datasource=datasource,
         grain=Grain(components=all_concepts),
         conditions=datasource.where.conditional if datasource.where else None,
-        render_condition=not partial_is_full,
+        preexisting_conditions=(
+            conditions.conditional if partial_is_full and conditions else None
+        ),
     )
 
     # we need to nest the group node one further
@@ -263,6 +264,9 @@ def create_select_node(
             depth=depth,
             partial_concepts=bcandidate.partial_concepts,
             nullable_concepts=bcandidate.nullable_concepts,
+            preexisting_conditions=(
+                conditions.conditional if partial_is_full and conditions else None
+            ),
         )
     else:
         candidate = bcandidate
@@ -340,6 +344,15 @@ def gen_select_merge_node(
 
     if len(parents) == 1:
         return parents[0]
+    preexisting_conditions = None
+    if conditions and all(
+        [
+            x.preexisting_conditions
+            and x.preexisting_conditions == conditions.conditional
+            for x in parents
+        ]
+    ):
+        preexisting_conditions = conditions.conditional
     return MergeNode(
         output_concepts=all_concepts,
         input_concepts=non_constant,
@@ -347,4 +360,5 @@ def gen_select_merge_node(
         g=g,
         depth=depth,
         parents=parents,
+        preexisting_conditions=preexisting_conditions,
     )

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/trilogy/core/processing/nodes/base_node.py

@@ -165,7 +165,6 @@ class StrategyNode:
         hidden_concepts: List[Concept] | None = None,
         existence_concepts: List[Concept] | None = None,
         virtual_output_concepts: List[Concept] | None = None,
-        render_condition: bool = True,
     ):
         self.input_concepts: List[Concept] = (
             unique(input_concepts, "address") if input_concepts else []
@@ -209,7 +208,6 @@ class StrategyNode:
             )
         self.validate_parents()
         self.log = True
-        self.render_condition = render_condition
 
     def add_parents(self, parents: list["StrategyNode"]):
         self.parents += parents
@@ -382,7 +380,6 @@ class StrategyNode:
             hidden_concepts=list(self.hidden_concepts),
             existence_concepts=list(self.existence_concepts),
             virtual_output_concepts=list(self.virtual_output_concepts),
-            render_condition=self.render_condition,
         )
 
 

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/trilogy/core/processing/nodes/select_node_v2.py

@@ -49,7 +49,6 @@ class SelectNode(StrategyNode):
         conditions: Conditional | Comparison | Parenthetical | None = None,
         preexisting_conditions: Conditional | Comparison | Parenthetical | None = None,
         hidden_concepts: List[Concept] | None = None,
-        render_condition: bool = True,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -66,7 +65,6 @@ class SelectNode(StrategyNode):
             conditions=conditions,
             preexisting_conditions=preexisting_conditions,
             hidden_concepts=hidden_concepts,
-            render_condition=render_condition,
         )
         self.accept_partial = accept_partial
         self.datasource = datasource
@@ -123,7 +121,7 @@ class SelectNode(StrategyNode):
             nullable_concepts=[c.concept for c in datasource.columns if c.is_nullable],
             source_type=SourceType.DIRECT_SELECT,
             # we can skip rendering conditions
-            condition=self.conditions if self.render_condition else None,
+            condition=self.conditions,
             # select nodes should never group
             force_group=self.force_group,
             hidden_concepts=self.hidden_concepts,
@@ -208,7 +206,6 @@ class SelectNode(StrategyNode):
             conditions=self.conditions,
             preexisting_conditions=self.preexisting_conditions,
             hidden_concepts=self.hidden_concepts,
-            render_condition=self.render_condition,
         )
 
 

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/trilogy/core/processing/utility.py

@@ -29,6 +29,10 @@ from trilogy.core.models import (
     NumericType,
     ListType,
     TupleWrapper,
+    CTE,
+    MultiSelectStatement,
+    SelectStatement,
+    ProcessedQuery,
 )
 
 from trilogy.core.enums import Purpose, Granularity, BooleanOperator
@@ -66,9 +70,6 @@ def resolve_join_order_v2(
 ) -> list[JoinOrderOutput]:
     datasources = [x for x in g.nodes if x.startswith("ds~")]
     concepts = [x for x in g.nodes if x.startswith("c~")]
-    # from trilogy.hooks.graph_hook import GraphHook
-
-    # GraphHook().query_graph_built(g)
 
     output: list[JoinOrderOutput] = []
     pivot_map = {
@@ -78,7 +79,7 @@ def resolve_join_order_v2(
     pivots = list(
         sorted(
             [x for x in pivot_map if len(pivot_map[x]) > 1],
-            key=lambda x: len(pivot_map[x]),
+            key=lambda x: (len(pivot_map[x]), len(x), x),
         )
     )
     solo = [x for x in pivot_map if len(pivot_map[x]) == 1]
@@ -95,7 +96,7 @@ def resolve_join_order_v2(
             root = pivots.pop()
 
         # sort so less partials is last and eligible lefts are
-        def score_key(x: str) -> int:
+        def score_key(x: str) -> tuple[int, int, str]:
             base = 1
             # if it's left, higher weight
             if x in eligible_left:
@@ -103,7 +104,7 @@ def resolve_join_order_v2(
             # if it has the concept as a partial, lower weight
             if root in partials.get(x, []):
                 base -= 1
-            return base
+            return (base, len(x), x)
 
         # get remainig un-joined datasets
         to_join = sorted(
@@ -531,3 +532,37 @@ def find_nullable_concepts(
             if set(v).issubset(all_ds):
                 final_nullable.add(k)
     return list(sorted(final_nullable))
+
+
+def sort_select_output_processed(cte: CTE, query: ProcessedQuery) -> CTE:
+    hidden_addresses = [c.address for c in query.hidden_columns]
+    output_addresses = [
+        c.address for c in query.output_columns if c.address not in hidden_addresses
+    ]
+
+    mapping = {x.address: x for x in cte.output_columns}
+
+    new_output = []
+    for x in output_addresses:
+        new_output.append(mapping[x])
+    cte.output_columns = new_output
+    return cte
+
+
+def sort_select_output(
+    cte: CTE, query: SelectStatement | MultiSelectStatement | ProcessedQuery
+) -> CTE:
+    if isinstance(query, ProcessedQuery):
+        return sort_select_output_processed(cte, query)
+    hidden_addresses = [c.address for c in query.hidden_components]
+    output_addresses = [
+        c.address for c in query.output_components if c.address not in hidden_addresses
+    ]
+
+    mapping = {x.address: x for x in cte.output_columns}
+
+    new_output = []
+    for x in output_addresses:
+        new_output.append(mapping[x])
+    cte.output_columns = new_output
+    return cte

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/trilogy/dialect/base.py

@@ -2,7 +2,11 @@ from typing import List, Union, Optional, Dict, Any, Sequence, Callable
 
 from jinja2 import Template
 
-from trilogy.core.processing.utility import is_scalar_condition, decompose_condition
+from trilogy.core.processing.utility import (
+    is_scalar_condition,
+    decompose_condition,
+    sort_select_output,
+)
 from trilogy.constants import CONFIG, logger, MagicConstants
 from trilogy.core.internal import DEFAULT_CONCEPTS
 from trilogy.core.enums import (
@@ -537,7 +541,7 @@ class BaseDialect:
         else:
             raise ValueError(f"Unable to render type {type(e)} {e}")
 
-    def render_cte(self, cte: CTE):
+    def render_cte(self, cte: CTE, auto_sort: bool = True):
         if self.UNNEST_MODE in (
             UnnestMode.CROSS_APPLY,
             UnnestMode.CROSS_JOIN,
@@ -561,6 +565,8 @@ class BaseDialect:
                 for c in cte.output_columns
                 if c.address not in [y.address for y in cte.hidden_concepts]
             ]
+        if auto_sort:
+            select_columns = sorted(select_columns, key=lambda x: x)
         source: str | None = cte.base_name
         if not cte.render_from_clause:
             if len(cte.joins) > 0:
@@ -657,8 +663,11 @@ class BaseDialect:
     def generate_ctes(
         self,
         query: ProcessedQuery,
-    ):
-        return [self.render_cte(cte) for cte in query.ctes]
+    ) -> List[CompiledCTE]:
+        return [self.render_cte(cte) for cte in query.ctes[:-1]] + [
+            # last CTE needs to respect the user output order
+            self.render_cte(sort_select_output(query.ctes[-1], query), auto_sort=False)
+        ]
 
     def generate_queries(
         self,
@@ -790,11 +799,6 @@ class BaseDialect:
 
         compiled_ctes = self.generate_ctes(query)
 
-        # restort selections by the order they were written in
-        sorted_select: List[str] = []
-        for output_c in output_addresses:
-            sorted_select.append(select_columns[output_c])
-
         final = self.SQL_TEMPLATE.render(
             output=(
                 query.output_to if isinstance(query, ProcessedQueryPersist) else None

{pytrilogy-0.0.2.27 → pytrilogy-0.0.2.29}/trilogy/executor.py

@@ -276,6 +276,20 @@ class Executor(object):
             output.append(compiled_sql)
         return output
 
+    def parse_file(self, file: str | Path, persist: bool = False) -> Generator[
+        ProcessedQuery
+        | ProcessedQueryPersist
+        | ProcessedShowStatement
+        | ProcessedRawSQLStatement
+        | ProcessedCopyStatement,
+        None,
+        None,
+    ]:
+        file = Path(file)
+        with open(file, "r") as f:
+            command = f.read()
+            return self.parse_text_generator(command, persist=persist)
+
     def parse_text(
         self, command: str, persist: bool = False
     ) -> List[
@@ -319,9 +333,11 @@ class Executor(object):
             x = self.generator.generate_queries(
                 self.environment, [t], hooks=self.hooks
             )[0]
+
+            yield x
+
             if persist and isinstance(x, ProcessedQueryPersist):
                 self.environment.add_datasource(x.datasource)
-            yield x
 
     def execute_raw_sql(
         self, command: str, variables: dict | None = None