pytrilogy 0.0.2.40__tar.gz → 0.0.2.42__tar.gz

{pytrilogy-0.0.2.40/pytrilogy.egg-info → pytrilogy-0.0.2.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.40
+Version: 0.0.2.42
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -186,7 +186,7 @@ datasource usa_names(
     gender:gender,
     state:state
 )
-address bigquery-public-data.usa_names.usa_1910_2013;
+address `bigquery-public-data.usa_names.usa_1910_2013`;
 
 '''
 )

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/README.md

@@ -156,7 +156,7 @@ datasource usa_names(
     gender:gender,
     state:state
 )
-address bigquery-public-data.usa_names.usa_1910_2013;
+address `bigquery-public-data.usa_names.usa_1910_2013`;
 
 '''
 )

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.40
+Version: 0.0.2.42
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -186,7 +186,7 @@ datasource usa_names(
     gender:gender,
     state:state
 )
-address bigquery-public-data.usa_names.usa_1910_2013;
+address `bigquery-public-data.usa_names.usa_1910_2013`;
 
 '''
 )

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/tests/test_functions.py

@@ -89,6 +89,16 @@ def test_window_functions(test_environment):
         dialect.compile_statement(process_query(test_environment, select))
 
 
+def test_window_datatype(test_environment):
+    declarations = """
+
+    auto category_rank <- rank category_name order by count(order_id) desc;
+
+        """
+    env, parsed = parse(declarations, environment=test_environment)
+    assert env.concepts["category_rank"].datatype == DataType.INTEGER
+
+
 def test_date_functions(test_environment):
     declarations = """
 

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/tests/test_select.py

@@ -19,7 +19,7 @@ datasource posts (
     id: post_id
     )
     grain (post_id)
-    address bigquery-public-data.stackoverflow.post_history
+    address `bigquery-public-data.stackoverflow.post_history`
 ;
 
 
@@ -29,7 +29,7 @@ datasource users (
     about_me: about_me,
     )
     grain (user_id)
-    address bigquery-public-data.stackoverflow.users
+    address `bigquery-public-data.stackoverflow.users`
 ;
 
 
@@ -70,7 +70,7 @@ def test_double_aggregate():
         id: post_id
         )
         grain (post_id)
-        address bigquery-public-data.stackoverflow.post_history
+        address `bigquery-public-data.stackoverflow.post_history`
     ;
 
 
@@ -80,7 +80,7 @@ def test_double_aggregate():
         about_me: about_me,
         )
         grain (user_id)
-        address bigquery-public-data.stackoverflow.users
+        address `bigquery-public-data.stackoverflow.users`
     ;
 
 

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/tests/test_show.py

@@ -16,7 +16,7 @@ def test_show_bigquery():
         id: post_id
         )
         grain (post_id)
-        address bigquery-public-data.stackoverflow.post_history
+        address `bigquery-public-data.stackoverflow.post_history`
     ;
 
 
@@ -26,7 +26,7 @@ def test_show_bigquery():
         about_me: about_me,
         )
         grain (user_id)
-        address bigquery-public-data.stackoverflow.users
+        address `bigquery-public-data.stackoverflow.users`
     ;
 
 

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/tests/test_statements.py

@@ -40,7 +40,7 @@ datasource posts (
     id: post_id
     )
     grain (post_id)
-    address bigquery-public-data.stackoverflow.post_history
+    address `bigquery-public-data.stackoverflow.post_history`
 ;
 """
     parse(text)
@@ -76,7 +76,7 @@ datasource x_posts (
     id: post_id
     )
     grain (post_id)
-    address bigquery-public-data.stackoverflow.post_history
+    address `bigquery-public-data.stackoverflow.post_history`
     where post_id = 2
 ;
 """

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.40"
+__version__ = "0.0.2.42"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/trilogy/core/models.py

@@ -1375,6 +1375,9 @@ class WindowItem(Mergeable, Namespaced, SelectContext, BaseModel):
     over: List["Concept"] = Field(default_factory=list)
     index: Optional[int] = None
 
+    def __repr__(self) -> str:
+        return f"{self.type}({self.content} {self.index}, {self.over}, {self.order_by})"
+
     def with_merge(
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ) -> "WindowItem":
@@ -1383,6 +1386,7 @@ class WindowItem(Mergeable, Namespaced, SelectContext, BaseModel):
             content=self.content.with_merge(source, target, modifiers),
             over=[x.with_merge(source, target, modifiers) for x in self.over],
             order_by=[x.with_merge(source, target, modifiers) for x in self.order_by],
+            index=self.index,
         )
 
     def with_namespace(self, namespace: str) -> "WindowItem":
@@ -1391,6 +1395,7 @@ class WindowItem(Mergeable, Namespaced, SelectContext, BaseModel):
             content=self.content.with_namespace(namespace),
             over=[x.with_namespace(namespace) for x in self.over],
             order_by=[x.with_namespace(namespace) for x in self.order_by],
+            index=self.index,
         )
 
     def with_select_context(
@@ -1410,6 +1415,7 @@ class WindowItem(Mergeable, Namespaced, SelectContext, BaseModel):
                 x.with_select_context(grain, conditional, environment)
                 for x in self.order_by
             ],
+            index=self.index,
         )
 
     @property

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/trilogy/core/optimizations/predicate_pushdown.py

@@ -6,6 +6,7 @@ from trilogy.core.models import (
     ConceptArgs,
     Comparison,
     Parenthetical,
+    WindowItem,
 )
 from trilogy.core.optimizations.base_optimization import OptimizationRule
 from trilogy.core.processing.utility import is_scalar_condition
@@ -45,7 +46,15 @@ class PredicatePushdown(OptimizationRule):
         all_inputs = {x.address for x in candidate.concept_arguments}
         if is_child_of(candidate, parent_cte.condition):
             return False
-
+        non_materialized = [k for k, v in parent_cte.source_map.items() if v == []]
+        concrete = [
+            x for x in parent_cte.output_columns if x.address in non_materialized
+        ]
+        if any(isinstance(x.lineage, WindowItem) for x in concrete):
+            self.debug(
+                f"CTE {parent_cte.name} has window clause calculation, cannot push up to this without changing results"
+            )
+            return False
         materialized = {k for k, v in parent_cte.source_map.items() if v != []}
         if not row_conditions or not materialized:
             return False
@@ -108,12 +117,12 @@ class PredicatePushdown(OptimizationRule):
 
         if not cte.parent_ctes:
             self.debug(f"No parent CTEs for {cte.name}")
-
             return False
 
         if not cte.condition:
             self.debug(f"No CTE condition for {cte.name}")
             return False
+
         if self.complete.get(cte.name):
             self.debug("Have done this CTE before")
             return False

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/trilogy/core/processing/node_generators/basic_node.py

@@ -1,7 +1,12 @@
 # directly select out a basic derivation
 from typing import List
 
-from trilogy.core.models import Concept, WhereClause
+from trilogy.core.models import (
+    Concept,
+    WhereClause,
+    Function,
+    FunctionClass,
+)
 from trilogy.core.processing.nodes import StrategyNode, History
 from trilogy.core.processing.node_generators.common import (
     resolve_function_parent_concepts,
@@ -12,6 +17,22 @@ from trilogy.core.enums import SourceType
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
 
 
+def is_equivalent_basic_function_lineage(
+    x: Concept,
+    y: Concept,
+):
+    if not isinstance(x.lineage, Function) or not isinstance(y.lineage, Function):
+        return False
+    if x.lineage.operator == y.lineage.operator:
+        return True
+    if (
+        y.lineage.operator in FunctionClass.AGGREGATE_FUNCTIONS.value
+        or y.lineage.operator in FunctionClass.ONE_TO_MANY.value
+    ):
+        return False
+    return True
+
+
 def gen_basic_node(
     concept: Concept,
     local_optional: List[Concept],
@@ -32,8 +53,15 @@ def gen_basic_node(
     equivalent_optional = [
         x
         for x in local_optional
-        if x.lineage == concept.lineage and x.address != concept.address
+        if is_equivalent_basic_function_lineage(concept, x)
+        and x.address != concept.address
     ]
+    if equivalent_optional:
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has equivalent optional {[x.address for x in equivalent_optional]}"
+        )
+    for eo in equivalent_optional:
+        parent_concepts += resolve_function_parent_concepts(eo)
     non_equivalent_optional = [
         x for x in local_optional if x not in equivalent_optional
     ]

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/trilogy/core/processing/node_generators/window_node.py

@@ -10,16 +10,23 @@ from trilogy.core.processing.utility import padding
 LOGGER_PREFIX = "[GEN_WINDOW_NODE]"
 
 
-def resolve_window_parent_concepts(concept: Concept) -> List[Concept]:
+def resolve_window_parent_concepts(
+    concept: Concept, environment: Environment
+) -> tuple[Concept, List[Concept]]:
     if not isinstance(concept.lineage, WindowItem):
         raise ValueError
-    base = [concept.lineage.content]
+    base = []
     if concept.lineage.over:
         base += concept.lineage.over
     if concept.lineage.order_by:
         for item in concept.lineage.order_by:
+            # TODO: we do want to use the rehydrated value, but
+            # that introduces a circular dependency on an aggregate
+            # that is grouped by a window
+            # need to figure out how to resolve this
+            # base += [environment.concepts[item.expr.output.address]]
             base += [item.expr.output]
-    return unique(base, "address")
+    return concept.lineage.content, unique(base, "address")
 
 
 def gen_window_node(
@@ -32,9 +39,25 @@ def gen_window_node(
     history: History | None = None,
     conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
-    parent_concepts = resolve_window_parent_concepts(concept)
-    parent_node = source_concepts(
-        mandatory_list=parent_concepts + local_optional,
+    base, parent_concepts = resolve_window_parent_concepts(concept, environment)
+    equivalent_optional = [
+        x
+        for x in local_optional
+        if isinstance(x.lineage, WindowItem)
+        and resolve_window_parent_concepts(x, environment)[1] == parent_concepts
+    ]
+
+    non_equivalent_optional = [
+        x for x in local_optional if x.address not in equivalent_optional
+    ]
+    targets = [base]
+    if equivalent_optional:
+        for x in equivalent_optional:
+            assert isinstance(x.lineage, WindowItem)
+            targets.append(x.lineage.content)
+
+    parent_node: StrategyNode = source_concepts(
+        mandatory_list=parent_concepts + targets + non_equivalent_optional,
         environment=environment,
         g=g,
         depth=depth + 1,
@@ -61,7 +84,7 @@ def gen_window_node(
         )
         raise SyntaxError
     _window_node = WindowNode(
-        input_concepts=parent_concepts + local_optional,
+        input_concepts=parent_concepts + targets + non_equivalent_optional,
         output_concepts=[concept] + parent_concepts + local_optional,
         environment=environment,
         g=g,

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/trilogy/dialect/base.py

@@ -75,7 +75,7 @@ def window_factory(string: str, include_concept: bool = False) -> Callable:
     ) -> str:
         if not include_concept:
             concept = ""
-        if offset:
+        if offset is not None:
             base = f"{string}({concept}, {offset})"
         else:
             base = f"{string}({concept})"
@@ -313,7 +313,10 @@ class BaseDialect:
                     )
                     for x in c.lineage.over
                 ]
-                rval = f"{self.WINDOW_FUNCTION_MAP[c.lineage.type](concept = self.render_concept_sql(c.lineage.content, cte=cte, alias=False, raise_invalid=raise_invalid), window=','.join(rendered_over_components), sort=','.join(rendered_order_components))}"  # noqa: E501
+                rval = f"{self.WINDOW_FUNCTION_MAP[c.lineage.type](concept = self.render_concept_sql(c.lineage.content,
+                                                                                                      cte=cte, alias=False, raise_invalid=raise_invalid), 
+                                                                   window=','.join(rendered_over_components), sort=','.join(rendered_order_components), 
+                                                                   offset=c.lineage.index)}"  # noqa: E501
             elif isinstance(c.lineage, FilterItem):
                 # for cases when we've optimized this
                 if cte.condition == c.lineage.where.conditional:

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/trilogy/parsing/common.py

@@ -13,6 +13,7 @@ from trilogy.core.models import (
     Parenthetical,
     FunctionClass,
     Environment,
+    DataType,
 )
 from typing import List, Tuple
 from trilogy.core.functions import (
@@ -25,7 +26,7 @@ from trilogy.core.enums import PurposeLineage
 from trilogy.constants import (
     VIRTUAL_CONCEPT_PREFIX,
 )
-from trilogy.core.enums import Modifier
+from trilogy.core.enums import Modifier, WindowType
 
 
 def get_upstream_modifiers(keys: List[Concept]) -> list[Modifier]:
@@ -234,9 +235,18 @@ def window_item_to_concept(
     else:
         grain = parent.over + [parent.content.output]
     modifiers = get_upstream_modifiers(parent.content.concept_arguments)
+    datatype = parent.content.datatype
+    if parent.type in (
+        WindowType.RANK,
+        WindowType.ROW_NUMBER,
+        WindowType.COUNT,
+        WindowType.COUNT_DISTINCT,
+    ):
+        datatype = DataType.INTEGER
+
     return Concept(
         name=name,
-        datatype=parent.content.datatype,
+        datatype=datatype,
         purpose=local_purpose,
         lineage=parent,
         metadata=fmetadata,

{pytrilogy-0.0.2.40 → pytrilogy-0.0.2.42}/trilogy/parsing/trilogy.lark

@@ -290,7 +290,7 @@
 
     // base language constructs
     concept_lit: IDENTIFIER
-    IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\-\.]*/
+    IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\.]*/
     WILDCARD_IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\-\.\*]*/
     QUOTED_IDENTIFIER:  /`[a-zA-Z\_][a-zA-Z0-9\_\.\-\*\:\s]*`/
     QUOTED_ADDRESS:  /`[a-zA-Z\_][a-zA-Z0-9\_\.\-\*\:]*`/