pytrilogy 0.0.2.39__tar.gz → 0.0.2.41__tar.gz

{pytrilogy-0.0.2.39/pytrilogy.egg-info → pytrilogy-0.0.2.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.39
+Version: 0.0.2.41
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -186,7 +186,7 @@ datasource usa_names(
     gender:gender,
     state:state
 )
-address bigquery-public-data.usa_names.usa_1910_2013;
+address `bigquery-public-data.usa_names.usa_1910_2013`;
 
 '''
 )

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/README.md

@@ -156,7 +156,7 @@ datasource usa_names(
     gender:gender,
     state:state
 )
-address bigquery-public-data.usa_names.usa_1910_2013;
+address `bigquery-public-data.usa_names.usa_1910_2013`;
 
 '''
 )

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.39
+Version: 0.0.2.41
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -186,7 +186,7 @@ datasource usa_names(
     gender:gender,
     state:state
 )
-address bigquery-public-data.usa_names.usa_1910_2013;
+address `bigquery-public-data.usa_names.usa_1910_2013`;
 
 '''
 )

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/tests/test_select.py

@@ -19,7 +19,7 @@ datasource posts (
     id: post_id
     )
     grain (post_id)
-    address bigquery-public-data.stackoverflow.post_history
+    address `bigquery-public-data.stackoverflow.post_history`
 ;
 
 
@@ -29,7 +29,7 @@ datasource users (
     about_me: about_me,
     )
     grain (user_id)
-    address bigquery-public-data.stackoverflow.users
+    address `bigquery-public-data.stackoverflow.users`
 ;
 
 
@@ -70,7 +70,7 @@ def test_double_aggregate():
         id: post_id
         )
         grain (post_id)
-        address bigquery-public-data.stackoverflow.post_history
+        address `bigquery-public-data.stackoverflow.post_history`
     ;
 
 
@@ -80,7 +80,7 @@ def test_double_aggregate():
         about_me: about_me,
         )
         grain (user_id)
-        address bigquery-public-data.stackoverflow.users
+        address `bigquery-public-data.stackoverflow.users`
     ;
 
 

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/tests/test_show.py

@@ -16,7 +16,7 @@ def test_show_bigquery():
         id: post_id
         )
         grain (post_id)
-        address bigquery-public-data.stackoverflow.post_history
+        address `bigquery-public-data.stackoverflow.post_history`
     ;
 
 
@@ -26,7 +26,7 @@ def test_show_bigquery():
         about_me: about_me,
         )
         grain (user_id)
-        address bigquery-public-data.stackoverflow.users
+        address `bigquery-public-data.stackoverflow.users`
     ;
 
 

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/tests/test_statements.py

@@ -40,7 +40,7 @@ datasource posts (
     id: post_id
     )
     grain (post_id)
-    address bigquery-public-data.stackoverflow.post_history
+    address `bigquery-public-data.stackoverflow.post_history`
 ;
 """
     parse(text)
@@ -76,7 +76,7 @@ datasource x_posts (
     id: post_id
     )
     grain (post_id)
-    address bigquery-public-data.stackoverflow.post_history
+    address `bigquery-public-data.stackoverflow.post_history`
     where post_id = 2
 ;
 """

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.39"
+__version__ = "0.0.2.41"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/trilogy/core/models.py

@@ -448,8 +448,8 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     ] = None
     namespace: Optional[str] = Field(default=DEFAULT_NAMESPACE, validate_default=True)
     keys: Optional[Tuple["Concept", ...]] = None
-    grain: "Grain" = Field(default=None, validate_default=True)
-    modifiers: Optional[List[Modifier]] = Field(default_factory=list)
+    grain: "Grain" = Field(default=None, validate_default=True)  # type: ignore
+    modifiers: List[Modifier] = Field(default_factory=list)  # type: ignore
     pseudonyms: set[str] = Field(default_factory=set)
     _address_cache: str | None = None
 
@@ -1375,6 +1375,9 @@ class WindowItem(Mergeable, Namespaced, SelectContext, BaseModel):
     over: List["Concept"] = Field(default_factory=list)
     index: Optional[int] = None
 
+    def __repr__(self) -> str:
+        return f"{self.type}({self.content} {self.index}, {self.over}, {self.order_by})"
+
     def with_merge(
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ) -> "WindowItem":
@@ -1383,6 +1386,7 @@ class WindowItem(Mergeable, Namespaced, SelectContext, BaseModel):
             content=self.content.with_merge(source, target, modifiers),
             over=[x.with_merge(source, target, modifiers) for x in self.over],
             order_by=[x.with_merge(source, target, modifiers) for x in self.order_by],
+            index=self.index,
         )
 
     def with_namespace(self, namespace: str) -> "WindowItem":
@@ -1391,6 +1395,7 @@ class WindowItem(Mergeable, Namespaced, SelectContext, BaseModel):
             content=self.content.with_namespace(namespace),
             over=[x.with_namespace(namespace) for x in self.over],
             order_by=[x.with_namespace(namespace) for x in self.order_by],
+            index=self.index,
         )
 
     def with_select_context(
@@ -1410,6 +1415,7 @@ class WindowItem(Mergeable, Namespaced, SelectContext, BaseModel):
                 x.with_select_context(grain, conditional, environment)
                 for x in self.order_by
             ],
+            index=self.index,
         )
 
     @property
@@ -3372,7 +3378,7 @@ class Environment(BaseModel):
     functions: Dict[str, Function] = Field(default_factory=dict)
     data_types: Dict[str, DataType] = Field(default_factory=dict)
     imports: Dict[str, list[ImportStatement]] = Field(
-        default_factory=lambda: defaultdict(list)
+        default_factory=lambda: defaultdict(list)  # type: ignore
     )
     namespace: str = DEFAULT_NAMESPACE
     working_path: str | Path = Field(default_factory=lambda: os.getcwd())

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/trilogy/core/optimizations/predicate_pushdown.py

@@ -6,6 +6,7 @@ from trilogy.core.models import (
     ConceptArgs,
     Comparison,
     Parenthetical,
+    WindowItem,
 )
 from trilogy.core.optimizations.base_optimization import OptimizationRule
 from trilogy.core.processing.utility import is_scalar_condition
@@ -45,7 +46,15 @@ class PredicatePushdown(OptimizationRule):
         all_inputs = {x.address for x in candidate.concept_arguments}
         if is_child_of(candidate, parent_cte.condition):
             return False
-
+        non_materialized = [k for k, v in parent_cte.source_map.items() if v == []]
+        concrete = [
+            x for x in parent_cte.output_columns if x.address in non_materialized
+        ]
+        if any(isinstance(x.lineage, WindowItem) for x in concrete):
+            self.debug(
+                f"CTE {parent_cte.name} has window clause calculation, cannot push up to this without changing results"
+            )
+            return False
         materialized = {k for k, v in parent_cte.source_map.items() if v != []}
         if not row_conditions or not materialized:
             return False
@@ -108,12 +117,12 @@ class PredicatePushdown(OptimizationRule):
 
         if not cte.parent_ctes:
             self.debug(f"No parent CTEs for {cte.name}")
-
             return False
 
         if not cte.condition:
             self.debug(f"No CTE condition for {cte.name}")
             return False
+
         if self.complete.get(cte.name):
             self.debug("Have done this CTE before")
             return False

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/trilogy/core/processing/node_generators/basic_node.py

@@ -1,7 +1,12 @@
 # directly select out a basic derivation
 from typing import List
 
-from trilogy.core.models import Concept, WhereClause
+from trilogy.core.models import (
+    Concept,
+    WhereClause,
+    Function,
+    FunctionClass,
+)
 from trilogy.core.processing.nodes import StrategyNode, History
 from trilogy.core.processing.node_generators.common import (
     resolve_function_parent_concepts,
@@ -12,6 +17,22 @@ from trilogy.core.enums import SourceType
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
 
 
+def is_equivalent_basic_function_lineage(
+    x: Concept,
+    y: Concept,
+):
+    if not isinstance(x.lineage, Function) or not isinstance(y.lineage, Function):
+        return False
+    if x.lineage.operator == y.lineage.operator:
+        return True
+    if (
+        y.lineage.operator in FunctionClass.AGGREGATE_FUNCTIONS.value
+        or y.lineage.operator in FunctionClass.ONE_TO_MANY.value
+    ):
+        return False
+    return True
+
+
 def gen_basic_node(
     concept: Concept,
     local_optional: List[Concept],
@@ -32,8 +53,15 @@ def gen_basic_node(
     equivalent_optional = [
         x
         for x in local_optional
-        if x.lineage == concept.lineage and x.address != concept.address
+        if is_equivalent_basic_function_lineage(concept, x)
+        and x.address != concept.address
     ]
+    if equivalent_optional:
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has equivalent optional {[x.address for x in equivalent_optional]}"
+        )
+    for eo in equivalent_optional:
+        parent_concepts += resolve_function_parent_concepts(eo)
     non_equivalent_optional = [
         x for x in local_optional if x not in equivalent_optional
     ]

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/trilogy/core/processing/node_generators/window_node.py

@@ -10,16 +10,16 @@ from trilogy.core.processing.utility import padding
 LOGGER_PREFIX = "[GEN_WINDOW_NODE]"
 
 
-def resolve_window_parent_concepts(concept: Concept) -> List[Concept]:
+def resolve_window_parent_concepts(concept: Concept) -> tuple[Concept, List[Concept]]:
     if not isinstance(concept.lineage, WindowItem):
         raise ValueError
-    base = [concept.lineage.content]
+    base = []
     if concept.lineage.over:
         base += concept.lineage.over
     if concept.lineage.order_by:
         for item in concept.lineage.order_by:
             base += [item.expr.output]
-    return unique(base, "address")
+    return concept.lineage.content, unique(base, "address")
 
 
 def gen_window_node(
@@ -32,9 +32,25 @@ def gen_window_node(
     history: History | None = None,
     conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
-    parent_concepts = resolve_window_parent_concepts(concept)
-    parent_node = source_concepts(
-        mandatory_list=parent_concepts + local_optional,
+    base, parent_concepts = resolve_window_parent_concepts(concept)
+    equivalent_optional = [
+        x
+        for x in local_optional
+        if isinstance(x.lineage, WindowItem)
+        and resolve_window_parent_concepts(x)[1] == parent_concepts
+    ]
+
+    non_equivalent_optional = [
+        x for x in local_optional if x.address not in equivalent_optional
+    ]
+    targets = [base]
+    if equivalent_optional:
+        for x in equivalent_optional:
+            assert isinstance(x.lineage, WindowItem)
+            targets.append(x.lineage.content)
+
+    parent_node: StrategyNode = source_concepts(
+        mandatory_list=parent_concepts + targets + non_equivalent_optional,
         environment=environment,
         g=g,
         depth=depth + 1,
@@ -61,7 +77,7 @@ def gen_window_node(
         )
         raise SyntaxError
     _window_node = WindowNode(
-        input_concepts=parent_concepts + local_optional,
+        input_concepts=parent_concepts + targets + non_equivalent_optional,
         output_concepts=[concept] + parent_concepts + local_optional,
         environment=environment,
         g=g,

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/trilogy/dialect/base.py

@@ -75,7 +75,7 @@ def window_factory(string: str, include_concept: bool = False) -> Callable:
     ) -> str:
         if not include_concept:
             concept = ""
-        if offset:
+        if offset is not None:
             base = f"{string}({concept}, {offset})"
         else:
             base = f"{string}({concept})"
@@ -138,10 +138,10 @@ FUNCTION_MAP = {
     FunctionType.STRUCT: lambda x: f"{{{', '.join(struct_arg(x))}}}",
     FunctionType.ARRAY: lambda x: f"[{', '.join(x)}]",
     # math
-    FunctionType.ADD: lambda x: f"{x[0]} + {x[1]}",
-    FunctionType.SUBTRACT: lambda x: f"{x[0]} - {x[1]}",
-    FunctionType.DIVIDE: lambda x: f"{x[0]} / {x[1]}",
-    FunctionType.MULTIPLY: lambda x: f"{x[0]} * {x[1]}",
+    FunctionType.ADD: lambda x: " + ".join(x),
+    FunctionType.SUBTRACT: lambda x: " - ".join(x),
+    FunctionType.DIVIDE: lambda x: " / ".join(x),
+    FunctionType.MULTIPLY: lambda x: " * ".join(x),
     FunctionType.ROUND: lambda x: f"round({x[0]},{x[1]})",
     FunctionType.MOD: lambda x: f"({x[0]} % {x[1]})",
     # aggregate types
@@ -313,7 +313,10 @@ class BaseDialect:
                     )
                     for x in c.lineage.over
                 ]
-                rval = f"{self.WINDOW_FUNCTION_MAP[c.lineage.type](concept = self.render_concept_sql(c.lineage.content, cte=cte, alias=False, raise_invalid=raise_invalid), window=','.join(rendered_over_components), sort=','.join(rendered_order_components))}"  # noqa: E501
+                rval = f"{self.WINDOW_FUNCTION_MAP[c.lineage.type](concept = self.render_concept_sql(c.lineage.content,
+                                                                                                      cte=cte, alias=False, raise_invalid=raise_invalid), 
+                                                                   window=','.join(rendered_over_components), sort=','.join(rendered_order_components), 
+                                                                   offset=c.lineage.index)}"  # noqa: E501
             elif isinstance(c.lineage, FilterItem):
                 # for cases when we've optimized this
                 if cte.condition == c.lineage.where.conditional:

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/trilogy/parsing/parse_engine.py

@@ -1777,7 +1777,7 @@ class ParseToObjects(Transformer):
             output_datatype=output_datatype,
             output_purpose=function_args_to_output_purpose(args),
             valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
-            arg_count=2,
+            arg_count=-1,
         )
 
     @v_args(meta=True)
@@ -1790,7 +1790,7 @@ class ParseToObjects(Transformer):
             output_datatype=output_datatype,
             output_purpose=function_args_to_output_purpose(args),
             valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
-            arg_count=2,
+            arg_count=-1,
         )
 
     @v_args(meta=True)
@@ -1803,7 +1803,7 @@ class ParseToObjects(Transformer):
             output_datatype=output_datatype,
             output_purpose=function_args_to_output_purpose(args),
             valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
-            arg_count=2,
+            arg_count=-1,
         )
 
     @v_args(meta=True)
@@ -1817,7 +1817,7 @@ class ParseToObjects(Transformer):
             output_datatype=DataType.FLOAT,  # division always returns a float
             output_purpose=function_args_to_output_purpose(args),
             valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
-            arg_count=2,
+            arg_count=-1,
         )
 
     @v_args(meta=True)

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/trilogy/parsing/render.py

@@ -395,13 +395,13 @@ class Renderer:
         args = [self.to_string(c) for c in arg.arguments]
 
         if arg.operator == FunctionType.SUBTRACT:
-            return f"{args[0]} - {args[1]}"
+            return " - ".join(args)
         if arg.operator == FunctionType.ADD:
-            return f"{args[0]} + {args[1]}"
+            return " + ".join(args)
         if arg.operator == FunctionType.MULTIPLY:
-            return f"{args[0]} * {args[1]}"
+            return " * ".join(args)
         if arg.operator == FunctionType.DIVIDE:
-            return f"{args[0]} / {args[1]}"
+            return " / ".join(args)
         if arg.operator == FunctionType.MOD:
             return f"{args[0]} % {args[1]}"
 

{pytrilogy-0.0.2.39 → pytrilogy-0.0.2.41}/trilogy/parsing/trilogy.lark

@@ -290,7 +290,7 @@
 
     // base language constructs
     concept_lit: IDENTIFIER
-    IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\-\.]*/
+    IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\.]*/
     WILDCARD_IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\-\.\*]*/
     QUOTED_IDENTIFIER:  /`[a-zA-Z\_][a-zA-Z0-9\_\.\-\*\:\s]*`/
     QUOTED_ADDRESS:  /`[a-zA-Z\_][a-zA-Z0-9\_\.\-\*\:]*`/