pytrilogy 0.0.1.116__tar.gz → 0.0.1.117__tar.gz

{pytrilogy-0.0.1.116/pytrilogy.egg-info → pytrilogy-0.0.1.117}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.116
+Version: 0.0.1.117
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.116
+Version: 0.0.1.117
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117}/pytrilogy.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ pytrilogy.egg-info/dependency_links.txt
 pytrilogy.egg-info/entry_points.txt
 pytrilogy.egg-info/requires.txt
 pytrilogy.egg-info/top_level.txt
+tests/test_datatypes.py
 tests/test_declarations.py
 tests/test_derived_concepts.py
 tests/test_discovery_nodes.py

pytrilogy-0.0.1.117/tests/test_datatypes.py

@@ -0,0 +1,13 @@
+from trilogy.core.models import (
+    NumericType,
+)
+from trilogy.parsing.parse_engine import (
+    parse_text,
+)
+
+
+def test_numeric():
+    env, _ = parse_text(
+        "const order_id numeric(12,2); const rounded <- cast(order_id as numeric(15,2));"
+    )
+    assert env.concepts["order_id"].datatype == NumericType(precision=12, scale=2)

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.1.116"
+__version__ = "0.0.1.117"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117}/trilogy/core/functions.py

@@ -9,6 +9,7 @@ from trilogy.core.models import (
     ListType,
     StructType,
     MapType,
+    NumericType,
 )
 from trilogy.core.enums import FunctionType, Purpose, Granularity, DatePart
 from trilogy.core.exceptions import InvalidSyntaxException
@@ -21,7 +22,9 @@ def create_function_derived_concept(
     namespace: str,
     operator: FunctionType,
     arguments: list[Concept],
-    output_type: Optional[DataType | ListType | StructType | MapType] = None,
+    output_type: Optional[
+        DataType | ListType | StructType | MapType | NumericType
+    ] = None,
     output_purpose: Optional[Purpose] = None,
 ) -> Concept:
     purpose = (
@@ -56,9 +59,7 @@ def argument_to_purpose(arg) -> Purpose:
         return Purpose.PROPERTY
     elif isinstance(arg, Concept):
         return arg.purpose
-    elif isinstance(arg, (int, float, str, bool, list)):
-        return Purpose.CONSTANT
-    elif isinstance(arg, DataType):
+    elif isinstance(arg, (int, float, str, bool, list, NumericType, DataType)):
         return Purpose.CONSTANT
     elif isinstance(arg, DatePart):
         return Purpose.CONSTANT

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117}/trilogy/core/models.py

@@ -105,7 +105,9 @@ def get_concept_arguments(expr) -> List["Concept"]:
     return output
 
 
-ALL_TYPES = Union["DataType", "MapType", "ListType", "StructType", "Concept"]
+ALL_TYPES = Union[
+    "DataType", "MapType", "ListType", "NumericType", "StructType", "Concept"
+]
 
 NAMESPACED_TYPES = Union[
     "WindowItem",
@@ -176,6 +178,19 @@ class DataType(Enum):
         return self
 
 
+class NumericType(BaseModel):
+    precision: int = 20
+    scale: int = 5
+
+    @property
+    def data_type(self):
+        return DataType.NUMERIC
+
+    @property
+    def value(self):
+        return self.data_type.value
+
+
 class ListType(BaseModel):
     model_config = ConfigDict(frozen=True)
     type: ALL_TYPES
@@ -192,7 +207,9 @@ class ListType(BaseModel):
         return self.data_type.value
 
     @property
-    def value_data_type(self) -> DataType | StructType | MapType | ListType:
+    def value_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
         if isinstance(self.type, Concept):
             return self.type.datatype
         return self.type
@@ -270,7 +287,7 @@ def empty_grain() -> Grain:
 
 class Concept(Namespaced, SelectGrain, BaseModel):
     name: str
-    datatype: DataType | ListType | StructType | MapType
+    datatype: DataType | ListType | StructType | MapType | NumericType
     purpose: Purpose
     metadata: Optional[Metadata] = Field(
         default_factory=lambda: Metadata(description=None, line_number=None),
@@ -790,12 +807,12 @@ class LooseConceptList(BaseModel):
 class Function(Namespaced, SelectGrain, BaseModel):
     operator: FunctionType
     arg_count: int = Field(default=1)
-    output_datatype: DataType | ListType | StructType | MapType
+    output_datatype: DataType | ListType | StructType | MapType | NumericType
     output_purpose: Purpose
     valid_inputs: Optional[
         Union[
-            Set[DataType | ListType | StructType],
-            List[Set[DataType | ListType | StructType]],
+            Set[DataType | ListType | StructType | NumericType],
+            List[Set[DataType | ListType | StructType] | NumericType],
         ]
     ] = None
     arguments: Sequence[
@@ -808,6 +825,7 @@ class Function(Namespaced, SelectGrain, BaseModel):
             str,
             DataType,
             ListType,
+            NumericType,
             DatePart,
             "Parenthetical",
             CaseWhen,
@@ -1499,7 +1517,7 @@ class DatasourceMetadata(BaseModel):
 
 class MergeStatement(Namespaced, BaseModel):
     concepts: List[Concept]
-    datatype: DataType | ListType | StructType | MapType
+    datatype: DataType | ListType | StructType | MapType | NumericType
 
     @cached_property
     def concepts_lcl(self):
@@ -3498,7 +3516,7 @@ def list_to_wrapper(args):
     return ListWrapper(args, type=types[0])
 
 
-def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType:
+def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | NumericType:
     if isinstance(arg, Function):
         return arg.output_datatype
     elif isinstance(arg, Concept):
@@ -3511,6 +3529,8 @@ def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType:
         return DataType.STRING
     elif isinstance(arg, float):
         return DataType.FLOAT
+    elif isinstance(arg, NumericType):
+        return arg
     elif isinstance(arg, ListWrapper):
         return ListType(type=arg.type)
     elif isinstance(arg, AggregateWrapper):

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117}/trilogy/core/optimization.py

@@ -50,7 +50,9 @@ def is_direct_return_eligible(
     if isinstance(select, (PersistStatement, MultiSelectStatement)):
         return False
     derived_concepts = [
-        c for c in cte.source.output_concepts if c not in cte.source.input_concepts
+        c
+        for c in cte.source.output_concepts + cte.source.hidden_concepts
+        if c not in cte.source.input_concepts
     ]
     eligible = True
     conditions = (
@@ -90,26 +92,9 @@ def sort_select_output(cte: CTE, query: SelectStatement | MultiSelectStatement):
 
 def optimize_ctes(
     input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
-):
+) -> list[CTE]:
     complete = False
     REGISTERED_RULES: list["OptimizationRule"] = []
-
-    if CONFIG.optimizations.datasource_inlining:
-        REGISTERED_RULES.append(InlineDatasource())
-    if CONFIG.optimizations.predicate_pushdown:
-        REGISTERED_RULES.append(PredicatePushdown())
-    if CONFIG.optimizations.constant_inlining:
-        REGISTERED_RULES.append(InlineConstant())
-    loops = 0
-    while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
-        actions_taken = False
-        for rule in REGISTERED_RULES:
-            for cte in input:
-                inverse_map = gen_inverse_map(input)
-                actions_taken = rule.optimize(cte, inverse_map)
-        complete = not actions_taken
-        loops += 1
-
     if CONFIG.optimizations.direct_return and is_direct_return_eligible(
         root_cte, select
     ):
@@ -127,5 +112,20 @@ def optimize_ctes(
                 root_cte.condition = select.where_clause.conditional
         root_cte.requires_nesting = False
         sort_select_output(root_cte, select)
+    if CONFIG.optimizations.datasource_inlining:
+        REGISTERED_RULES.append(InlineDatasource())
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdown())
+    if CONFIG.optimizations.constant_inlining:
+        REGISTERED_RULES.append(InlineConstant())
+    loops = 0
+    while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
+        actions_taken = False
+        for rule in REGISTERED_RULES:
+            for cte in input:
+                inverse_map = gen_inverse_map(input)
+                actions_taken = actions_taken or rule.optimize(cte, inverse_map)
+        complete = not actions_taken
+        loops += 1
 
     return filter_irrelevant_ctes(input, root_cte)

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117}/trilogy/core/optimizations/predicate_pushdown.py

@@ -2,8 +2,8 @@ from trilogy.core.models import (
     CTE,
     Conditional,
     BooleanOperator,
+    Datasource,
 )
-from trilogy.constants import logger
 from trilogy.core.optimizations.base_optimization import OptimizationRule
 
 
@@ -48,16 +48,31 @@ class PredicatePushdown(OptimizationRule):
             candidates = cte.condition.decompose()
         else:
             candidates = [cte.condition]
-        logger.info(f"Have {len(candidates)} candidates to try to push down")
+        self.log(f"Have {len(candidates)} candidates to try to push down")
         for candidate in candidates:
             conditions = {x.address for x in candidate.concept_arguments}
             for parent_cte in cte.parent_ctes:
+                if is_child_of(cte.condition, parent_cte.condition):
+                    continue
                 materialized = {k for k, v in parent_cte.source_map.items() if v != []}
+                # if it's a root datasource, we can filter on _any_ of the output concepts
+                if parent_cte.is_root_datasource:
+                    extra_check = {
+                        x.address
+                        for x in parent_cte.source.datasources[0].output_concepts
+                    }
+                    if conditions.issubset(extra_check):
+                        for x in conditions:
+                            if x not in materialized:
+                                materialized.add(x)
+                                parent_cte.source_map[x] = [
+                                    parent_cte.source.datasources[0].name
+                                ]
                 if conditions.issubset(materialized):
                     if all(
                         [
                             is_child_of(candidate, child.condition)
-                            for child in inverse_map[parent_cte.name]
+                            for child in inverse_map.get(parent_cte.name, [])
                         ]
                     ):
                         self.log(
@@ -73,14 +88,16 @@ class PredicatePushdown(OptimizationRule):
                             parent_cte.condition = candidate
                         optimized = True
                 else:
-                    logger.info("conditions not subset of parent materialized")
+                    self.log(
+                        f"conditions {conditions} not subset of parent {parent_cte.name} parent has {materialized} "
+                    )
 
         if all(
             [
                 is_child_of(cte.condition, parent_cte.condition)
                 for parent_cte in cte.parent_ctes
             ]
-        ):
+        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
             self.log("All parents have same filter, removing filter")
             cte.condition = None
             optimized = True

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117}/trilogy/dialect/base.py

@@ -46,6 +46,7 @@ from trilogy.core.models import (
     ImportStatement,
     RawSQLStatement,
     ProcessedRawSQLStatement,
+    NumericType,
 )
 from trilogy.core.query_processor import process_query, process_persist
 from trilogy.dialect.common import render_join
@@ -97,6 +98,7 @@ DATATYPE_MAP = {
     DataType.INTEGER: "int",
     DataType.FLOAT: "float",
     DataType.BOOL: "bool",
+    DataType.NUMERIC: "numeric",
 }
 
 
@@ -334,6 +336,7 @@ class BaseDialect:
             Parenthetical,
             AggregateWrapper,
             MagicConstants,
+            NumericType,
             ListType,
             ListWrapper[int],
             ListWrapper[str],
@@ -439,6 +442,8 @@ class BaseDialect:
             return str(e.value)
         elif isinstance(e, DatePart):
             return str(e.value)
+        elif isinstance(e, NumericType):
+            return f"{self.DATATYPE_MAP[DataType.NUMERIC]}({e.precision},{e.scale})"
         elif isinstance(e, MagicConstants):
             if e == MagicConstants.NULL:
                 return "null"

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117}/trilogy/dialect/presto.py

@@ -4,6 +4,7 @@ from jinja2 import Template
 
 from trilogy.core.enums import FunctionType, WindowType
 from trilogy.dialect.base import BaseDialect
+from trilogy.core.models import DataType
 
 
 WINDOW_FUNCTION_MAP: Mapping[WindowType, Callable[[Any, Any, Any], str]] = {}
@@ -26,7 +27,7 @@ FUNCTION_MAP = {
     FunctionType.WEEK: lambda x: f"EXTRACT(WEEK from {x[0]})",
     FunctionType.QUARTER: lambda x: f"EXTRACT(QUARTER from {x[0]})",
     # math
-    FunctionType.DIVIDE: lambda x: f"SAFE_DIVIDE({x[0]},{x[1]})",
+    FunctionType.DIVIDE: lambda x: f"{x[0]}/{x[1]}",
     FunctionType.DATE_ADD: lambda x: f"DATE_ADD('{x[1]}', {x[2]}, {x[0]})",
     FunctionType.CURRENT_DATE: lambda x: "CURRENT_DATE",
     FunctionType.CURRENT_DATETIME: lambda x: "CURRENT_TIMESTAMP",
@@ -80,6 +81,7 @@ class PrestoDialect(BaseDialect):
     }
     QUOTE_CHARACTER = '"'
     SQL_TEMPLATE = SQL_TEMPLATE
+    DATATYPE_MAP = {**BaseDialect.DATATYPE_MAP, DataType.NUMERIC: "DECIMAL"}
 
 
 class TrinoDialect(PrestoDialect):

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117}/trilogy/parsing/parse_engine.py

@@ -103,6 +103,7 @@ from trilogy.core.models import (
     RowsetDerivationStatement,
     LooseConceptList,
     list_to_wrapper,
+    NumericType,
 )
 from trilogy.parsing.exceptions import ParseError
 from trilogy.utility import string_to_hash
@@ -309,7 +310,9 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def struct_type(self, meta: Meta, args) -> StructType:
-        final: list[DataType | MapType | ListType | StructType | Concept] = []
+        final: list[
+            DataType | MapType | ListType | StructType | NumericType | Concept
+        ] = []
         for arg in args:
             if not isinstance(arg, (DataType, ListType, StructType)):
                 new = self.environment.concepts.__getitem__(  # type: ignore
@@ -323,12 +326,17 @@ class ParseToObjects(Transformer):
     def list_type(self, args) -> ListType:
         return ListType(type=args[0])
 
-    def data_type(self, args) -> DataType | ListType | StructType:
+    def numeric_type(self, args) -> NumericType:
+        return NumericType(precision=args[0], scale=args[1])
+
+    def data_type(self, args) -> DataType | ListType | StructType | NumericType:
         resolved = args[0]
         if isinstance(resolved, StructType):
             return resolved
         elif isinstance(resolved, ListType):
             return resolved
+        elif isinstance(resolved, NumericType):
+            return resolved
         return DataType(args[0].lower())
 
     def array_comparison(self, args) -> ComparisonOperator:
@@ -1551,6 +1559,8 @@ class ParseToObjects(Transformer):
                 DataType.STRING,
                 DataType.FLOAT,
                 DataType.NUMBER,
+                DataType.NUMERIC,
+                DataType.BOOL,
             },
             arg_count=2,
         )

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117}/trilogy/parsing/render.py

@@ -39,6 +39,7 @@ from trilogy.core.models import (
     AlignClause,
     AlignItem,
     RawSQLStatement,
+    NumericType,
 )
 from trilogy.core.enums import Modifier
 
@@ -172,6 +173,10 @@ class Renderer:
     def _(self, arg: DataType):
         return arg.value
 
+    @to_string.register
+    def _(self, arg: "NumericType"):
+        return f"""Numeric({arg.precision},{arg.scale})"""
+
     @to_string.register
     def _(self, arg: ListWrapper):
         return "[" + ", ".join([self.to_string(x) for x in arg]) + "]"

{pytrilogy-0.0.1.116 → pytrilogy-0.0.1.117}/trilogy/parsing/trilogy.lark

@@ -280,12 +280,14 @@
 
     SHORTHAND_MODIFIER: "~"
 
-    struct_type: "struct" "<" ((data_type | IDENTIFIER) ",")* (data_type | IDENTIFIER) ","? ">" 
+    struct_type: "struct"i "<" ((data_type | IDENTIFIER) ",")* (data_type | IDENTIFIER) ","? ">" 
 
-    list_type: "list" "<" data_type ">"
+    list_type: "list"i "<" data_type ">"
 
+    numeric_type: "numeric"i "(" int_lit "," int_lit ")"
 
-    !data_type: "string"i | "number"i | "numeric"i | "map"i | "list"i | "array"i | "any"i | "int"i | "bigint" | "date"i | "datetime"i | "timestamp"i | "float"i | "bool"i | struct_type | list_type
+
+    !data_type: "string"i | "number"i | "numeric"i | "map"i | "list"i | "array"i | "any"i | "int"i | "bigint"i | "date"i | "datetime"i | "timestamp"i | "float"i | "bool"i | numeric_type | struct_type | list_type
     
     PURPOSE:  "key"i | "metric"i  | CONST
     PROPERTY: "property"i