pytrilogy 0.0.1.116__py3-none-any.whl → 0.0.1.118__py3-none-any.whl

{pytrilogy-0.0.1.116.dist-info → pytrilogy-0.0.1.118.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.116
+Version: 0.0.1.118
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.116.dist-info → pytrilogy-0.0.1.118.dist-info}/RECORD

@@ -1,8 +1,8 @@
-trilogy/__init__.py,sha256=0jWazkuszqLHMaM49JtN8aEur-f9lwCZ5PqZbNhO10E,292
+trilogy/__init__.py,sha256=fYN8J2EkHXTX5cKVWpbBMQHilscbkhmZF-pMLoMqHFc,292
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=u2dNxhwy0v-6HrvG1GcpDVvuhzdTH5fuyYNCxDPlr2E,770
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
-trilogy/executor.py,sha256=auuDykCHeqlRWIHOfBfgIIIntEctWaUC-VPJr1DQbYk,10217
+trilogy/executor.py,sha256=5cRbU4Rj7p1pNV76rfp1pz704Hx_0q8_O8HFURjgXxQ,11016
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
@@ -13,17 +13,17 @@ trilogy/core/env_processor.py,sha256=SU-jpaGfoWLe9sGTeQYG1qjVnwGQ7TwctmnJRlfzluc
 trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
 trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
-trilogy/core/functions.py,sha256=zkRReytiotOBAW-a3Ri5eoejZDYTt2-7Op80ZxZxUmw,9129
+trilogy/core/functions.py,sha256=fWuuPOeWRAiukTkXvTEu_NCSbu6eUmogtLGpC4ey2FI,9465
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=_k_ZqrsZ6HYNz3CJ8yiOVXMej1pHb_QO0KZNxkDyvno,113767
-trilogy/core/optimization.py,sha256=1xAFn7aw8skqDFUQCel5xJc0hVUYHs-oW1DckN8Z4n4,4043
+trilogy/core/models.py,sha256=Mp6MB8Gl2CZcPDWkofPX8Mym1ZvT0OUS0dKAUWCj7K8,116419
+trilogy/core/optimization.py,sha256=oM3Ry7UpbpTSm2xNkmWx70OHd2V2vWRjM72sZpsZfb8,4116
 trilogy/core/query_processor.py,sha256=clIRJ6IcsqIVBPKFsxt8bqCLsLyajvAu02MUIcKQhTo,15713
 trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=neZOFjX7M2pzQ-8m-f8nApy_MfJuowX6SzcGwGFt5w4,927
 trilogy/core/optimizations/inline_datasource.py,sha256=BSp54fwF4RRwInd-09pggemC7JuXj-uqGzi32ufeqYo,2171
-trilogy/core/optimizations/predicate_pushdown.py,sha256=s5BMLADcQy4_UsMCchHTsInaJUtLGh1l2kTpqzAzcs4,3199
+trilogy/core/optimizations/predicate_pushdown.py,sha256=sIojWvoYp6k_ANCyVqxCpEyLY_GLmzsG-Sghj0cbk3k,4135
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/concept_strategies_v3.py,sha256=MYrpNMidqvPOg123RekOcqVTjcj03i_538gBo0MzoWE,23432
 trilogy/core/processing/graph_utils.py,sha256=ulCJ4hYAISbUxLD6VM2fah9RBPGIXSEHEPeRBSFl0Rs,1197
@@ -50,14 +50,14 @@ trilogy/core/processing/nodes/select_node_v2.py,sha256=ERCflBFzKpD5SzweMevnJLyQn
 trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=ro9fKV6RrPggUVmJPlIEdySLsR3Pq5rm0UMQho6Bx_k,31518
+trilogy/dialect/base.py,sha256=ycsxbUL68DsodOsHEgEoNLKMn5vgRN3sDIRfiH9fQDs,31719
 trilogy/dialect/bigquery.py,sha256=9vxQn2BMv_oTGQSWQpoN5ho_OgqMWaHH9e-5vQVf44c,2906
 trilogy/dialect/common.py,sha256=zWrYmvevlXznocw9uGHmY5Ws1rp_kICm9zA_ulTe4eg,2165
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
 trilogy/dialect/duckdb.py,sha256=Ddyt68sr8IL2HnZMenyytoD65FXwY_O2pz1McyS0bis,3075
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
 trilogy/dialect/postgres.py,sha256=r47xbCA7nfEYENofiVfLZ-SnReNfDmUmW4OSHVkkP4E,3206
-trilogy/dialect/presto.py,sha256=8zjRn8AeYXZQGuUi-afyBWLet8o-LSt6gm5IH7bTdiw,2987
+trilogy/dialect/presto.py,sha256=UxBodRiV3szpFcQlcjoJaGXEwAhZJf_OT7dHczYvO80,3092
 trilogy/dialect/snowflake.py,sha256=N3HknYgN-fjD7BLX1Ucj-ss_ku2Ox8DgLsF3BIHutHo,2941
 trilogy/dialect/sql_server.py,sha256=HX68vNTrcDaTnOxe6Zbx_PBgrO42e2VuThxO6CYQ2cY,3026
 trilogy/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -70,14 +70,14 @@ trilogy/parsing/common.py,sha256=iR3fiiZ7w8VJuUGrQ0v06XGDXov81f4z1ZlFnj6y40E,580
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=cT1IoLP0LzykNX0UfH9-eu8PzjfzSNCtZ_hZvf58TRg,56761
-trilogy/parsing/render.py,sha256=cOKHvl2ZrpkD4lpng9iXc37gc3XyZ1e51woPxB7QqsI,11672
-trilogy/parsing/trilogy.lark,sha256=xYpVKneNhXruxIyzFqcK1b8-NSqzzEwtPrTxmpXd_Vc,10786
+trilogy/parsing/parse_engine.py,sha256=F1ok96qT6EhKRKV1Q_YzfHxMFtNV8qAXopK8NaePgU4,57080
+trilogy/parsing/render.py,sha256=TnLf5fg4wimpd9EvhLU-FMDwpyW9pesoedBZ0RrmWD4,11810
+trilogy/parsing/trilogy.lark,sha256=1AZbQGpNmpm4KamAXA5IWcuOr2B8Gb8kUJcAOmKf_zY,10862
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.1.116.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.1.116.dist-info/METADATA,sha256=kyUsh-1RPnsedKLexiwJCKTl9ZzZHl5Uu4LUQcV9hj0,7878
-pytrilogy-0.0.1.116.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-pytrilogy-0.0.1.116.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.1.116.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.1.116.dist-info/RECORD,,
+pytrilogy-0.0.1.118.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.1.118.dist-info/METADATA,sha256=tZ1kQcOTJ5JTYb5tqlfdo7g-o94TC0tKmcdqLtJ35bI,7878
+pytrilogy-0.0.1.118.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+pytrilogy-0.0.1.118.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.1.118.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.1.118.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.1.116"
+__version__ = "0.0.1.118"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/functions.py

@@ -9,6 +9,7 @@ from trilogy.core.models import (
     ListType,
     StructType,
     MapType,
+    NumericType,
 )
 from trilogy.core.enums import FunctionType, Purpose, Granularity, DatePart
 from trilogy.core.exceptions import InvalidSyntaxException
@@ -21,7 +22,9 @@ def create_function_derived_concept(
     namespace: str,
     operator: FunctionType,
     arguments: list[Concept],
-    output_type: Optional[DataType | ListType | StructType | MapType] = None,
+    output_type: Optional[
+        DataType | ListType | StructType | MapType | NumericType
+    ] = None,
     output_purpose: Optional[Purpose] = None,
 ) -> Concept:
     purpose = (
@@ -49,16 +52,24 @@ def argument_to_purpose(arg) -> Purpose:
     if isinstance(arg, Function):
         return arg.output_purpose
     elif isinstance(arg, AggregateWrapper):
+        base = arg.function.output_purpose
+        if arg.by and base == Purpose.METRIC:
+            return Purpose.PROPERTY
         return arg.function.output_purpose
     elif isinstance(arg, Parenthetical):
         return argument_to_purpose(arg.content)
     elif isinstance(arg, WindowItem):
         return Purpose.PROPERTY
     elif isinstance(arg, Concept):
+        base = arg.purpose
+        if (
+            isinstance(arg.lineage, AggregateWrapper)
+            and arg.lineage.by
+            and base == Purpose.METRIC
+        ):
+            return Purpose.PROPERTY
         return arg.purpose
-    elif isinstance(arg, (int, float, str, bool, list)):
-        return Purpose.CONSTANT
-    elif isinstance(arg, DataType):
+    elif isinstance(arg, (int, float, str, bool, list, NumericType, DataType)):
         return Purpose.CONSTANT
     elif isinstance(arg, DatePart):
         return Purpose.CONSTANT

trilogy/core/models.py

@@ -105,7 +105,9 @@ def get_concept_arguments(expr) -> List["Concept"]:
     return output
 
 
-ALL_TYPES = Union["DataType", "MapType", "ListType", "StructType", "Concept"]
+ALL_TYPES = Union[
+    "DataType", "MapType", "ListType", "NumericType", "StructType", "Concept"
+]
 
 NAMESPACED_TYPES = Union[
     "WindowItem",
@@ -147,6 +149,11 @@ class SelectGrain(ABC):
         raise NotImplementedError
 
 
+class ConstantInlineable(ABC):
+    def inline_concept(self, concept: Concept):
+        raise NotImplementedError
+
+
 class DataType(Enum):
     # PRIMITIVES
     STRING = "string"
@@ -176,6 +183,19 @@ class DataType(Enum):
         return self
 
 
+class NumericType(BaseModel):
+    precision: int = 20
+    scale: int = 5
+
+    @property
+    def data_type(self):
+        return DataType.NUMERIC
+
+    @property
+    def value(self):
+        return self.data_type.value
+
+
 class ListType(BaseModel):
     model_config = ConfigDict(frozen=True)
     type: ALL_TYPES
@@ -192,7 +212,9 @@ class ListType(BaseModel):
         return self.data_type.value
 
     @property
-    def value_data_type(self) -> DataType | StructType | MapType | ListType:
+    def value_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
         if isinstance(self.type, Concept):
             return self.type.datatype
         return self.type
@@ -270,7 +292,7 @@ def empty_grain() -> Grain:
 
 class Concept(Namespaced, SelectGrain, BaseModel):
     name: str
-    datatype: DataType | ListType | StructType | MapType
+    datatype: DataType | ListType | StructType | MapType | NumericType
     purpose: Purpose
     metadata: Optional[Metadata] = Field(
         default_factory=lambda: Metadata(description=None, line_number=None),
@@ -790,12 +812,12 @@ class LooseConceptList(BaseModel):
 class Function(Namespaced, SelectGrain, BaseModel):
     operator: FunctionType
     arg_count: int = Field(default=1)
-    output_datatype: DataType | ListType | StructType | MapType
+    output_datatype: DataType | ListType | StructType | MapType | NumericType
     output_purpose: Purpose
     valid_inputs: Optional[
         Union[
-            Set[DataType | ListType | StructType],
-            List[Set[DataType | ListType | StructType]],
+            Set[DataType | ListType | StructType | NumericType],
+            List[Set[DataType | ListType | StructType] | NumericType],
         ]
     ] = None
     arguments: Sequence[
@@ -808,6 +830,7 @@ class Function(Namespaced, SelectGrain, BaseModel):
             str,
             DataType,
             ListType,
+            NumericType,
             DatePart,
             "Parenthetical",
             CaseWhen,
@@ -1499,7 +1522,7 @@ class DatasourceMetadata(BaseModel):
 
 class MergeStatement(Namespaced, BaseModel):
     concepts: List[Concept]
-    datatype: DataType | ListType | StructType | MapType
+    datatype: DataType | ListType | StructType | MapType | NumericType
 
     @cached_property
     def concepts_lcl(self):
@@ -2057,6 +2080,9 @@ class CTE(BaseModel):
         if concept.address in self.source_map:
             removed = removed.union(self.source_map[concept.address])
             del self.source_map[concept.address]
+
+        if self.condition:
+            self.condition = self.condition.inline_constant(concept)
         # if we've entirely removed the need to join to someplace to get the concept
         # drop the join as well.
         for removed_cte in removed:
@@ -2745,7 +2771,7 @@ class LazyEnvironment(Environment):
         return super().__getattribute__(name)
 
 
-class Comparison(ConceptArgs, Namespaced, SelectGrain, BaseModel):
+class Comparison(ConceptArgs, Namespaced, ConstantInlineable, SelectGrain, BaseModel):
     left: Union[
         int,
         str,
@@ -2807,6 +2833,32 @@ class Comparison(ConceptArgs, Namespaced, SelectGrain, BaseModel):
     def __str__(self):
         return self.__repr__()
 
+    def inline_constant(self, constant: Concept) -> "Comparison":
+        assert isinstance(constant.lineage, Function)
+        new_val = constant.lineage.arguments[0]
+        if isinstance(self.left, ConstantInlineable):
+            new_left = self.left.inline_constant(constant)
+        elif self.left == constant:
+            new_left = new_val
+        else:
+            new_left = self.left
+
+        if isinstance(self.right, ConstantInlineable):
+            new_right = self.right.inline_constant(constant)
+        elif self.right == constant:
+            new_right = new_val
+        else:
+            new_right = self.right
+
+        if self.right == constant:
+            new_right = new_val
+
+        return Comparison(
+            left=new_left,
+            right=new_right,
+            operator=self.operator,
+        )
+
     def with_namespace(self, namespace: str):
         return self.__class__(
             left=(
@@ -2983,7 +3035,7 @@ class CaseElse(Namespaced, SelectGrain, BaseModel):
         )
 
 
-class Conditional(ConceptArgs, Namespaced, SelectGrain, BaseModel):
+class Conditional(ConceptArgs, Namespaced, ConstantInlineable, SelectGrain, BaseModel):
     left: Union[
         int,
         str,
@@ -3029,6 +3081,32 @@ class Conditional(ConceptArgs, Namespaced, SelectGrain, BaseModel):
     def __repr__(self):
         return f"{str(self.left)} {self.operator.value} {str(self.right)}"
 
+    def inline_constant(self, constant: Concept) -> "Conditional":
+        assert isinstance(constant.lineage, Function)
+        new_val = constant.lineage.arguments[0]
+        if isinstance(self.left, ConstantInlineable):
+            new_left = self.left.inline_constant(constant)
+        elif self.left == constant:
+            new_left = new_val
+        else:
+            new_left = self.left
+
+        if isinstance(self.right, ConstantInlineable):
+            new_right = self.right.inline_constant(constant)
+        elif self.right == constant:
+            new_right = new_val
+        else:
+            new_right = self.right
+
+        if self.right == constant:
+            new_right = new_val
+
+        return Conditional(
+            left=new_left,
+            right=new_right,
+            operator=self.operator,
+        )
+
     def with_namespace(self, namespace: str):
         return Conditional(
             left=(
@@ -3368,7 +3446,9 @@ class RowsetItem(Namespaced, BaseModel):
         return [self.content]
 
 
-class Parenthetical(ConceptArgs, Namespaced, SelectGrain, BaseModel):
+class Parenthetical(
+    ConceptArgs, Namespaced, ConstantInlineable, SelectGrain, BaseModel
+):
     content: "Expr"
 
     def __str__(self):
@@ -3402,6 +3482,15 @@ class Parenthetical(ConceptArgs, Namespaced, SelectGrain, BaseModel):
             )
         )
 
+    def inline_constant(self, concept: Concept):
+        return Parenthetical(
+            content=(
+                self.content.inline_constant(concept)
+                if isinstance(self.content, ConstantInlineable)
+                else self.content
+            )
+        )
+
     @property
     def concept_arguments(self) -> List[Concept]:
         base: List[Concept] = []
@@ -3498,7 +3587,7 @@ def list_to_wrapper(args):
     return ListWrapper(args, type=types[0])
 
 
-def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType:
+def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | NumericType:
     if isinstance(arg, Function):
         return arg.output_datatype
     elif isinstance(arg, Concept):
@@ -3511,6 +3600,8 @@ def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType:
         return DataType.STRING
     elif isinstance(arg, float):
         return DataType.FLOAT
+    elif isinstance(arg, NumericType):
+        return arg
     elif isinstance(arg, ListWrapper):
         return ListType(type=arg.type)
     elif isinstance(arg, AggregateWrapper):

trilogy/core/optimization.py

@@ -50,7 +50,9 @@ def is_direct_return_eligible(
     if isinstance(select, (PersistStatement, MultiSelectStatement)):
         return False
     derived_concepts = [
-        c for c in cte.source.output_concepts if c not in cte.source.input_concepts
+        c
+        for c in cte.source.output_concepts + cte.source.hidden_concepts
+        if c not in cte.source.input_concepts
     ]
     eligible = True
     conditions = (
@@ -90,26 +92,9 @@ def sort_select_output(cte: CTE, query: SelectStatement | MultiSelectStatement):
 
 def optimize_ctes(
     input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
-):
+) -> list[CTE]:
     complete = False
     REGISTERED_RULES: list["OptimizationRule"] = []
-
-    if CONFIG.optimizations.datasource_inlining:
-        REGISTERED_RULES.append(InlineDatasource())
-    if CONFIG.optimizations.predicate_pushdown:
-        REGISTERED_RULES.append(PredicatePushdown())
-    if CONFIG.optimizations.constant_inlining:
-        REGISTERED_RULES.append(InlineConstant())
-    loops = 0
-    while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
-        actions_taken = False
-        for rule in REGISTERED_RULES:
-            for cte in input:
-                inverse_map = gen_inverse_map(input)
-                actions_taken = rule.optimize(cte, inverse_map)
-        complete = not actions_taken
-        loops += 1
-
     if CONFIG.optimizations.direct_return and is_direct_return_eligible(
         root_cte, select
     ):
@@ -127,5 +112,20 @@ def optimize_ctes(
                 root_cte.condition = select.where_clause.conditional
         root_cte.requires_nesting = False
         sort_select_output(root_cte, select)
+    if CONFIG.optimizations.datasource_inlining:
+        REGISTERED_RULES.append(InlineDatasource())
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdown())
+    if CONFIG.optimizations.constant_inlining:
+        REGISTERED_RULES.append(InlineConstant())
+    loops = 0
+    while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
+        actions_taken = False
+        for rule in REGISTERED_RULES:
+            for cte in input:
+                inverse_map = gen_inverse_map(input)
+                actions_taken = actions_taken or rule.optimize(cte, inverse_map)
+        complete = not actions_taken
+        loops += 1
 
     return filter_irrelevant_ctes(input, root_cte)

trilogy/core/optimizations/predicate_pushdown.py

@@ -2,8 +2,8 @@ from trilogy.core.models import (
     CTE,
     Conditional,
     BooleanOperator,
+    Datasource,
 )
-from trilogy.constants import logger
 from trilogy.core.optimizations.base_optimization import OptimizationRule
 
 
@@ -48,16 +48,31 @@ class PredicatePushdown(OptimizationRule):
             candidates = cte.condition.decompose()
         else:
             candidates = [cte.condition]
-        logger.info(f"Have {len(candidates)} candidates to try to push down")
+        self.log(f"Have {len(candidates)} candidates to try to push down")
         for candidate in candidates:
             conditions = {x.address for x in candidate.concept_arguments}
             for parent_cte in cte.parent_ctes:
+                if is_child_of(cte.condition, parent_cte.condition):
+                    continue
                 materialized = {k for k, v in parent_cte.source_map.items() if v != []}
+                # if it's a root datasource, we can filter on _any_ of the output concepts
+                if parent_cte.is_root_datasource:
+                    extra_check = {
+                        x.address
+                        for x in parent_cte.source.datasources[0].output_concepts
+                    }
+                    if conditions.issubset(extra_check):
+                        for x in conditions:
+                            if x not in materialized:
+                                materialized.add(x)
+                                parent_cte.source_map[x] = [
+                                    parent_cte.source.datasources[0].name
+                                ]
                 if conditions.issubset(materialized):
                     if all(
                         [
                             is_child_of(candidate, child.condition)
-                            for child in inverse_map[parent_cte.name]
+                            for child in inverse_map.get(parent_cte.name, [])
                         ]
                     ):
                         self.log(
@@ -73,14 +88,16 @@ class PredicatePushdown(OptimizationRule):
                             parent_cte.condition = candidate
                         optimized = True
                 else:
-                    logger.info("conditions not subset of parent materialized")
+                    self.log(
+                        f"conditions {conditions} not subset of parent {parent_cte.name} parent has {materialized} "
+                    )
 
         if all(
             [
                 is_child_of(cte.condition, parent_cte.condition)
                 for parent_cte in cte.parent_ctes
             ]
-        ):
+        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
             self.log("All parents have same filter, removing filter")
             cte.condition = None
             optimized = True

trilogy/dialect/base.py

@@ -46,6 +46,7 @@ from trilogy.core.models import (
     ImportStatement,
     RawSQLStatement,
     ProcessedRawSQLStatement,
+    NumericType,
 )
 from trilogy.core.query_processor import process_query, process_persist
 from trilogy.dialect.common import render_join
@@ -97,6 +98,7 @@ DATATYPE_MAP = {
     DataType.INTEGER: "int",
     DataType.FLOAT: "float",
     DataType.BOOL: "bool",
+    DataType.NUMERIC: "numeric",
 }
 
 
@@ -334,6 +336,7 @@ class BaseDialect:
             Parenthetical,
             AggregateWrapper,
             MagicConstants,
+            NumericType,
             ListType,
             ListWrapper[int],
             ListWrapper[str],
@@ -439,6 +442,8 @@ class BaseDialect:
             return str(e.value)
         elif isinstance(e, DatePart):
             return str(e.value)
+        elif isinstance(e, NumericType):
+            return f"{self.DATATYPE_MAP[DataType.NUMERIC]}({e.precision},{e.scale})"
         elif isinstance(e, MagicConstants):
             if e == MagicConstants.NULL:
                 return "null"

trilogy/dialect/presto.py

@@ -4,6 +4,7 @@ from jinja2 import Template
 
 from trilogy.core.enums import FunctionType, WindowType
 from trilogy.dialect.base import BaseDialect
+from trilogy.core.models import DataType
 
 
 WINDOW_FUNCTION_MAP: Mapping[WindowType, Callable[[Any, Any, Any], str]] = {}
@@ -26,7 +27,7 @@ FUNCTION_MAP = {
     FunctionType.WEEK: lambda x: f"EXTRACT(WEEK from {x[0]})",
     FunctionType.QUARTER: lambda x: f"EXTRACT(QUARTER from {x[0]})",
     # math
-    FunctionType.DIVIDE: lambda x: f"SAFE_DIVIDE({x[0]},{x[1]})",
+    FunctionType.DIVIDE: lambda x: f"{x[0]}/{x[1]}",
     FunctionType.DATE_ADD: lambda x: f"DATE_ADD('{x[1]}', {x[2]}, {x[0]})",
     FunctionType.CURRENT_DATE: lambda x: "CURRENT_DATE",
     FunctionType.CURRENT_DATETIME: lambda x: "CURRENT_TIMESTAMP",
@@ -80,6 +81,7 @@ class PrestoDialect(BaseDialect):
     }
     QUOTE_CHARACTER = '"'
     SQL_TEMPLATE = SQL_TEMPLATE
+    DATATYPE_MAP = {**BaseDialect.DATATYPE_MAP, DataType.NUMERIC: "DECIMAL"}
 
 
 class TrinoDialect(PrestoDialect):

trilogy/executor.py

@@ -16,6 +16,8 @@ from trilogy.core.models import (
     PersistStatement,
     ShowStatement,
     Concept,
+    ConceptDeclarationStatement,
+    Datasource,
 )
 from trilogy.dialect.base import BaseDialect
 from trilogy.dialect.enums import Dialects
@@ -100,6 +102,33 @@ class Executor(object):
     def execute_query(self, query) -> CursorResult:
         raise NotImplementedError("Cannot execute type {}".format(type(query)))
 
+    @execute_query.register
+    def _(self, query: ConceptDeclarationStatement) -> CursorResult:
+        concept = query.concept
+        return MockResult(
+            [
+                {
+                    "address": concept.address,
+                    "type": concept.datatype.value,
+                    "purpose": concept.purpose.value,
+                    "derivation": concept.derivation.value,
+                }
+            ],
+            ["address", "type", "purpose", "derivation"],
+        )
+
+    @execute_query.register
+    def _(self, query: Datasource) -> CursorResult:
+
+        return MockResult(
+            [
+                {
+                    "name": query.name,
+                }
+            ],
+            ["name"],
+        )
+
     @execute_query.register
     def _(self, query: SelectStatement) -> CursorResult:
         sql = self.generator.generate_queries(

trilogy/parsing/parse_engine.py

@@ -103,6 +103,7 @@ from trilogy.core.models import (
     RowsetDerivationStatement,
     LooseConceptList,
     list_to_wrapper,
+    NumericType,
 )
 from trilogy.parsing.exceptions import ParseError
 from trilogy.utility import string_to_hash
@@ -309,7 +310,9 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def struct_type(self, meta: Meta, args) -> StructType:
-        final: list[DataType | MapType | ListType | StructType | Concept] = []
+        final: list[
+            DataType | MapType | ListType | StructType | NumericType | Concept
+        ] = []
         for arg in args:
             if not isinstance(arg, (DataType, ListType, StructType)):
                 new = self.environment.concepts.__getitem__(  # type: ignore
@@ -323,12 +326,17 @@ class ParseToObjects(Transformer):
     def list_type(self, args) -> ListType:
         return ListType(type=args[0])
 
-    def data_type(self, args) -> DataType | ListType | StructType:
+    def numeric_type(self, args) -> NumericType:
+        return NumericType(precision=args[0], scale=args[1])
+
+    def data_type(self, args) -> DataType | ListType | StructType | NumericType:
         resolved = args[0]
         if isinstance(resolved, StructType):
             return resolved
         elif isinstance(resolved, ListType):
             return resolved
+        elif isinstance(resolved, NumericType):
+            return resolved
         return DataType(args[0].lower())
 
     def array_comparison(self, args) -> ComparisonOperator:
@@ -1551,6 +1559,8 @@ class ParseToObjects(Transformer):
                 DataType.STRING,
                 DataType.FLOAT,
                 DataType.NUMBER,
+                DataType.NUMERIC,
+                DataType.BOOL,
             },
             arg_count=2,
         )

trilogy/parsing/render.py

@@ -39,6 +39,7 @@ from trilogy.core.models import (
     AlignClause,
     AlignItem,
     RawSQLStatement,
+    NumericType,
 )
 from trilogy.core.enums import Modifier
 
@@ -172,6 +173,10 @@ class Renderer:
     def _(self, arg: DataType):
         return arg.value
 
+    @to_string.register
+    def _(self, arg: "NumericType"):
+        return f"""Numeric({arg.precision},{arg.scale})"""
+
     @to_string.register
     def _(self, arg: ListWrapper):
         return "[" + ", ".join([self.to_string(x) for x in arg]) + "]"

trilogy/parsing/trilogy.lark

@@ -280,12 +280,14 @@
 
     SHORTHAND_MODIFIER: "~"
 
-    struct_type: "struct" "<" ((data_type | IDENTIFIER) ",")* (data_type | IDENTIFIER) ","? ">" 
+    struct_type: "struct"i "<" ((data_type | IDENTIFIER) ",")* (data_type | IDENTIFIER) ","? ">" 
 
-    list_type: "list" "<" data_type ">"
+    list_type: "list"i "<" data_type ">"
 
+    numeric_type: "numeric"i "(" int_lit "," int_lit ")"
 
-    !data_type: "string"i | "number"i | "numeric"i | "map"i | "list"i | "array"i | "any"i | "int"i | "bigint" | "date"i | "datetime"i | "timestamp"i | "float"i | "bool"i | struct_type | list_type
+
+    !data_type: "string"i | "number"i | "numeric"i | "map"i | "list"i | "array"i | "any"i | "int"i | "bigint"i | "date"i | "datetime"i | "timestamp"i | "float"i | "bool"i | numeric_type | struct_type | list_type
     
     PURPOSE:  "key"i | "metric"i  | CONST
     PROPERTY: "property"i