pytrilogy 0.0.1.115__tar.gz → 0.0.1.117__tar.gz

{pytrilogy-0.0.1.115/pytrilogy.egg-info → pytrilogy-0.0.1.117}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.115
+Version: 0.0.1.117
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.115 → pytrilogy-0.0.1.117/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.115
+Version: 0.0.1.117
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.115 → pytrilogy-0.0.1.117}/pytrilogy.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ pytrilogy.egg-info/dependency_links.txt
 pytrilogy.egg-info/entry_points.txt
 pytrilogy.egg-info/requires.txt
 pytrilogy.egg-info/top_level.txt
+tests/test_datatypes.py
 tests/test_declarations.py
 tests/test_derived_concepts.py
 tests/test_discovery_nodes.py
@@ -45,6 +46,11 @@ trilogy/core/internal.py
 trilogy/core/models.py
 trilogy/core/optimization.py
 trilogy/core/query_processor.py
+trilogy/core/optimizations/__init__.py
+trilogy/core/optimizations/base_optimization.py
+trilogy/core/optimizations/inline_constant.py
+trilogy/core/optimizations/inline_datasource.py
+trilogy/core/optimizations/predicate_pushdown.py
 trilogy/core/processing/__init__.py
 trilogy/core/processing/concept_strategies_v3.py
 trilogy/core/processing/graph_utils.py

pytrilogy-0.0.1.117/tests/test_datatypes.py

@@ -0,0 +1,13 @@
+from trilogy.core.models import (
+    NumericType,
+)
+from trilogy.parsing.parse_engine import (
+    parse_text,
+)
+
+
+def test_numeric():
+    env, _ = parse_text(
+        "const order_id numeric(12,2); const rounded <- cast(order_id as numeric(15,2));"
+    )
+    assert env.concepts["order_id"].datatype == NumericType(precision=12, scale=2)

{pytrilogy-0.0.1.115 → pytrilogy-0.0.1.117}/tests/test_parsing.py

@@ -292,3 +292,15 @@ CASE WHEN dates.year BETWEEN 1883 AND 1900 THEN 'Lost Generation'
     )
 
     assert env2.concepts["dates.generation"].purpose == Purpose.PROPERTY
+
+
+def test_rawsql():
+    env, parsed = parse_text(
+        """
+raw_sql('''select 1''');
+
+select 1 as test;
+
+"""
+    )
+    assert parsed[0].text == "select 1"

{pytrilogy-0.0.1.115 → pytrilogy-0.0.1.117}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.1.115"
+__version__ = "0.0.1.117"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.1.115 → pytrilogy-0.0.1.117}/trilogy/constants.py

@@ -22,6 +22,7 @@ NULL_VALUE = MagicConstants.NULL
 class Optimizations:
     predicate_pushdown: bool = True
     datasource_inlining: bool = True
+    constant_inlining: bool = True
     direct_return: bool = True
 
 

{pytrilogy-0.0.1.115 → pytrilogy-0.0.1.117}/trilogy/core/functions.py

@@ -9,6 +9,7 @@ from trilogy.core.models import (
     ListType,
     StructType,
     MapType,
+    NumericType,
 )
 from trilogy.core.enums import FunctionType, Purpose, Granularity, DatePart
 from trilogy.core.exceptions import InvalidSyntaxException
@@ -21,7 +22,9 @@ def create_function_derived_concept(
     namespace: str,
     operator: FunctionType,
     arguments: list[Concept],
-    output_type: Optional[DataType | ListType | StructType | MapType] = None,
+    output_type: Optional[
+        DataType | ListType | StructType | MapType | NumericType
+    ] = None,
     output_purpose: Optional[Purpose] = None,
 ) -> Concept:
     purpose = (
@@ -56,9 +59,7 @@ def argument_to_purpose(arg) -> Purpose:
         return Purpose.PROPERTY
     elif isinstance(arg, Concept):
         return arg.purpose
-    elif isinstance(arg, (int, float, str, bool, list)):
-        return Purpose.CONSTANT
-    elif isinstance(arg, DataType):
+    elif isinstance(arg, (int, float, str, bool, list, NumericType, DataType)):
         return Purpose.CONSTANT
     elif isinstance(arg, DatePart):
         return Purpose.CONSTANT

{pytrilogy-0.0.1.115 → pytrilogy-0.0.1.117}/trilogy/core/models.py

@@ -105,7 +105,9 @@ def get_concept_arguments(expr) -> List["Concept"]:
     return output
 
 
-ALL_TYPES = Union["DataType", "MapType", "ListType", "StructType", "Concept"]
+ALL_TYPES = Union[
+    "DataType", "MapType", "ListType", "NumericType", "StructType", "Concept"
+]
 
 NAMESPACED_TYPES = Union[
     "WindowItem",
@@ -176,6 +178,19 @@ class DataType(Enum):
         return self
 
 
+class NumericType(BaseModel):
+    precision: int = 20
+    scale: int = 5
+
+    @property
+    def data_type(self):
+        return DataType.NUMERIC
+
+    @property
+    def value(self):
+        return self.data_type.value
+
+
 class ListType(BaseModel):
     model_config = ConfigDict(frozen=True)
     type: ALL_TYPES
@@ -192,7 +207,9 @@ class ListType(BaseModel):
         return self.data_type.value
 
     @property
-    def value_data_type(self) -> DataType | StructType | MapType | ListType:
+    def value_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
         if isinstance(self.type, Concept):
             return self.type.datatype
         return self.type
@@ -270,7 +287,7 @@ def empty_grain() -> Grain:
 
 class Concept(Namespaced, SelectGrain, BaseModel):
     name: str
-    datatype: DataType | ListType | StructType | MapType
+    datatype: DataType | ListType | StructType | MapType | NumericType
     purpose: Purpose
     metadata: Optional[Metadata] = Field(
         default_factory=lambda: Metadata(description=None, line_number=None),
@@ -790,12 +807,12 @@ class LooseConceptList(BaseModel):
 class Function(Namespaced, SelectGrain, BaseModel):
     operator: FunctionType
     arg_count: int = Field(default=1)
-    output_datatype: DataType | ListType | StructType | MapType
+    output_datatype: DataType | ListType | StructType | MapType | NumericType
     output_purpose: Purpose
     valid_inputs: Optional[
         Union[
-            Set[DataType | ListType | StructType],
-            List[Set[DataType | ListType | StructType]],
+            Set[DataType | ListType | StructType | NumericType],
+            List[Set[DataType | ListType | StructType] | NumericType],
         ]
     ] = None
     arguments: Sequence[
@@ -808,6 +825,7 @@ class Function(Namespaced, SelectGrain, BaseModel):
             str,
             DataType,
             ListType,
+            NumericType,
             DatePart,
             "Parenthetical",
             CaseWhen,
@@ -1165,11 +1183,17 @@ class OrderBy(Namespaced, BaseModel):
         return OrderBy(items=[x.with_namespace(namespace) for x in self.items])
 
 
+class RawSQLStatement(BaseModel):
+    text: str
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
+
+
 class SelectStatement(Namespaced, BaseModel):
     selection: List[SelectItem]
     where_clause: Optional["WhereClause"] = None
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     def __str__(self):
         from trilogy.parsing.render import render_query
@@ -1371,6 +1395,7 @@ class MultiSelectStatement(Namespaced, BaseModel):
     where_clause: Optional["WhereClause"] = None
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     def __repr__(self):
         return "MultiSelect<" + " MERGE ".join([str(s) for s in self.selects]) + ">"
@@ -1492,7 +1517,7 @@ class DatasourceMetadata(BaseModel):
 
 class MergeStatement(Namespaced, BaseModel):
     concepts: List[Concept]
-    datatype: DataType | ListType | StructType | MapType
+    datatype: DataType | ListType | StructType | MapType | NumericType
 
     @cached_property
     def concepts_lcl(self):
@@ -2038,6 +2063,40 @@ class CTE(BaseModel):
     def validate_output_columns(cls, v):
         return unique(v, "address")
 
+    def inline_constant(self, concept: Concept):
+        if not concept.derivation == PurposeLineage.CONSTANT:
+            return False
+        if not isinstance(concept.lineage, Function):
+            return False
+        if not concept.lineage.operator == FunctionType.CONSTANT:
+            return False
+        # remove the constant
+        removed: set = set()
+        if concept.address in self.source_map:
+            removed = removed.union(self.source_map[concept.address])
+            del self.source_map[concept.address]
+        # if we've entirely removed the need to join to someplace to get the concept
+        # drop the join as well.
+        for removed_cte in removed:
+            still_required = any([removed_cte in x for x in self.source_map.values()])
+            if not still_required:
+                self.joins = [
+                    join
+                    for join in self.joins
+                    if not isinstance(join, Join)
+                    or (
+                        join.right_cte.name != removed_cte
+                        and join.left_cte.name != removed_cte
+                    )
+                ]
+                self.parent_ctes = [
+                    x for x in self.parent_ctes if x.name != removed_cte
+                ]
+                if removed_cte == self.base_name_override:
+                    candidates = [x.name for x in self.parent_ctes]
+                    self.base_name_override = candidates[0] if candidates else None
+                    self.base_alias_override = candidates[0] if candidates else None
+
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
         qds_being_inlined = parent.source
         ds_being_inlined = qds_being_inlined.datasources[0]
@@ -2186,6 +2245,11 @@ class CTE(BaseModel):
             and not self.group_to_grain
         ):
             return False
+        # if we don't need to source any concepts from anywhere
+        # render without from
+        # most likely to happen from inlining constants
+        if not any([v for v in self.source_map.values()]):
+            return False
         if (
             len(self.source.datasources) == 1
             and self.source.datasources[0].name == CONSTANT_DATASET
@@ -3184,6 +3248,10 @@ class ProcessedShowStatement(BaseModel):
     output_values: List[Union[Concept, Datasource, ProcessedQuery]]
 
 
+class ProcessedRawSQLStatement(BaseModel):
+    text: str
+
+
 class Limit(BaseModel):
     count: int
 
@@ -3386,6 +3454,7 @@ class Parenthetical(ConceptArgs, Namespaced, SelectGrain, BaseModel):
 class PersistStatement(BaseModel):
     datasource: Datasource
     select: SelectStatement
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     @property
     def identifier(self):
@@ -3447,7 +3516,7 @@ def list_to_wrapper(args):
     return ListWrapper(args, type=types[0])
 
 
-def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType:
+def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | NumericType:
     if isinstance(arg, Function):
         return arg.output_datatype
     elif isinstance(arg, Concept):
@@ -3460,6 +3529,8 @@ def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType:
         return DataType.STRING
     elif isinstance(arg, float):
         return DataType.FLOAT
+    elif isinstance(arg, NumericType):
+        return arg
     elif isinstance(arg, ListWrapper):
         return ListType(type=arg.type)
     elif isinstance(arg, AggregateWrapper):

pytrilogy-0.0.1.117/trilogy/core/optimization.py

@@ -0,0 +1,131 @@
+from trilogy.core.models import (
+    CTE,
+    SelectStatement,
+    PersistStatement,
+    MultiSelectStatement,
+    Conditional,
+    BooleanOperator,
+)
+from trilogy.core.enums import PurposeLineage
+from trilogy.constants import logger, CONFIG
+from trilogy.core.optimizations import (
+    OptimizationRule,
+    InlineConstant,
+    PredicatePushdown,
+    InlineDatasource,
+)
+
+
+MAX_OPTIMIZATION_LOOPS = 100
+
+
+def filter_irrelevant_ctes(
+    input: list[CTE],
+    root_cte: CTE,
+):
+    relevant_ctes = set()
+
+    def recurse(cte: CTE):
+        relevant_ctes.add(cte.name)
+        for cte in cte.parent_ctes:
+            recurse(cte)
+
+    recurse(root_cte)
+    return [cte for cte in input if cte.name in relevant_ctes]
+
+
+def gen_inverse_map(input: list[CTE]) -> dict[str, list[CTE]]:
+    inverse_map: dict[str, list[CTE]] = {}
+    for cte in input:
+        for parent in cte.parent_ctes:
+            if parent.name not in inverse_map:
+                inverse_map[parent.name] = []
+            inverse_map[parent.name].append(cte)
+    return inverse_map
+
+
+def is_direct_return_eligible(
+    cte: CTE, select: SelectStatement | PersistStatement | MultiSelectStatement
+) -> bool:
+    if isinstance(select, (PersistStatement, MultiSelectStatement)):
+        return False
+    derived_concepts = [
+        c
+        for c in cte.source.output_concepts + cte.source.hidden_concepts
+        if c not in cte.source.input_concepts
+    ]
+    eligible = True
+    conditions = (
+        set(x.address for x in select.where_clause.concept_arguments)
+        if select.where_clause
+        else set()
+    )
+    if conditions and select.limit:
+        return False
+    for x in derived_concepts:
+        if x.derivation == PurposeLineage.WINDOW:
+            return False
+        if x.derivation == PurposeLineage.UNNEST:
+            return False
+        if x.derivation == PurposeLineage.AGGREGATE:
+            if x.address in conditions:
+                return False
+    logger.info(
+        f"Upleveling output select to final CTE with derived_concepts {[x.address for x in derived_concepts]}"
+    )
+    return eligible
+
+
+def sort_select_output(cte: CTE, query: SelectStatement | MultiSelectStatement):
+    hidden_addresses = [c.address for c in query.hidden_components]
+    output_addresses = [
+        c.address for c in query.output_components if c.address not in hidden_addresses
+    ]
+
+    mapping = {x.address: x for x in cte.output_columns}
+
+    new_output = []
+    for x in output_addresses:
+        new_output.append(mapping[x])
+    cte.output_columns = new_output
+
+
+def optimize_ctes(
+    input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
+) -> list[CTE]:
+    complete = False
+    REGISTERED_RULES: list["OptimizationRule"] = []
+    if CONFIG.optimizations.direct_return and is_direct_return_eligible(
+        root_cte, select
+    ):
+        root_cte.order_by = select.order_by
+        root_cte.limit = select.limit
+        if select.where_clause:
+
+            if root_cte.condition:
+                root_cte.condition = Conditional(
+                    left=root_cte.condition,
+                    operator=BooleanOperator.AND,
+                    right=select.where_clause.conditional,
+                )
+            else:
+                root_cte.condition = select.where_clause.conditional
+        root_cte.requires_nesting = False
+        sort_select_output(root_cte, select)
+    if CONFIG.optimizations.datasource_inlining:
+        REGISTERED_RULES.append(InlineDatasource())
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdown())
+    if CONFIG.optimizations.constant_inlining:
+        REGISTERED_RULES.append(InlineConstant())
+    loops = 0
+    while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
+        actions_taken = False
+        for rule in REGISTERED_RULES:
+            for cte in input:
+                inverse_map = gen_inverse_map(input)
+                actions_taken = actions_taken or rule.optimize(cte, inverse_map)
+        complete = not actions_taken
+        loops += 1
+
+    return filter_irrelevant_ctes(input, root_cte)

pytrilogy-0.0.1.117/trilogy/core/optimizations/__init__.py

@@ -0,0 +1,11 @@
+from .inline_constant import InlineConstant
+from .inline_datasource import InlineDatasource
+from .predicate_pushdown import PredicatePushdown
+from .base_optimization import OptimizationRule
+
+__all__ = [
+    "OptimizationRule",
+    "InlineConstant",
+    "InlineDatasource",
+    "PredicatePushdown",
+]

pytrilogy-0.0.1.117/trilogy/core/optimizations/base_optimization.py

@@ -0,0 +1,17 @@
+from trilogy.core.models import (
+    CTE,
+)
+from trilogy.constants import logger
+from abc import ABC
+
+
+class OptimizationRule(ABC):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        raise NotImplementedError
+
+    def log(self, message: str):
+        logger.info(f"[Optimization][{self.__class__.__name__}] {message}")
+
+    def debug(self, message: str):
+        logger.debug(f"[Optimization][{self.__class__.__name__}] {message}")

pytrilogy-0.0.1.117/trilogy/core/optimizations/inline_constant.py

@@ -0,0 +1,29 @@
+from trilogy.core.models import (
+    CTE,
+    Concept,
+)
+from trilogy.core.enums import PurposeLineage
+
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class InlineConstant(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+
+        to_inline: list[Concept] = []
+        for x in cte.source.input_concepts:
+            if x.address not in cte.source_map:
+                continue
+            if x.derivation == PurposeLineage.CONSTANT:
+                self.log(f"Found constant {x.address} on {cte.name}")
+                to_inline.append(x)
+        if to_inline:
+            inlined = False
+            for c in to_inline:
+                self.log(f"Inlining constant {c.address} on  {cte.name}")
+                test = cte.inline_constant(c)
+                if test:
+                    inlined = True
+            return inlined
+        return False

pytrilogy-0.0.1.117/trilogy/core/optimizations/inline_datasource.py

@@ -0,0 +1,54 @@
+from trilogy.core.models import (
+    CTE,
+    Datasource,
+)
+
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class InlineDatasource(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        if not cte.parent_ctes:
+            return False
+
+        optimized = False
+        self.log(
+            f"Checking {cte.name} for consolidating inline tables with {len(cte.parent_ctes)} parents"
+        )
+        to_inline: list[CTE] = []
+        force_group = False
+        for parent_cte in cte.parent_ctes:
+            if not parent_cte.is_root_datasource:
+                self.log(f"parent {parent_cte.name} is not root")
+                continue
+            if parent_cte.parent_ctes:
+                self.log(f"parent {parent_cte.name} has parents")
+                continue
+            raw_root = parent_cte.source.datasources[0]
+            if not isinstance(raw_root, Datasource):
+                self.log(f"parent {parent_cte.name} is not datasource")
+                continue
+            root: Datasource = raw_root
+            if not root.can_be_inlined:
+                self.log(f"parent {parent_cte.name} datasource is not inlineable")
+                continue
+            root_outputs = {x.address for x in root.output_concepts}
+            cte_outputs = {x.address for x in parent_cte.output_columns}
+            grain_components = {x.address for x in root.grain.components}
+            if not cte_outputs.issubset(root_outputs):
+                self.log(f"Not all {parent_cte.name} outputs are found on datasource")
+                continue
+            if not grain_components.issubset(cte_outputs):
+                self.log("Not all datasource components in cte outputs, forcing group")
+                force_group = True
+            to_inline.append(parent_cte)
+
+        for replaceable in to_inline:
+
+            result = cte.inline_parent_datasource(replaceable, force_group=force_group)
+            if result:
+                self.log(f"Inlined parent {replaceable.name}")
+            else:
+                self.log(f"Failed to inline {replaceable.name}")
+        return optimized

pytrilogy-0.0.1.117/trilogy/core/optimizations/predicate_pushdown.py

@@ -0,0 +1,105 @@
+from trilogy.core.models import (
+    CTE,
+    Conditional,
+    BooleanOperator,
+    Datasource,
+)
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+def decompose_condition(conditional: Conditional):
+    chunks = []
+    if conditional.operator == BooleanOperator.AND:
+        for val in [conditional.left, conditional.right]:
+            if isinstance(val, Conditional):
+                chunks.extend(decompose_condition(val))
+            else:
+                chunks.append(val)
+    else:
+        chunks.append(conditional)
+    return chunks
+
+
+def is_child_of(a, comparison):
+    if isinstance(comparison, Conditional):
+        return (
+            is_child_of(a, comparison.left) or is_child_of(a, comparison.right)
+        ) and comparison.operator == BooleanOperator.AND
+    return comparison == a
+
+
+class PredicatePushdown(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+
+            return False
+
+        optimized = False
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+        self.log(
+            f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
+        )
+        if isinstance(cte.condition, Conditional):
+            candidates = cte.condition.decompose()
+        else:
+            candidates = [cte.condition]
+        self.log(f"Have {len(candidates)} candidates to try to push down")
+        for candidate in candidates:
+            conditions = {x.address for x in candidate.concept_arguments}
+            for parent_cte in cte.parent_ctes:
+                if is_child_of(cte.condition, parent_cte.condition):
+                    continue
+                materialized = {k for k, v in parent_cte.source_map.items() if v != []}
+                # if it's a root datasource, we can filter on _any_ of the output concepts
+                if parent_cte.is_root_datasource:
+                    extra_check = {
+                        x.address
+                        for x in parent_cte.source.datasources[0].output_concepts
+                    }
+                    if conditions.issubset(extra_check):
+                        for x in conditions:
+                            if x not in materialized:
+                                materialized.add(x)
+                                parent_cte.source_map[x] = [
+                                    parent_cte.source.datasources[0].name
+                                ]
+                if conditions.issubset(materialized):
+                    if all(
+                        [
+                            is_child_of(candidate, child.condition)
+                            for child in inverse_map.get(parent_cte.name, [])
+                        ]
+                    ):
+                        self.log(
+                            f"All concepts are found on {parent_cte.name} and all it's children include same filter; pushing up filter"
+                        )
+                        if parent_cte.condition:
+                            parent_cte.condition = Conditional(
+                                left=parent_cte.condition,
+                                operator=BooleanOperator.AND,
+                                right=candidate,
+                            )
+                        else:
+                            parent_cte.condition = candidate
+                        optimized = True
+                else:
+                    self.log(
+                        f"conditions {conditions} not subset of parent {parent_cte.name} parent has {materialized} "
+                    )
+
+        if all(
+            [
+                is_child_of(cte.condition, parent_cte.condition)
+                for parent_cte in cte.parent_ctes
+            ]
+        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
+            self.log("All parents have same filter, removing filter")
+            cte.condition = None
+            optimized = True
+
+        return optimized