pytrilogy 0.0.1.114__tar.gz → 0.0.1.116__tar.gz

{pytrilogy-0.0.1.114/pytrilogy.egg-info → pytrilogy-0.0.1.116}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.114
+Version: 0.0.1.116
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.114 → pytrilogy-0.0.1.116/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.114
+Version: 0.0.1.116
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.114 → pytrilogy-0.0.1.116}/pytrilogy.egg-info/SOURCES.txt

@@ -45,6 +45,11 @@ trilogy/core/internal.py
 trilogy/core/models.py
 trilogy/core/optimization.py
 trilogy/core/query_processor.py
+trilogy/core/optimizations/__init__.py
+trilogy/core/optimizations/base_optimization.py
+trilogy/core/optimizations/inline_constant.py
+trilogy/core/optimizations/inline_datasource.py
+trilogy/core/optimizations/predicate_pushdown.py
 trilogy/core/processing/__init__.py
 trilogy/core/processing/concept_strategies_v3.py
 trilogy/core/processing/graph_utils.py

{pytrilogy-0.0.1.114 → pytrilogy-0.0.1.116}/tests/test_parsing.py

@@ -263,3 +263,44 @@ def test_the_comments():
     assert isinstance(right, MagicConstants), type(right)
     rendered = BaseDialect().render_expr(right)
     assert rendered == "null"
+
+
+def test_purpose_nesting():
+
+    env, parsed = parse_text(
+        """key year int;
+"""
+    )
+
+    env2: Environment = Environment()
+    env2.add_import("dates", env)
+
+    env2, _ = parse_text(
+        """
+property <dates.year>.generation <-
+CASE WHEN dates.year BETWEEN 1883 AND 1900 THEN 'Lost Generation'
+        WHEN dates.year BETWEEN 1901 AND 1927 THEN 'The Greatest Generation'
+        WHEN dates.year BETWEEN 1928 AND 1945 THEN 'The Silent Generation'
+        WHEN dates.year BETWEEN 1946 AND 1964 THEN 'Baby Boomer'
+        WHEN dates.year BETWEEN 1965 AND 1980 THEN 'Generation X'
+        WHEN dates.year BETWEEN 1981 AND 1996 THEN 'Millennials'
+        WHEN dates.year BETWEEN 1997 AND 2012 THEN 'Generation Z'
+        ELSE 'Unknown'
+    END;
+    """,
+        env2,
+    )
+
+    assert env2.concepts["dates.generation"].purpose == Purpose.PROPERTY
+
+
+def test_rawsql():
+    env, parsed = parse_text(
+        """
+raw_sql('''select 1''');
+
+select 1 as test;
+
+"""
+    )
+    assert parsed[0].text == "select 1"

{pytrilogy-0.0.1.114 → pytrilogy-0.0.1.116}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.1.114"
+__version__ = "0.0.1.116"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.1.114 → pytrilogy-0.0.1.116}/trilogy/constants.py

@@ -22,6 +22,7 @@ NULL_VALUE = MagicConstants.NULL
 class Optimizations:
     predicate_pushdown: bool = True
     datasource_inlining: bool = True
+    constant_inlining: bool = True
     direct_return: bool = True
 
 

{pytrilogy-0.0.1.114 → pytrilogy-0.0.1.116}/trilogy/core/models.py

@@ -1165,11 +1165,17 @@ class OrderBy(Namespaced, BaseModel):
         return OrderBy(items=[x.with_namespace(namespace) for x in self.items])
 
 
+class RawSQLStatement(BaseModel):
+    text: str
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
+
+
 class SelectStatement(Namespaced, BaseModel):
     selection: List[SelectItem]
     where_clause: Optional["WhereClause"] = None
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     def __str__(self):
         from trilogy.parsing.render import render_query
@@ -1371,6 +1377,7 @@ class MultiSelectStatement(Namespaced, BaseModel):
     where_clause: Optional["WhereClause"] = None
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     def __repr__(self):
         return "MultiSelect<" + " MERGE ".join([str(s) for s in self.selects]) + ">"
@@ -2038,6 +2045,40 @@ class CTE(BaseModel):
     def validate_output_columns(cls, v):
         return unique(v, "address")
 
+    def inline_constant(self, concept: Concept):
+        if not concept.derivation == PurposeLineage.CONSTANT:
+            return False
+        if not isinstance(concept.lineage, Function):
+            return False
+        if not concept.lineage.operator == FunctionType.CONSTANT:
+            return False
+        # remove the constant
+        removed: set = set()
+        if concept.address in self.source_map:
+            removed = removed.union(self.source_map[concept.address])
+            del self.source_map[concept.address]
+        # if we've entirely removed the need to join to someplace to get the concept
+        # drop the join as well.
+        for removed_cte in removed:
+            still_required = any([removed_cte in x for x in self.source_map.values()])
+            if not still_required:
+                self.joins = [
+                    join
+                    for join in self.joins
+                    if not isinstance(join, Join)
+                    or (
+                        join.right_cte.name != removed_cte
+                        and join.left_cte.name != removed_cte
+                    )
+                ]
+                self.parent_ctes = [
+                    x for x in self.parent_ctes if x.name != removed_cte
+                ]
+                if removed_cte == self.base_name_override:
+                    candidates = [x.name for x in self.parent_ctes]
+                    self.base_name_override = candidates[0] if candidates else None
+                    self.base_alias_override = candidates[0] if candidates else None
+
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
         qds_being_inlined = parent.source
         ds_being_inlined = qds_being_inlined.datasources[0]
@@ -2186,6 +2227,11 @@ class CTE(BaseModel):
             and not self.group_to_grain
         ):
             return False
+        # if we don't need to source any concepts from anywhere
+        # render without from
+        # most likely to happen from inlining constants
+        if not any([v for v in self.source_map.values()]):
+            return False
         if (
             len(self.source.datasources) == 1
             and self.source.datasources[0].name == CONSTANT_DATASET
@@ -2669,7 +2715,8 @@ class Environment(BaseModel):
 
 
 class LazyEnvironment(Environment):
-    """Variant of environment to defer parsing of a path"""
+    """Variant of environment to defer parsing of a path
+    until relevant attributes accessed."""
 
     load_path: Path
     loaded: bool = False
@@ -2681,6 +2728,7 @@ class LazyEnvironment(Environment):
             "working_path",
             "model_config",
             "model_fields",
+            "model_post_init",
         ) or name.startswith("_"):
             return super().__getattribute__(name)
         if not self.loaded:
@@ -3182,6 +3230,10 @@ class ProcessedShowStatement(BaseModel):
     output_values: List[Union[Concept, Datasource, ProcessedQuery]]
 
 
+class ProcessedRawSQLStatement(BaseModel):
+    text: str
+
+
 class Limit(BaseModel):
     count: int
 
@@ -3384,6 +3436,7 @@ class Parenthetical(ConceptArgs, Namespaced, SelectGrain, BaseModel):
 class PersistStatement(BaseModel):
     datasource: Datasource
     select: SelectStatement
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     @property
     def identifier(self):

pytrilogy-0.0.1.116/trilogy/core/optimization.py

@@ -0,0 +1,131 @@
+from trilogy.core.models import (
+    CTE,
+    SelectStatement,
+    PersistStatement,
+    MultiSelectStatement,
+    Conditional,
+    BooleanOperator,
+)
+from trilogy.core.enums import PurposeLineage
+from trilogy.constants import logger, CONFIG
+from trilogy.core.optimizations import (
+    OptimizationRule,
+    InlineConstant,
+    PredicatePushdown,
+    InlineDatasource,
+)
+
+
+MAX_OPTIMIZATION_LOOPS = 100
+
+
+def filter_irrelevant_ctes(
+    input: list[CTE],
+    root_cte: CTE,
+):
+    relevant_ctes = set()
+
+    def recurse(cte: CTE):
+        relevant_ctes.add(cte.name)
+        for cte in cte.parent_ctes:
+            recurse(cte)
+
+    recurse(root_cte)
+    return [cte for cte in input if cte.name in relevant_ctes]
+
+
+def gen_inverse_map(input: list[CTE]) -> dict[str, list[CTE]]:
+    inverse_map: dict[str, list[CTE]] = {}
+    for cte in input:
+        for parent in cte.parent_ctes:
+            if parent.name not in inverse_map:
+                inverse_map[parent.name] = []
+            inverse_map[parent.name].append(cte)
+    return inverse_map
+
+
+def is_direct_return_eligible(
+    cte: CTE, select: SelectStatement | PersistStatement | MultiSelectStatement
+) -> bool:
+    if isinstance(select, (PersistStatement, MultiSelectStatement)):
+        return False
+    derived_concepts = [
+        c for c in cte.source.output_concepts if c not in cte.source.input_concepts
+    ]
+    eligible = True
+    conditions = (
+        set(x.address for x in select.where_clause.concept_arguments)
+        if select.where_clause
+        else set()
+    )
+    if conditions and select.limit:
+        return False
+    for x in derived_concepts:
+        if x.derivation == PurposeLineage.WINDOW:
+            return False
+        if x.derivation == PurposeLineage.UNNEST:
+            return False
+        if x.derivation == PurposeLineage.AGGREGATE:
+            if x.address in conditions:
+                return False
+    logger.info(
+        f"Upleveling output select to final CTE with derived_concepts {[x.address for x in derived_concepts]}"
+    )
+    return eligible
+
+
+def sort_select_output(cte: CTE, query: SelectStatement | MultiSelectStatement):
+    hidden_addresses = [c.address for c in query.hidden_components]
+    output_addresses = [
+        c.address for c in query.output_components if c.address not in hidden_addresses
+    ]
+
+    mapping = {x.address: x for x in cte.output_columns}
+
+    new_output = []
+    for x in output_addresses:
+        new_output.append(mapping[x])
+    cte.output_columns = new_output
+
+
+def optimize_ctes(
+    input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
+):
+    complete = False
+    REGISTERED_RULES: list["OptimizationRule"] = []
+
+    if CONFIG.optimizations.datasource_inlining:
+        REGISTERED_RULES.append(InlineDatasource())
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdown())
+    if CONFIG.optimizations.constant_inlining:
+        REGISTERED_RULES.append(InlineConstant())
+    loops = 0
+    while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
+        actions_taken = False
+        for rule in REGISTERED_RULES:
+            for cte in input:
+                inverse_map = gen_inverse_map(input)
+                actions_taken = rule.optimize(cte, inverse_map)
+        complete = not actions_taken
+        loops += 1
+
+    if CONFIG.optimizations.direct_return and is_direct_return_eligible(
+        root_cte, select
+    ):
+        root_cte.order_by = select.order_by
+        root_cte.limit = select.limit
+        if select.where_clause:
+
+            if root_cte.condition:
+                root_cte.condition = Conditional(
+                    left=root_cte.condition,
+                    operator=BooleanOperator.AND,
+                    right=select.where_clause.conditional,
+                )
+            else:
+                root_cte.condition = select.where_clause.conditional
+        root_cte.requires_nesting = False
+        sort_select_output(root_cte, select)
+
+    return filter_irrelevant_ctes(input, root_cte)

pytrilogy-0.0.1.116/trilogy/core/optimizations/__init__.py

@@ -0,0 +1,11 @@
+from .inline_constant import InlineConstant
+from .inline_datasource import InlineDatasource
+from .predicate_pushdown import PredicatePushdown
+from .base_optimization import OptimizationRule
+
+__all__ = [
+    "OptimizationRule",
+    "InlineConstant",
+    "InlineDatasource",
+    "PredicatePushdown",
+]

pytrilogy-0.0.1.116/trilogy/core/optimizations/base_optimization.py

@@ -0,0 +1,17 @@
+from trilogy.core.models import (
+    CTE,
+)
+from trilogy.constants import logger
+from abc import ABC
+
+
+class OptimizationRule(ABC):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        raise NotImplementedError
+
+    def log(self, message: str):
+        logger.info(f"[Optimization][{self.__class__.__name__}] {message}")
+
+    def debug(self, message: str):
+        logger.debug(f"[Optimization][{self.__class__.__name__}] {message}")

pytrilogy-0.0.1.116/trilogy/core/optimizations/inline_constant.py

@@ -0,0 +1,29 @@
+from trilogy.core.models import (
+    CTE,
+    Concept,
+)
+from trilogy.core.enums import PurposeLineage
+
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class InlineConstant(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+
+        to_inline: list[Concept] = []
+        for x in cte.source.input_concepts:
+            if x.address not in cte.source_map:
+                continue
+            if x.derivation == PurposeLineage.CONSTANT:
+                self.log(f"Found constant {x.address} on {cte.name}")
+                to_inline.append(x)
+        if to_inline:
+            inlined = False
+            for c in to_inline:
+                self.log(f"Inlining constant {c.address} on  {cte.name}")
+                test = cte.inline_constant(c)
+                if test:
+                    inlined = True
+            return inlined
+        return False

pytrilogy-0.0.1.116/trilogy/core/optimizations/inline_datasource.py

@@ -0,0 +1,54 @@
+from trilogy.core.models import (
+    CTE,
+    Datasource,
+)
+
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class InlineDatasource(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        if not cte.parent_ctes:
+            return False
+
+        optimized = False
+        self.log(
+            f"Checking {cte.name} for consolidating inline tables with {len(cte.parent_ctes)} parents"
+        )
+        to_inline: list[CTE] = []
+        force_group = False
+        for parent_cte in cte.parent_ctes:
+            if not parent_cte.is_root_datasource:
+                self.log(f"parent {parent_cte.name} is not root")
+                continue
+            if parent_cte.parent_ctes:
+                self.log(f"parent {parent_cte.name} has parents")
+                continue
+            raw_root = parent_cte.source.datasources[0]
+            if not isinstance(raw_root, Datasource):
+                self.log(f"parent {parent_cte.name} is not datasource")
+                continue
+            root: Datasource = raw_root
+            if not root.can_be_inlined:
+                self.log(f"parent {parent_cte.name} datasource is not inlineable")
+                continue
+            root_outputs = {x.address for x in root.output_concepts}
+            cte_outputs = {x.address for x in parent_cte.output_columns}
+            grain_components = {x.address for x in root.grain.components}
+            if not cte_outputs.issubset(root_outputs):
+                self.log(f"Not all {parent_cte.name} outputs are found on datasource")
+                continue
+            if not grain_components.issubset(cte_outputs):
+                self.log("Not all datasource components in cte outputs, forcing group")
+                force_group = True
+            to_inline.append(parent_cte)
+
+        for replaceable in to_inline:
+
+            result = cte.inline_parent_datasource(replaceable, force_group=force_group)
+            if result:
+                self.log(f"Inlined parent {replaceable.name}")
+            else:
+                self.log(f"Failed to inline {replaceable.name}")
+        return optimized

pytrilogy-0.0.1.116/trilogy/core/optimizations/predicate_pushdown.py

@@ -0,0 +1,88 @@
+from trilogy.core.models import (
+    CTE,
+    Conditional,
+    BooleanOperator,
+)
+from trilogy.constants import logger
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+def decompose_condition(conditional: Conditional):
+    chunks = []
+    if conditional.operator == BooleanOperator.AND:
+        for val in [conditional.left, conditional.right]:
+            if isinstance(val, Conditional):
+                chunks.extend(decompose_condition(val))
+            else:
+                chunks.append(val)
+    else:
+        chunks.append(conditional)
+    return chunks
+
+
+def is_child_of(a, comparison):
+    if isinstance(comparison, Conditional):
+        return (
+            is_child_of(a, comparison.left) or is_child_of(a, comparison.right)
+        ) and comparison.operator == BooleanOperator.AND
+    return comparison == a
+
+
+class PredicatePushdown(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+
+            return False
+
+        optimized = False
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+        self.log(
+            f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
+        )
+        if isinstance(cte.condition, Conditional):
+            candidates = cte.condition.decompose()
+        else:
+            candidates = [cte.condition]
+        logger.info(f"Have {len(candidates)} candidates to try to push down")
+        for candidate in candidates:
+            conditions = {x.address for x in candidate.concept_arguments}
+            for parent_cte in cte.parent_ctes:
+                materialized = {k for k, v in parent_cte.source_map.items() if v != []}
+                if conditions.issubset(materialized):
+                    if all(
+                        [
+                            is_child_of(candidate, child.condition)
+                            for child in inverse_map[parent_cte.name]
+                        ]
+                    ):
+                        self.log(
+                            f"All concepts are found on {parent_cte.name} and all it's children include same filter; pushing up filter"
+                        )
+                        if parent_cte.condition:
+                            parent_cte.condition = Conditional(
+                                left=parent_cte.condition,
+                                operator=BooleanOperator.AND,
+                                right=candidate,
+                            )
+                        else:
+                            parent_cte.condition = candidate
+                        optimized = True
+                else:
+                    logger.info("conditions not subset of parent materialized")
+
+        if all(
+            [
+                is_child_of(cte.condition, parent_cte.condition)
+                for parent_cte in cte.parent_ctes
+            ]
+        ):
+            self.log("All parents have same filter, removing filter")
+            cte.condition = None
+            optimized = True
+
+        return optimized

{pytrilogy-0.0.1.114 → pytrilogy-0.0.1.116}/trilogy/dialect/base.py

@@ -44,6 +44,8 @@ from trilogy.core.models import (
     RowsetDerivationStatement,
     ConceptDeclarationStatement,
     ImportStatement,
+    RawSQLStatement,
+    ProcessedRawSQLStatement,
 )
 from trilogy.core.query_processor import process_query, process_persist
 from trilogy.dialect.common import render_join
@@ -558,11 +560,20 @@ class BaseDialect:
             | RowsetDerivationStatement
             | MergeStatement
             | ImportStatement
+            | RawSQLStatement
         ],
         hooks: Optional[List[BaseHook]] = None,
-    ) -> List[ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement]:
+    ) -> List[
+        ProcessedQuery
+        | ProcessedQueryPersist
+        | ProcessedShowStatement
+        | ProcessedRawSQLStatement
+    ]:
         output: List[
-            ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement
+            ProcessedQuery
+            | ProcessedQueryPersist
+            | ProcessedShowStatement
+            | ProcessedRawSQLStatement
         ] = []
         for statement in statements:
             if isinstance(statement, PersistStatement):
@@ -604,6 +615,8 @@ class BaseDialect:
                     )
                 else:
                     raise NotImplementedError(type(statement))
+            elif isinstance(statement, RawSQLStatement):
+                output.append(ProcessedRawSQLStatement(text=statement.text))
             elif isinstance(
                 statement,
                 (
@@ -619,10 +632,18 @@ class BaseDialect:
         return output
 
     def compile_statement(
-        self, query: ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement
+        self,
+        query: (
+            ProcessedQuery
+            | ProcessedQueryPersist
+            | ProcessedShowStatement
+            | ProcessedRawSQLStatement
+        ),
     ) -> str:
         if isinstance(query, ProcessedShowStatement):
             return ";\n".join([str(x) for x in query.output_values])
+        elif isinstance(query, ProcessedRawSQLStatement):
+            return query.text
         select_columns: Dict[str, str] = {}
         cte_output_map = {}
         selected = set()

{pytrilogy-0.0.1.114 → pytrilogy-0.0.1.116}/trilogy/dialect/sql_server.py

@@ -9,6 +9,7 @@ from trilogy.core.models import (
     ProcessedQuery,
     ProcessedQueryPersist,
     ProcessedShowStatement,
+    ProcessedRawSQLStatement,
 )
 from trilogy.dialect.base import BaseDialect
 
@@ -81,7 +82,13 @@ class SqlServerDialect(BaseDialect):
     SQL_TEMPLATE = TSQL_TEMPLATE
 
     def compile_statement(
-        self, query: ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement
+        self,
+        query: (
+            ProcessedQuery
+            | ProcessedQueryPersist
+            | ProcessedShowStatement
+            | ProcessedRawSQLStatement
+        ),
     ) -> str:
         base = super().compile_statement(query)
         if isinstance(base, (ProcessedQuery, ProcessedQueryPersist)):