pytrilogy 0.0.1.115__py3-none-any.whl → 0.0.1.116__py3-none-any.whl

{pytrilogy-0.0.1.115.dist-info → pytrilogy-0.0.1.116.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.115
+Version: 0.0.1.116
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -15,17 +15,17 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: lark
 Requires-Dist: jinja2
-Requires-Dist: sqlalchemy <2.0.0
+Requires-Dist: sqlalchemy<2.0.0
 Requires-Dist: networkx
 Requires-Dist: pyodbc
 Requires-Dist: pydantic
 Requires-Dist: duckdb-engine
 Provides-Extra: bigquery
-Requires-Dist: sqlalchemy-bigquery ; extra == 'bigquery'
+Requires-Dist: sqlalchemy-bigquery; extra == "bigquery"
 Provides-Extra: postgres
-Requires-Dist: psycopg2-binary ; extra == 'postgres'
+Requires-Dist: psycopg2-binary; extra == "postgres"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-sqlalchemy ; extra == 'snowflake'
+Requires-Dist: snowflake-sqlalchemy; extra == "snowflake"
 
 ## Trilogy
 [![Website](https://img.shields.io/badge/INTRO-WEB-orange?)](https://trilogydata.dev/)

{pytrilogy-0.0.1.115.dist-info → pytrilogy-0.0.1.116.dist-info}/RECORD

@@ -1,8 +1,8 @@
-trilogy/__init__.py,sha256=FLkaJwdfgAlanF07q1NrFtMnJiCyOuGWPWbUpfXnttg,292
+trilogy/__init__.py,sha256=0jWazkuszqLHMaM49JtN8aEur-f9lwCZ5PqZbNhO10E,292
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=DJi3ESttmvqgy6fPRXiaQzqJVye6jYwf6XM89NHv0_M,735
+trilogy/constants.py,sha256=u2dNxhwy0v-6HrvG1GcpDVvuhzdTH5fuyYNCxDPlr2E,770
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
-trilogy/executor.py,sha256=_ZbjrKsUdWL52tWgpxqZnmccAuPXcIPEPN_dDSLNeAQ,9696
+trilogy/executor.py,sha256=auuDykCHeqlRWIHOfBfgIIIntEctWaUC-VPJr1DQbYk,10217
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
@@ -16,9 +16,14 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=zkRReytiotOBAW-a3Ri5eoejZDYTt2-7Op80ZxZxUmw,9129
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=2qkebdlVsHeGp1foupR6LJ5q9YewlZcZq6utxdLBqIw,111565
-trilogy/core/optimization.py,sha256=942MnGRzscAHcG9LsfMslIRRQBslbIiPHnAvJ3w8YRg,9157
+trilogy/core/models.py,sha256=_k_ZqrsZ6HYNz3CJ8yiOVXMej1pHb_QO0KZNxkDyvno,113767
+trilogy/core/optimization.py,sha256=1xAFn7aw8skqDFUQCel5xJc0hVUYHs-oW1DckN8Z4n4,4043
 trilogy/core/query_processor.py,sha256=clIRJ6IcsqIVBPKFsxt8bqCLsLyajvAu02MUIcKQhTo,15713
+trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
+trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
+trilogy/core/optimizations/inline_constant.py,sha256=neZOFjX7M2pzQ-8m-f8nApy_MfJuowX6SzcGwGFt5w4,927
+trilogy/core/optimizations/inline_datasource.py,sha256=BSp54fwF4RRwInd-09pggemC7JuXj-uqGzi32ufeqYo,2171
+trilogy/core/optimizations/predicate_pushdown.py,sha256=s5BMLADcQy4_UsMCchHTsInaJUtLGh1l2kTpqzAzcs4,3199
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/concept_strategies_v3.py,sha256=MYrpNMidqvPOg123RekOcqVTjcj03i_538gBo0MzoWE,23432
 trilogy/core/processing/graph_utils.py,sha256=ulCJ4hYAISbUxLD6VM2fah9RBPGIXSEHEPeRBSFl0Rs,1197
@@ -45,7 +50,7 @@ trilogy/core/processing/nodes/select_node_v2.py,sha256=ERCflBFzKpD5SzweMevnJLyQn
 trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=iy2jb43CvJ0xSozyNVLuJSCaZ4Uzb5o--2p2-ZKNzSM,30991
+trilogy/dialect/base.py,sha256=ro9fKV6RrPggUVmJPlIEdySLsR3Pq5rm0UMQho6Bx_k,31518
 trilogy/dialect/bigquery.py,sha256=9vxQn2BMv_oTGQSWQpoN5ho_OgqMWaHH9e-5vQVf44c,2906
 trilogy/dialect/common.py,sha256=zWrYmvevlXznocw9uGHmY5Ws1rp_kICm9zA_ulTe4eg,2165
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
@@ -54,7 +59,7 @@ trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
 trilogy/dialect/postgres.py,sha256=r47xbCA7nfEYENofiVfLZ-SnReNfDmUmW4OSHVkkP4E,3206
 trilogy/dialect/presto.py,sha256=8zjRn8AeYXZQGuUi-afyBWLet8o-LSt6gm5IH7bTdiw,2987
 trilogy/dialect/snowflake.py,sha256=N3HknYgN-fjD7BLX1Ucj-ss_ku2Ox8DgLsF3BIHutHo,2941
-trilogy/dialect/sql_server.py,sha256=UrLeA9bxiFJ4qpGsqVJqBybQCyJhetMebe8IzQW1q9s,2900
+trilogy/dialect/sql_server.py,sha256=HX68vNTrcDaTnOxe6Zbx_PBgrO42e2VuThxO6CYQ2cY,3026
 trilogy/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/hooks/base_hook.py,sha256=Xkb-A2qCHozYjum0A36zOy5PwTVwrP3NLDF0U2GpgHo,1100
 trilogy/hooks/graph_hook.py,sha256=i-Tv9sxZU0sMc-God8bLLz-nAg4-wYafogZtHaU8LXw,801
@@ -65,14 +70,14 @@ trilogy/parsing/common.py,sha256=iR3fiiZ7w8VJuUGrQ0v06XGDXov81f4z1ZlFnj6y40E,580
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=9j0mUN6D48iH39YRPRTp5JsDaB9YLFaF-xmF62ALMSA,56256
-trilogy/parsing/render.py,sha256=fxjpq2FZLgllw_d4cru-t_IXNPAz2DmYkT7v9ED0XRI,11540
-trilogy/parsing/trilogy.lark,sha256=GpjGQaDmK7GOTIqK1n4lMbDXqpt_w3EkqlYhlFw3HEA,10679
+trilogy/parsing/parse_engine.py,sha256=cT1IoLP0LzykNX0UfH9-eu8PzjfzSNCtZ_hZvf58TRg,56761
+trilogy/parsing/render.py,sha256=cOKHvl2ZrpkD4lpng9iXc37gc3XyZ1e51woPxB7QqsI,11672
+trilogy/parsing/trilogy.lark,sha256=xYpVKneNhXruxIyzFqcK1b8-NSqzzEwtPrTxmpXd_Vc,10786
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.1.115.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.1.115.dist-info/METADATA,sha256=6lmw6TXDHhgLTQoHRIqZxy3YK7oq0hQPMtV9Z9wzk0g,7882
-pytrilogy-0.0.1.115.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-pytrilogy-0.0.1.115.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.1.115.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.1.115.dist-info/RECORD,,
+pytrilogy-0.0.1.116.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.1.116.dist-info/METADATA,sha256=kyUsh-1RPnsedKLexiwJCKTl9ZzZHl5Uu4LUQcV9hj0,7878
+pytrilogy-0.0.1.116.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+pytrilogy-0.0.1.116.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.1.116.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.1.116.dist-info/RECORD,,

{pytrilogy-0.0.1.115.dist-info → pytrilogy-0.0.1.116.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (71.1.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.1.115"
+__version__ = "0.0.1.116"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -22,6 +22,7 @@ NULL_VALUE = MagicConstants.NULL
 class Optimizations:
     predicate_pushdown: bool = True
     datasource_inlining: bool = True
+    constant_inlining: bool = True
     direct_return: bool = True
 
 

trilogy/core/models.py

@@ -1165,11 +1165,17 @@ class OrderBy(Namespaced, BaseModel):
         return OrderBy(items=[x.with_namespace(namespace) for x in self.items])
 
 
+class RawSQLStatement(BaseModel):
+    text: str
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
+
+
 class SelectStatement(Namespaced, BaseModel):
     selection: List[SelectItem]
     where_clause: Optional["WhereClause"] = None
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     def __str__(self):
         from trilogy.parsing.render import render_query
@@ -1371,6 +1377,7 @@ class MultiSelectStatement(Namespaced, BaseModel):
     where_clause: Optional["WhereClause"] = None
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     def __repr__(self):
         return "MultiSelect<" + " MERGE ".join([str(s) for s in self.selects]) + ">"
@@ -2038,6 +2045,40 @@ class CTE(BaseModel):
     def validate_output_columns(cls, v):
         return unique(v, "address")
 
+    def inline_constant(self, concept: Concept):
+        if not concept.derivation == PurposeLineage.CONSTANT:
+            return False
+        if not isinstance(concept.lineage, Function):
+            return False
+        if not concept.lineage.operator == FunctionType.CONSTANT:
+            return False
+        # remove the constant
+        removed: set = set()
+        if concept.address in self.source_map:
+            removed = removed.union(self.source_map[concept.address])
+            del self.source_map[concept.address]
+        # if we've entirely removed the need to join to someplace to get the concept
+        # drop the join as well.
+        for removed_cte in removed:
+            still_required = any([removed_cte in x for x in self.source_map.values()])
+            if not still_required:
+                self.joins = [
+                    join
+                    for join in self.joins
+                    if not isinstance(join, Join)
+                    or (
+                        join.right_cte.name != removed_cte
+                        and join.left_cte.name != removed_cte
+                    )
+                ]
+                self.parent_ctes = [
+                    x for x in self.parent_ctes if x.name != removed_cte
+                ]
+                if removed_cte == self.base_name_override:
+                    candidates = [x.name for x in self.parent_ctes]
+                    self.base_name_override = candidates[0] if candidates else None
+                    self.base_alias_override = candidates[0] if candidates else None
+
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
         qds_being_inlined = parent.source
         ds_being_inlined = qds_being_inlined.datasources[0]
@@ -2186,6 +2227,11 @@ class CTE(BaseModel):
             and not self.group_to_grain
         ):
             return False
+        # if we don't need to source any concepts from anywhere
+        # render without from
+        # most likely to happen from inlining constants
+        if not any([v for v in self.source_map.values()]):
+            return False
         if (
             len(self.source.datasources) == 1
             and self.source.datasources[0].name == CONSTANT_DATASET
@@ -3184,6 +3230,10 @@ class ProcessedShowStatement(BaseModel):
     output_values: List[Union[Concept, Datasource, ProcessedQuery]]
 
 
+class ProcessedRawSQLStatement(BaseModel):
+    text: str
+
+
 class Limit(BaseModel):
     count: int
 
@@ -3386,6 +3436,7 @@ class Parenthetical(ConceptArgs, Namespaced, SelectGrain, BaseModel):
 class PersistStatement(BaseModel):
     datasource: Datasource
     select: SelectStatement
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     @property
     def identifier(self):

trilogy/core/optimization.py

@@ -2,155 +2,21 @@ from trilogy.core.models import (
     CTE,
     SelectStatement,
     PersistStatement,
-    Datasource,
     MultiSelectStatement,
     Conditional,
     BooleanOperator,
 )
 from trilogy.core.enums import PurposeLineage
 from trilogy.constants import logger, CONFIG
-from abc import ABC
-
-
-class OptimizationRule(ABC):
-
-    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
-        raise NotImplementedError
-
-    def log(self, message: str):
-        logger.info(f"[Optimization][{self.__class__.__name__}] {message}")
-
-    def debug(self, message: str):
-        logger.debug(f"[Optimization][{self.__class__.__name__}] {message}")
-
-
-class InlineDatasource(OptimizationRule):
-
-    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
-        if not cte.parent_ctes:
-            return False
-
-        optimized = False
-        self.log(
-            f"Checking {cte.name} for consolidating inline tables with {len(cte.parent_ctes)} parents"
-        )
-        to_inline: list[CTE] = []
-        force_group = False
-        for parent_cte in cte.parent_ctes:
-            if not parent_cte.is_root_datasource:
-                self.log(f"parent {parent_cte.name} is not root")
-                continue
-            if parent_cte.parent_ctes:
-                self.log(f"parent {parent_cte.name} has parents")
-                continue
-            raw_root = parent_cte.source.datasources[0]
-            if not isinstance(raw_root, Datasource):
-                self.log(f"parent {parent_cte.name} is not datasource")
-                continue
-            root: Datasource = raw_root
-            if not root.can_be_inlined:
-                self.log(f"parent {parent_cte.name} datasource is not inlineable")
-                continue
-            root_outputs = {x.address for x in root.output_concepts}
-            cte_outputs = {x.address for x in parent_cte.output_columns}
-            grain_components = {x.address for x in root.grain.components}
-            if not cte_outputs.issubset(root_outputs):
-                self.log(f"Not all {parent_cte.name} outputs are found on datasource")
-                continue
-            if not grain_components.issubset(cte_outputs):
-                self.log("Not all datasource components in cte outputs, forcing group")
-                force_group = True
-            to_inline.append(parent_cte)
-
-        for replaceable in to_inline:
-
-            result = cte.inline_parent_datasource(replaceable, force_group=force_group)
-            if result:
-                self.log(f"Inlined parent {replaceable.name}")
-            else:
-                self.log(f"Failed to inline {replaceable.name}")
-        return optimized
-
-
-def decompose_condition(conditional: Conditional):
-    chunks = []
-    if conditional.operator == BooleanOperator.AND:
-        for val in [conditional.left, conditional.right]:
-            if isinstance(val, Conditional):
-                chunks.extend(decompose_condition(val))
-            else:
-                chunks.append(val)
-    else:
-        chunks.append(conditional)
-    return chunks
-
-
-def is_child_of(a, comparison):
-    if isinstance(comparison, Conditional):
-        return (
-            is_child_of(a, comparison.left) or is_child_of(a, comparison.right)
-        ) and comparison.operator == BooleanOperator.AND
-    return comparison == a
-
-
-class PredicatePushdown(OptimizationRule):
-
-    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
-
-        if not cte.parent_ctes:
-            self.debug(f"No parent CTEs for {cte.name}")
-
-            return False
-
-        optimized = False
-        if not cte.condition:
-            self.debug(f"No CTE condition for {cte.name}")
-            return False
-        self.log(
-            f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
-        )
-        if isinstance(cte.condition, Conditional):
-            candidates = cte.condition.decompose()
-        else:
-            candidates = [cte.condition]
-        logger.info(f"Have {len(candidates)} candidates to try to push down")
-        for candidate in candidates:
-            conditions = {x.address for x in candidate.concept_arguments}
-            for parent_cte in cte.parent_ctes:
-                materialized = {k for k, v in parent_cte.source_map.items() if v != []}
-                if conditions.issubset(materialized):
-                    if all(
-                        [
-                            is_child_of(candidate, child.condition)
-                            for child in inverse_map[parent_cte.name]
-                        ]
-                    ):
-                        self.log(
-                            f"All concepts are found on {parent_cte.name} and all it's children include same filter; pushing up filter"
-                        )
-                        if parent_cte.condition:
-                            parent_cte.condition = Conditional(
-                                left=parent_cte.condition,
-                                operator=BooleanOperator.AND,
-                                right=candidate,
-                            )
-                        else:
-                            parent_cte.condition = candidate
-                        optimized = True
-                else:
-                    logger.info("conditions not subset of parent materialized")
+from trilogy.core.optimizations import (
+    OptimizationRule,
+    InlineConstant,
+    PredicatePushdown,
+    InlineDatasource,
+)
 
-        if all(
-            [
-                is_child_of(cte.condition, parent_cte.condition)
-                for parent_cte in cte.parent_ctes
-            ]
-        ):
-            self.log("All parents have same filter, removing filter")
-            cte.condition = None
-            optimized = True
 
-        return optimized
+MAX_OPTIMIZATION_LOOPS = 100
 
 
 def filter_irrelevant_ctes(
@@ -232,14 +98,17 @@ def optimize_ctes(
         REGISTERED_RULES.append(InlineDatasource())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdown())
-
-    while not complete:
+    if CONFIG.optimizations.constant_inlining:
+        REGISTERED_RULES.append(InlineConstant())
+    loops = 0
+    while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
         actions_taken = False
         for rule in REGISTERED_RULES:
             for cte in input:
                 inverse_map = gen_inverse_map(input)
                 actions_taken = rule.optimize(cte, inverse_map)
         complete = not actions_taken
+        loops += 1
 
     if CONFIG.optimizations.direct_return and is_direct_return_eligible(
         root_cte, select

trilogy/core/optimizations/__init__.py

@@ -0,0 +1,11 @@
+from .inline_constant import InlineConstant
+from .inline_datasource import InlineDatasource
+from .predicate_pushdown import PredicatePushdown
+from .base_optimization import OptimizationRule
+
+__all__ = [
+    "OptimizationRule",
+    "InlineConstant",
+    "InlineDatasource",
+    "PredicatePushdown",
+]

trilogy/core/optimizations/base_optimization.py

@@ -0,0 +1,17 @@
+from trilogy.core.models import (
+    CTE,
+)
+from trilogy.constants import logger
+from abc import ABC
+
+
+class OptimizationRule(ABC):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        raise NotImplementedError
+
+    def log(self, message: str):
+        logger.info(f"[Optimization][{self.__class__.__name__}] {message}")
+
+    def debug(self, message: str):
+        logger.debug(f"[Optimization][{self.__class__.__name__}] {message}")

trilogy/core/optimizations/inline_constant.py

@@ -0,0 +1,29 @@
+from trilogy.core.models import (
+    CTE,
+    Concept,
+)
+from trilogy.core.enums import PurposeLineage
+
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class InlineConstant(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+
+        to_inline: list[Concept] = []
+        for x in cte.source.input_concepts:
+            if x.address not in cte.source_map:
+                continue
+            if x.derivation == PurposeLineage.CONSTANT:
+                self.log(f"Found constant {x.address} on {cte.name}")
+                to_inline.append(x)
+        if to_inline:
+            inlined = False
+            for c in to_inline:
+                self.log(f"Inlining constant {c.address} on  {cte.name}")
+                test = cte.inline_constant(c)
+                if test:
+                    inlined = True
+            return inlined
+        return False

trilogy/core/optimizations/inline_datasource.py

@@ -0,0 +1,54 @@
+from trilogy.core.models import (
+    CTE,
+    Datasource,
+)
+
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class InlineDatasource(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        if not cte.parent_ctes:
+            return False
+
+        optimized = False
+        self.log(
+            f"Checking {cte.name} for consolidating inline tables with {len(cte.parent_ctes)} parents"
+        )
+        to_inline: list[CTE] = []
+        force_group = False
+        for parent_cte in cte.parent_ctes:
+            if not parent_cte.is_root_datasource:
+                self.log(f"parent {parent_cte.name} is not root")
+                continue
+            if parent_cte.parent_ctes:
+                self.log(f"parent {parent_cte.name} has parents")
+                continue
+            raw_root = parent_cte.source.datasources[0]
+            if not isinstance(raw_root, Datasource):
+                self.log(f"parent {parent_cte.name} is not datasource")
+                continue
+            root: Datasource = raw_root
+            if not root.can_be_inlined:
+                self.log(f"parent {parent_cte.name} datasource is not inlineable")
+                continue
+            root_outputs = {x.address for x in root.output_concepts}
+            cte_outputs = {x.address for x in parent_cte.output_columns}
+            grain_components = {x.address for x in root.grain.components}
+            if not cte_outputs.issubset(root_outputs):
+                self.log(f"Not all {parent_cte.name} outputs are found on datasource")
+                continue
+            if not grain_components.issubset(cte_outputs):
+                self.log("Not all datasource components in cte outputs, forcing group")
+                force_group = True
+            to_inline.append(parent_cte)
+
+        for replaceable in to_inline:
+
+            result = cte.inline_parent_datasource(replaceable, force_group=force_group)
+            if result:
+                self.log(f"Inlined parent {replaceable.name}")
+            else:
+                self.log(f"Failed to inline {replaceable.name}")
+        return optimized

trilogy/core/optimizations/predicate_pushdown.py

@@ -0,0 +1,88 @@
+from trilogy.core.models import (
+    CTE,
+    Conditional,
+    BooleanOperator,
+)
+from trilogy.constants import logger
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+def decompose_condition(conditional: Conditional):
+    chunks = []
+    if conditional.operator == BooleanOperator.AND:
+        for val in [conditional.left, conditional.right]:
+            if isinstance(val, Conditional):
+                chunks.extend(decompose_condition(val))
+            else:
+                chunks.append(val)
+    else:
+        chunks.append(conditional)
+    return chunks
+
+
+def is_child_of(a, comparison):
+    if isinstance(comparison, Conditional):
+        return (
+            is_child_of(a, comparison.left) or is_child_of(a, comparison.right)
+        ) and comparison.operator == BooleanOperator.AND
+    return comparison == a
+
+
+class PredicatePushdown(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+
+            return False
+
+        optimized = False
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+        self.log(
+            f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
+        )
+        if isinstance(cte.condition, Conditional):
+            candidates = cte.condition.decompose()
+        else:
+            candidates = [cte.condition]
+        logger.info(f"Have {len(candidates)} candidates to try to push down")
+        for candidate in candidates:
+            conditions = {x.address for x in candidate.concept_arguments}
+            for parent_cte in cte.parent_ctes:
+                materialized = {k for k, v in parent_cte.source_map.items() if v != []}
+                if conditions.issubset(materialized):
+                    if all(
+                        [
+                            is_child_of(candidate, child.condition)
+                            for child in inverse_map[parent_cte.name]
+                        ]
+                    ):
+                        self.log(
+                            f"All concepts are found on {parent_cte.name} and all it's children include same filter; pushing up filter"
+                        )
+                        if parent_cte.condition:
+                            parent_cte.condition = Conditional(
+                                left=parent_cte.condition,
+                                operator=BooleanOperator.AND,
+                                right=candidate,
+                            )
+                        else:
+                            parent_cte.condition = candidate
+                        optimized = True
+                else:
+                    logger.info("conditions not subset of parent materialized")
+
+        if all(
+            [
+                is_child_of(cte.condition, parent_cte.condition)
+                for parent_cte in cte.parent_ctes
+            ]
+        ):
+            self.log("All parents have same filter, removing filter")
+            cte.condition = None
+            optimized = True
+
+        return optimized

trilogy/dialect/base.py

@@ -44,6 +44,8 @@ from trilogy.core.models import (
     RowsetDerivationStatement,
     ConceptDeclarationStatement,
     ImportStatement,
+    RawSQLStatement,
+    ProcessedRawSQLStatement,
 )
 from trilogy.core.query_processor import process_query, process_persist
 from trilogy.dialect.common import render_join
@@ -558,11 +560,20 @@ class BaseDialect:
             | RowsetDerivationStatement
             | MergeStatement
             | ImportStatement
+            | RawSQLStatement
         ],
         hooks: Optional[List[BaseHook]] = None,
-    ) -> List[ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement]:
+    ) -> List[
+        ProcessedQuery
+        | ProcessedQueryPersist
+        | ProcessedShowStatement
+        | ProcessedRawSQLStatement
+    ]:
         output: List[
-            ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement
+            ProcessedQuery
+            | ProcessedQueryPersist
+            | ProcessedShowStatement
+            | ProcessedRawSQLStatement
         ] = []
         for statement in statements:
             if isinstance(statement, PersistStatement):
@@ -604,6 +615,8 @@ class BaseDialect:
                     )
                 else:
                     raise NotImplementedError(type(statement))
+            elif isinstance(statement, RawSQLStatement):
+                output.append(ProcessedRawSQLStatement(text=statement.text))
             elif isinstance(
                 statement,
                 (
@@ -619,10 +632,18 @@ class BaseDialect:
         return output
 
     def compile_statement(
-        self, query: ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement
+        self,
+        query: (
+            ProcessedQuery
+            | ProcessedQueryPersist
+            | ProcessedShowStatement
+            | ProcessedRawSQLStatement
+        ),
     ) -> str:
         if isinstance(query, ProcessedShowStatement):
             return ";\n".join([str(x) for x in query.output_values])
+        elif isinstance(query, ProcessedRawSQLStatement):
+            return query.text
         select_columns: Dict[str, str] = {}
         cte_output_map = {}
         selected = set()

trilogy/dialect/sql_server.py

@@ -9,6 +9,7 @@ from trilogy.core.models import (
     ProcessedQuery,
     ProcessedQueryPersist,
     ProcessedShowStatement,
+    ProcessedRawSQLStatement,
 )
 from trilogy.dialect.base import BaseDialect
 
@@ -81,7 +82,13 @@ class SqlServerDialect(BaseDialect):
     SQL_TEMPLATE = TSQL_TEMPLATE
 
     def compile_statement(
-        self, query: ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement
+        self,
+        query: (
+            ProcessedQuery
+            | ProcessedQueryPersist
+            | ProcessedShowStatement
+            | ProcessedRawSQLStatement
+        ),
     ) -> str:
         base = super().compile_statement(query)
         if isinstance(base, (ProcessedQuery, ProcessedQueryPersist)):

trilogy/executor.py

@@ -9,6 +9,8 @@ from trilogy.core.models import (
     ProcessedQuery,
     ProcessedShowStatement,
     ProcessedQueryPersist,
+    ProcessedRawSQLStatement,
+    RawSQLStatement,
     MultiSelectStatement,
     SelectStatement,
     PersistStatement,
@@ -112,6 +114,10 @@ class Executor(object):
         )
         return self.execute_query(sql[0])
 
+    @execute_query.register
+    def _(self, query: RawSQLStatement) -> CursorResult:
+        return self.execute_raw_sql(query.text)
+
     @execute_query.register
     def _(self, query: ProcessedShowStatement) -> CursorResult:
         return generate_result_set(
@@ -123,6 +129,10 @@ class Executor(object):
             ],
         )
 
+    @execute_query.register
+    def _(self, query: ProcessedRawSQLStatement) -> CursorResult:
+        return self.execute_raw_sql(query.text)
+
     @execute_query.register
     def _(self, query: ProcessedQuery) -> CursorResult:
         sql = self.generator.compile_statement(query)
@@ -195,7 +205,12 @@ class Executor(object):
 
     def parse_text(
         self, command: str, persist: bool = False
-    ) -> List[ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement]:
+    ) -> List[
+        ProcessedQuery
+        | ProcessedQueryPersist
+        | ProcessedShowStatement
+        | ProcessedRawSQLStatement
+    ]:
         """Process a preql text command"""
         _, parsed = parse_text(command, self.environment)
         generatable = [
@@ -208,6 +223,7 @@ class Executor(object):
                     PersistStatement,
                     MultiSelectStatement,
                     ShowStatement,
+                    RawSQLStatement,
                 ),
             )
         ]
@@ -222,10 +238,13 @@ class Executor(object):
             sql.append(x)
         return sql
 
-    def parse_text_generator(
-        self, command: str, persist: bool = False
-    ) -> Generator[
-        ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement, None, None
+    def parse_text_generator(self, command: str, persist: bool = False) -> Generator[
+        ProcessedQuery
+        | ProcessedQueryPersist
+        | ProcessedShowStatement
+        | ProcessedRawSQLStatement,
+        None,
+        None,
     ]:
         """Process a preql text command"""
         _, parsed = parse_text(command, self.environment)
@@ -239,6 +258,7 @@ class Executor(object):
                     PersistStatement,
                     MultiSelectStatement,
                     ShowStatement,
+                    RawSQLStatement,
                 ),
             )
         ]

trilogy/parsing/parse_engine.py

@@ -82,6 +82,7 @@ from trilogy.core.models import (
     Parenthetical,
     PersistStatement,
     Query,
+    RawSQLStatement,
     SelectStatement,
     SelectItem,
     WhereClause,
@@ -585,8 +586,11 @@ class ParseToObjects(Transformer):
         #            namespace=self.environment.namespace,
         return Grain(components=[self.environment.concepts[a] for a in args[0]])
 
+    def MULTILINE_STRING(self, args) -> str:
+        return args[3:-3]
+
     def raw_column_assignment(self, args):
-        return RawColumnExpr(text=args[0][3:-3])
+        return RawColumnExpr(text=args[0])
 
     @v_args(meta=True)
     def datasource(self, meta: Meta, args):
@@ -756,6 +760,10 @@ class ParseToObjects(Transformer):
         self.environment.add_concept(new, meta=meta)
         return merge
 
+    @v_args(meta=True)
+    def rawsql_statement(self, meta: Meta, args) -> RawSQLStatement:
+        return RawSQLStatement(meta=Metadata(line_number=meta.line), text=args[0])
+
     def import_statement(self, args: list[str]) -> ImportStatement:
         alias = args[-1]
         path = args[0].split(".")
@@ -822,7 +830,11 @@ class ParseToObjects(Transformer):
             address=Address(location=address),
             grain=grain,
         )
-        return PersistStatement(select=select, datasource=new_datasource)
+        return PersistStatement(
+            select=select,
+            datasource=new_datasource,
+            meta=Metadata(line_number=meta.line),
+        )
 
     @v_args(meta=True)
     def align_item(self, meta: Meta, args) -> AlignItem:
@@ -864,6 +876,7 @@ class ParseToObjects(Transformer):
             where_clause=where,
             order_by=order_by,
             limit=limit,
+            meta=Metadata(line_number=meta.line),
         )
         for concept in multi.derived_concepts:
             self.environment.add_concept(concept, meta=meta)
@@ -887,7 +900,11 @@ class ParseToObjects(Transformer):
         if not select_items:
             raise ValueError("Malformed select, missing select items")
         output = SelectStatement(
-            selection=select_items, where_clause=where, limit=limit, order_by=order_by
+            selection=select_items,
+            where_clause=where,
+            limit=limit,
+            order_by=order_by,
+            meta=Metadata(line_number=meta.line),
         )
         for item in select_items:
             # we don't know the grain of an aggregate at assignment time
@@ -912,7 +929,7 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def query(self, meta: Meta, args):
-        return Query(text=args[0][3:-3])
+        return Query(text=args[0])
 
     def where(self, args):
         root = args[0]
@@ -1692,6 +1709,7 @@ def parse_text(text: str, environment: Optional[Environment] = None) -> Tuple[
         | SelectStatement
         | PersistStatement
         | ShowStatement
+        | RawSQLStatement
         | None
     ],
 ]:

trilogy/parsing/render.py

@@ -38,6 +38,7 @@ from trilogy.core.models import (
     OrderBy,
     AlignClause,
     AlignItem,
+    RawSQLStatement,
 )
 from trilogy.core.enums import Modifier
 
@@ -183,6 +184,10 @@ class Renderer:
     def _(self, arg: "Address"):
         return f"address {arg.location}"
 
+    @to_string.register
+    def _(self, arg: "RawSQLStatement"):
+        return f"raw_sql('''{arg.text}''');"
+
     @to_string.register
     def _(self, arg: "MagicConstants"):
         if arg == MagicConstants.NULL:

trilogy/parsing/trilogy.lark

@@ -9,6 +9,7 @@
     | rowset_derivation_statement
     | import_statement
     | merge_statement
+    | rawsql_statement
     
     _TERMINATOR:  ";"i /\s*/
     
@@ -71,6 +72,9 @@
     // merge statemment
     merge_statement: "merge" IDENTIFIER ("," IDENTIFIER)* ","? 
 
+    // raw sql statement
+    rawsql_statement: "raw_sql"i "(" MULTILINE_STRING ")"
+
     // FUNCTION blocks
     function: raw_function
     function_binding_item: IDENTIFIER ":" data_type