pytrilogy 0.0.1.115__py3-none-any.whl → 0.0.1.117__py3-none-any.whl

{pytrilogy-0.0.1.115.dist-info → pytrilogy-0.0.1.117.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.115
+Version: 0.0.1.117
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -15,17 +15,17 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: lark
 Requires-Dist: jinja2
-Requires-Dist: sqlalchemy <2.0.0
+Requires-Dist: sqlalchemy<2.0.0
 Requires-Dist: networkx
 Requires-Dist: pyodbc
 Requires-Dist: pydantic
 Requires-Dist: duckdb-engine
 Provides-Extra: bigquery
-Requires-Dist: sqlalchemy-bigquery ; extra == 'bigquery'
+Requires-Dist: sqlalchemy-bigquery; extra == "bigquery"
 Provides-Extra: postgres
-Requires-Dist: psycopg2-binary ; extra == 'postgres'
+Requires-Dist: psycopg2-binary; extra == "postgres"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-sqlalchemy ; extra == 'snowflake'
+Requires-Dist: snowflake-sqlalchemy; extra == "snowflake"
 
 ## Trilogy
 [![Website](https://img.shields.io/badge/INTRO-WEB-orange?)](https://trilogydata.dev/)

{pytrilogy-0.0.1.115.dist-info → pytrilogy-0.0.1.117.dist-info}/RECORD

@@ -1,8 +1,8 @@
-trilogy/__init__.py,sha256=FLkaJwdfgAlanF07q1NrFtMnJiCyOuGWPWbUpfXnttg,292
+trilogy/__init__.py,sha256=1wHuIjygzuHzrQfHKZQ2GJsmw4tMvEXzwEJMg9Lb3Zc,292
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=DJi3ESttmvqgy6fPRXiaQzqJVye6jYwf6XM89NHv0_M,735
+trilogy/constants.py,sha256=u2dNxhwy0v-6HrvG1GcpDVvuhzdTH5fuyYNCxDPlr2E,770
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
-trilogy/executor.py,sha256=_ZbjrKsUdWL52tWgpxqZnmccAuPXcIPEPN_dDSLNeAQ,9696
+trilogy/executor.py,sha256=auuDykCHeqlRWIHOfBfgIIIntEctWaUC-VPJr1DQbYk,10217
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
@@ -13,12 +13,17 @@ trilogy/core/env_processor.py,sha256=SU-jpaGfoWLe9sGTeQYG1qjVnwGQ7TwctmnJRlfzluc
 trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
 trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
-trilogy/core/functions.py,sha256=zkRReytiotOBAW-a3Ri5eoejZDYTt2-7Op80ZxZxUmw,9129
+trilogy/core/functions.py,sha256=hXp-b29w3vNHQHpTU-VPXJqJaLferNwa681xQ3pf8R0,9129
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=2qkebdlVsHeGp1foupR6LJ5q9YewlZcZq6utxdLBqIw,111565
-trilogy/core/optimization.py,sha256=942MnGRzscAHcG9LsfMslIRRQBslbIiPHnAvJ3w8YRg,9157
+trilogy/core/models.py,sha256=U8gYAvRoob4uj3f-j3N5EwkbBs_tKmcA48IGwmhqrbM,114203
+trilogy/core/optimization.py,sha256=oM3Ry7UpbpTSm2xNkmWx70OHd2V2vWRjM72sZpsZfb8,4116
 trilogy/core/query_processor.py,sha256=clIRJ6IcsqIVBPKFsxt8bqCLsLyajvAu02MUIcKQhTo,15713
+trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
+trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
+trilogy/core/optimizations/inline_constant.py,sha256=neZOFjX7M2pzQ-8m-f8nApy_MfJuowX6SzcGwGFt5w4,927
+trilogy/core/optimizations/inline_datasource.py,sha256=BSp54fwF4RRwInd-09pggemC7JuXj-uqGzi32ufeqYo,2171
+trilogy/core/optimizations/predicate_pushdown.py,sha256=sIojWvoYp6k_ANCyVqxCpEyLY_GLmzsG-Sghj0cbk3k,4135
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/concept_strategies_v3.py,sha256=MYrpNMidqvPOg123RekOcqVTjcj03i_538gBo0MzoWE,23432
 trilogy/core/processing/graph_utils.py,sha256=ulCJ4hYAISbUxLD6VM2fah9RBPGIXSEHEPeRBSFl0Rs,1197
@@ -45,16 +50,16 @@ trilogy/core/processing/nodes/select_node_v2.py,sha256=ERCflBFzKpD5SzweMevnJLyQn
 trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=iy2jb43CvJ0xSozyNVLuJSCaZ4Uzb5o--2p2-ZKNzSM,30991
+trilogy/dialect/base.py,sha256=ycsxbUL68DsodOsHEgEoNLKMn5vgRN3sDIRfiH9fQDs,31719
 trilogy/dialect/bigquery.py,sha256=9vxQn2BMv_oTGQSWQpoN5ho_OgqMWaHH9e-5vQVf44c,2906
 trilogy/dialect/common.py,sha256=zWrYmvevlXznocw9uGHmY5Ws1rp_kICm9zA_ulTe4eg,2165
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
 trilogy/dialect/duckdb.py,sha256=Ddyt68sr8IL2HnZMenyytoD65FXwY_O2pz1McyS0bis,3075
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
 trilogy/dialect/postgres.py,sha256=r47xbCA7nfEYENofiVfLZ-SnReNfDmUmW4OSHVkkP4E,3206
-trilogy/dialect/presto.py,sha256=8zjRn8AeYXZQGuUi-afyBWLet8o-LSt6gm5IH7bTdiw,2987
+trilogy/dialect/presto.py,sha256=UxBodRiV3szpFcQlcjoJaGXEwAhZJf_OT7dHczYvO80,3092
 trilogy/dialect/snowflake.py,sha256=N3HknYgN-fjD7BLX1Ucj-ss_ku2Ox8DgLsF3BIHutHo,2941
-trilogy/dialect/sql_server.py,sha256=UrLeA9bxiFJ4qpGsqVJqBybQCyJhetMebe8IzQW1q9s,2900
+trilogy/dialect/sql_server.py,sha256=HX68vNTrcDaTnOxe6Zbx_PBgrO42e2VuThxO6CYQ2cY,3026
 trilogy/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/hooks/base_hook.py,sha256=Xkb-A2qCHozYjum0A36zOy5PwTVwrP3NLDF0U2GpgHo,1100
 trilogy/hooks/graph_hook.py,sha256=i-Tv9sxZU0sMc-God8bLLz-nAg4-wYafogZtHaU8LXw,801
@@ -65,14 +70,14 @@ trilogy/parsing/common.py,sha256=iR3fiiZ7w8VJuUGrQ0v06XGDXov81f4z1ZlFnj6y40E,580
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=9j0mUN6D48iH39YRPRTp5JsDaB9YLFaF-xmF62ALMSA,56256
-trilogy/parsing/render.py,sha256=fxjpq2FZLgllw_d4cru-t_IXNPAz2DmYkT7v9ED0XRI,11540
-trilogy/parsing/trilogy.lark,sha256=GpjGQaDmK7GOTIqK1n4lMbDXqpt_w3EkqlYhlFw3HEA,10679
+trilogy/parsing/parse_engine.py,sha256=F1ok96qT6EhKRKV1Q_YzfHxMFtNV8qAXopK8NaePgU4,57080
+trilogy/parsing/render.py,sha256=TnLf5fg4wimpd9EvhLU-FMDwpyW9pesoedBZ0RrmWD4,11810
+trilogy/parsing/trilogy.lark,sha256=1AZbQGpNmpm4KamAXA5IWcuOr2B8Gb8kUJcAOmKf_zY,10862
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.1.115.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.1.115.dist-info/METADATA,sha256=6lmw6TXDHhgLTQoHRIqZxy3YK7oq0hQPMtV9Z9wzk0g,7882
-pytrilogy-0.0.1.115.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-pytrilogy-0.0.1.115.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.1.115.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.1.115.dist-info/RECORD,,
+pytrilogy-0.0.1.117.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.1.117.dist-info/METADATA,sha256=jrHvRWl_dtmpVu_aw08SS-whIymZ6l051tcpYmPQPD0,7878
+pytrilogy-0.0.1.117.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+pytrilogy-0.0.1.117.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.1.117.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.1.117.dist-info/RECORD,,

{pytrilogy-0.0.1.115.dist-info → pytrilogy-0.0.1.117.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (71.1.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.1.115"
+__version__ = "0.0.1.117"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -22,6 +22,7 @@ NULL_VALUE = MagicConstants.NULL
 class Optimizations:
     predicate_pushdown: bool = True
     datasource_inlining: bool = True
+    constant_inlining: bool = True
     direct_return: bool = True
 
 

trilogy/core/functions.py

@@ -9,6 +9,7 @@ from trilogy.core.models import (
     ListType,
     StructType,
     MapType,
+    NumericType,
 )
 from trilogy.core.enums import FunctionType, Purpose, Granularity, DatePart
 from trilogy.core.exceptions import InvalidSyntaxException
@@ -21,7 +22,9 @@ def create_function_derived_concept(
     namespace: str,
     operator: FunctionType,
     arguments: list[Concept],
-    output_type: Optional[DataType | ListType | StructType | MapType] = None,
+    output_type: Optional[
+        DataType | ListType | StructType | MapType | NumericType
+    ] = None,
     output_purpose: Optional[Purpose] = None,
 ) -> Concept:
     purpose = (
@@ -56,9 +59,7 @@ def argument_to_purpose(arg) -> Purpose:
         return Purpose.PROPERTY
     elif isinstance(arg, Concept):
         return arg.purpose
-    elif isinstance(arg, (int, float, str, bool, list)):
-        return Purpose.CONSTANT
-    elif isinstance(arg, DataType):
+    elif isinstance(arg, (int, float, str, bool, list, NumericType, DataType)):
         return Purpose.CONSTANT
     elif isinstance(arg, DatePart):
         return Purpose.CONSTANT

trilogy/core/models.py

@@ -105,7 +105,9 @@ def get_concept_arguments(expr) -> List["Concept"]:
     return output
 
 
-ALL_TYPES = Union["DataType", "MapType", "ListType", "StructType", "Concept"]
+ALL_TYPES = Union[
+    "DataType", "MapType", "ListType", "NumericType", "StructType", "Concept"
+]
 
 NAMESPACED_TYPES = Union[
     "WindowItem",
@@ -176,6 +178,19 @@ class DataType(Enum):
         return self
 
 
+class NumericType(BaseModel):
+    precision: int = 20
+    scale: int = 5
+
+    @property
+    def data_type(self):
+        return DataType.NUMERIC
+
+    @property
+    def value(self):
+        return self.data_type.value
+
+
 class ListType(BaseModel):
     model_config = ConfigDict(frozen=True)
     type: ALL_TYPES
@@ -192,7 +207,9 @@ class ListType(BaseModel):
         return self.data_type.value
 
     @property
-    def value_data_type(self) -> DataType | StructType | MapType | ListType:
+    def value_data_type(
+        self,
+    ) -> DataType | StructType | MapType | ListType | NumericType:
         if isinstance(self.type, Concept):
             return self.type.datatype
         return self.type
@@ -270,7 +287,7 @@ def empty_grain() -> Grain:
 
 class Concept(Namespaced, SelectGrain, BaseModel):
     name: str
-    datatype: DataType | ListType | StructType | MapType
+    datatype: DataType | ListType | StructType | MapType | NumericType
     purpose: Purpose
     metadata: Optional[Metadata] = Field(
         default_factory=lambda: Metadata(description=None, line_number=None),
@@ -790,12 +807,12 @@ class LooseConceptList(BaseModel):
 class Function(Namespaced, SelectGrain, BaseModel):
     operator: FunctionType
     arg_count: int = Field(default=1)
-    output_datatype: DataType | ListType | StructType | MapType
+    output_datatype: DataType | ListType | StructType | MapType | NumericType
     output_purpose: Purpose
     valid_inputs: Optional[
         Union[
-            Set[DataType | ListType | StructType],
-            List[Set[DataType | ListType | StructType]],
+            Set[DataType | ListType | StructType | NumericType],
+            List[Set[DataType | ListType | StructType] | NumericType],
         ]
     ] = None
     arguments: Sequence[
@@ -808,6 +825,7 @@ class Function(Namespaced, SelectGrain, BaseModel):
             str,
             DataType,
             ListType,
+            NumericType,
             DatePart,
             "Parenthetical",
             CaseWhen,
@@ -1165,11 +1183,17 @@ class OrderBy(Namespaced, BaseModel):
         return OrderBy(items=[x.with_namespace(namespace) for x in self.items])
 
 
+class RawSQLStatement(BaseModel):
+    text: str
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
+
+
 class SelectStatement(Namespaced, BaseModel):
     selection: List[SelectItem]
     where_clause: Optional["WhereClause"] = None
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     def __str__(self):
         from trilogy.parsing.render import render_query
@@ -1371,6 +1395,7 @@ class MultiSelectStatement(Namespaced, BaseModel):
     where_clause: Optional["WhereClause"] = None
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     def __repr__(self):
         return "MultiSelect<" + " MERGE ".join([str(s) for s in self.selects]) + ">"
@@ -1492,7 +1517,7 @@ class DatasourceMetadata(BaseModel):
 
 class MergeStatement(Namespaced, BaseModel):
     concepts: List[Concept]
-    datatype: DataType | ListType | StructType | MapType
+    datatype: DataType | ListType | StructType | MapType | NumericType
 
     @cached_property
     def concepts_lcl(self):
@@ -2038,6 +2063,40 @@ class CTE(BaseModel):
     def validate_output_columns(cls, v):
         return unique(v, "address")
 
+    def inline_constant(self, concept: Concept):
+        if not concept.derivation == PurposeLineage.CONSTANT:
+            return False
+        if not isinstance(concept.lineage, Function):
+            return False
+        if not concept.lineage.operator == FunctionType.CONSTANT:
+            return False
+        # remove the constant
+        removed: set = set()
+        if concept.address in self.source_map:
+            removed = removed.union(self.source_map[concept.address])
+            del self.source_map[concept.address]
+        # if we've entirely removed the need to join to someplace to get the concept
+        # drop the join as well.
+        for removed_cte in removed:
+            still_required = any([removed_cte in x for x in self.source_map.values()])
+            if not still_required:
+                self.joins = [
+                    join
+                    for join in self.joins
+                    if not isinstance(join, Join)
+                    or (
+                        join.right_cte.name != removed_cte
+                        and join.left_cte.name != removed_cte
+                    )
+                ]
+                self.parent_ctes = [
+                    x for x in self.parent_ctes if x.name != removed_cte
+                ]
+                if removed_cte == self.base_name_override:
+                    candidates = [x.name for x in self.parent_ctes]
+                    self.base_name_override = candidates[0] if candidates else None
+                    self.base_alias_override = candidates[0] if candidates else None
+
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
         qds_being_inlined = parent.source
         ds_being_inlined = qds_being_inlined.datasources[0]
@@ -2186,6 +2245,11 @@ class CTE(BaseModel):
             and not self.group_to_grain
         ):
             return False
+        # if we don't need to source any concepts from anywhere
+        # render without from
+        # most likely to happen from inlining constants
+        if not any([v for v in self.source_map.values()]):
+            return False
         if (
             len(self.source.datasources) == 1
             and self.source.datasources[0].name == CONSTANT_DATASET
@@ -3184,6 +3248,10 @@ class ProcessedShowStatement(BaseModel):
     output_values: List[Union[Concept, Datasource, ProcessedQuery]]
 
 
+class ProcessedRawSQLStatement(BaseModel):
+    text: str
+
+
 class Limit(BaseModel):
     count: int
 
@@ -3386,6 +3454,7 @@ class Parenthetical(ConceptArgs, Namespaced, SelectGrain, BaseModel):
 class PersistStatement(BaseModel):
     datasource: Datasource
     select: SelectStatement
+    meta: Optional[Metadata] = Field(default_factory=lambda: Metadata())
 
     @property
     def identifier(self):
@@ -3447,7 +3516,7 @@ def list_to_wrapper(args):
     return ListWrapper(args, type=types[0])
 
 
-def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType:
+def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType | NumericType:
     if isinstance(arg, Function):
         return arg.output_datatype
     elif isinstance(arg, Concept):
@@ -3460,6 +3529,8 @@ def arg_to_datatype(arg) -> DataType | ListType | StructType | MapType:
         return DataType.STRING
     elif isinstance(arg, float):
         return DataType.FLOAT
+    elif isinstance(arg, NumericType):
+        return arg
     elif isinstance(arg, ListWrapper):
         return ListType(type=arg.type)
     elif isinstance(arg, AggregateWrapper):

trilogy/core/optimization.py

@@ -2,155 +2,21 @@ from trilogy.core.models import (
     CTE,
     SelectStatement,
     PersistStatement,
-    Datasource,
     MultiSelectStatement,
     Conditional,
     BooleanOperator,
 )
 from trilogy.core.enums import PurposeLineage
 from trilogy.constants import logger, CONFIG
-from abc import ABC
-
-
-class OptimizationRule(ABC):
-
-    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
-        raise NotImplementedError
-
-    def log(self, message: str):
-        logger.info(f"[Optimization][{self.__class__.__name__}] {message}")
-
-    def debug(self, message: str):
-        logger.debug(f"[Optimization][{self.__class__.__name__}] {message}")
-
-
-class InlineDatasource(OptimizationRule):
-
-    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
-        if not cte.parent_ctes:
-            return False
-
-        optimized = False
-        self.log(
-            f"Checking {cte.name} for consolidating inline tables with {len(cte.parent_ctes)} parents"
-        )
-        to_inline: list[CTE] = []
-        force_group = False
-        for parent_cte in cte.parent_ctes:
-            if not parent_cte.is_root_datasource:
-                self.log(f"parent {parent_cte.name} is not root")
-                continue
-            if parent_cte.parent_ctes:
-                self.log(f"parent {parent_cte.name} has parents")
-                continue
-            raw_root = parent_cte.source.datasources[0]
-            if not isinstance(raw_root, Datasource):
-                self.log(f"parent {parent_cte.name} is not datasource")
-                continue
-            root: Datasource = raw_root
-            if not root.can_be_inlined:
-                self.log(f"parent {parent_cte.name} datasource is not inlineable")
-                continue
-            root_outputs = {x.address for x in root.output_concepts}
-            cte_outputs = {x.address for x in parent_cte.output_columns}
-            grain_components = {x.address for x in root.grain.components}
-            if not cte_outputs.issubset(root_outputs):
-                self.log(f"Not all {parent_cte.name} outputs are found on datasource")
-                continue
-            if not grain_components.issubset(cte_outputs):
-                self.log("Not all datasource components in cte outputs, forcing group")
-                force_group = True
-            to_inline.append(parent_cte)
-
-        for replaceable in to_inline:
-
-            result = cte.inline_parent_datasource(replaceable, force_group=force_group)
-            if result:
-                self.log(f"Inlined parent {replaceable.name}")
-            else:
-                self.log(f"Failed to inline {replaceable.name}")
-        return optimized
-
-
-def decompose_condition(conditional: Conditional):
-    chunks = []
-    if conditional.operator == BooleanOperator.AND:
-        for val in [conditional.left, conditional.right]:
-            if isinstance(val, Conditional):
-                chunks.extend(decompose_condition(val))
-            else:
-                chunks.append(val)
-    else:
-        chunks.append(conditional)
-    return chunks
-
-
-def is_child_of(a, comparison):
-    if isinstance(comparison, Conditional):
-        return (
-            is_child_of(a, comparison.left) or is_child_of(a, comparison.right)
-        ) and comparison.operator == BooleanOperator.AND
-    return comparison == a
-
-
-class PredicatePushdown(OptimizationRule):
-
-    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
-
-        if not cte.parent_ctes:
-            self.debug(f"No parent CTEs for {cte.name}")
-
-            return False
-
-        optimized = False
-        if not cte.condition:
-            self.debug(f"No CTE condition for {cte.name}")
-            return False
-        self.log(
-            f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
-        )
-        if isinstance(cte.condition, Conditional):
-            candidates = cte.condition.decompose()
-        else:
-            candidates = [cte.condition]
-        logger.info(f"Have {len(candidates)} candidates to try to push down")
-        for candidate in candidates:
-            conditions = {x.address for x in candidate.concept_arguments}
-            for parent_cte in cte.parent_ctes:
-                materialized = {k for k, v in parent_cte.source_map.items() if v != []}
-                if conditions.issubset(materialized):
-                    if all(
-                        [
-                            is_child_of(candidate, child.condition)
-                            for child in inverse_map[parent_cte.name]
-                        ]
-                    ):
-                        self.log(
-                            f"All concepts are found on {parent_cte.name} and all it's children include same filter; pushing up filter"
-                        )
-                        if parent_cte.condition:
-                            parent_cte.condition = Conditional(
-                                left=parent_cte.condition,
-                                operator=BooleanOperator.AND,
-                                right=candidate,
-                            )
-                        else:
-                            parent_cte.condition = candidate
-                        optimized = True
-                else:
-                    logger.info("conditions not subset of parent materialized")
+from trilogy.core.optimizations import (
+    OptimizationRule,
+    InlineConstant,
+    PredicatePushdown,
+    InlineDatasource,
+)
 
-        if all(
-            [
-                is_child_of(cte.condition, parent_cte.condition)
-                for parent_cte in cte.parent_ctes
-            ]
-        ):
-            self.log("All parents have same filter, removing filter")
-            cte.condition = None
-            optimized = True
 
-        return optimized
+MAX_OPTIMIZATION_LOOPS = 100
 
 
 def filter_irrelevant_ctes(
@@ -184,7 +50,9 @@ def is_direct_return_eligible(
     if isinstance(select, (PersistStatement, MultiSelectStatement)):
         return False
     derived_concepts = [
-        c for c in cte.source.output_concepts if c not in cte.source.input_concepts
+        c
+        for c in cte.source.output_concepts + cte.source.hidden_concepts
+        if c not in cte.source.input_concepts
     ]
     eligible = True
     conditions = (
@@ -224,23 +92,9 @@ def sort_select_output(cte: CTE, query: SelectStatement | MultiSelectStatement):
 
 def optimize_ctes(
     input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
-):
+) -> list[CTE]:
     complete = False
     REGISTERED_RULES: list["OptimizationRule"] = []
-
-    if CONFIG.optimizations.datasource_inlining:
-        REGISTERED_RULES.append(InlineDatasource())
-    if CONFIG.optimizations.predicate_pushdown:
-        REGISTERED_RULES.append(PredicatePushdown())
-
-    while not complete:
-        actions_taken = False
-        for rule in REGISTERED_RULES:
-            for cte in input:
-                inverse_map = gen_inverse_map(input)
-                actions_taken = rule.optimize(cte, inverse_map)
-        complete = not actions_taken
-
     if CONFIG.optimizations.direct_return and is_direct_return_eligible(
         root_cte, select
     ):
@@ -258,5 +112,20 @@ def optimize_ctes(
                 root_cte.condition = select.where_clause.conditional
         root_cte.requires_nesting = False
         sort_select_output(root_cte, select)
+    if CONFIG.optimizations.datasource_inlining:
+        REGISTERED_RULES.append(InlineDatasource())
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdown())
+    if CONFIG.optimizations.constant_inlining:
+        REGISTERED_RULES.append(InlineConstant())
+    loops = 0
+    while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
+        actions_taken = False
+        for rule in REGISTERED_RULES:
+            for cte in input:
+                inverse_map = gen_inverse_map(input)
+                actions_taken = actions_taken or rule.optimize(cte, inverse_map)
+        complete = not actions_taken
+        loops += 1
 
     return filter_irrelevant_ctes(input, root_cte)

trilogy/core/optimizations/__init__.py

@@ -0,0 +1,11 @@
+from .inline_constant import InlineConstant
+from .inline_datasource import InlineDatasource
+from .predicate_pushdown import PredicatePushdown
+from .base_optimization import OptimizationRule
+
+__all__ = [
+    "OptimizationRule",
+    "InlineConstant",
+    "InlineDatasource",
+    "PredicatePushdown",
+]

trilogy/core/optimizations/base_optimization.py

@@ -0,0 +1,17 @@
+from trilogy.core.models import (
+    CTE,
+)
+from trilogy.constants import logger
+from abc import ABC
+
+
+class OptimizationRule(ABC):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        raise NotImplementedError
+
+    def log(self, message: str):
+        logger.info(f"[Optimization][{self.__class__.__name__}] {message}")
+
+    def debug(self, message: str):
+        logger.debug(f"[Optimization][{self.__class__.__name__}] {message}")

trilogy/core/optimizations/inline_constant.py

@@ -0,0 +1,29 @@
+from trilogy.core.models import (
+    CTE,
+    Concept,
+)
+from trilogy.core.enums import PurposeLineage
+
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class InlineConstant(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+
+        to_inline: list[Concept] = []
+        for x in cte.source.input_concepts:
+            if x.address not in cte.source_map:
+                continue
+            if x.derivation == PurposeLineage.CONSTANT:
+                self.log(f"Found constant {x.address} on {cte.name}")
+                to_inline.append(x)
+        if to_inline:
+            inlined = False
+            for c in to_inline:
+                self.log(f"Inlining constant {c.address} on  {cte.name}")
+                test = cte.inline_constant(c)
+                if test:
+                    inlined = True
+            return inlined
+        return False

trilogy/core/optimizations/inline_datasource.py

@@ -0,0 +1,54 @@
+from trilogy.core.models import (
+    CTE,
+    Datasource,
+)
+
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class InlineDatasource(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        if not cte.parent_ctes:
+            return False
+
+        optimized = False
+        self.log(
+            f"Checking {cte.name} for consolidating inline tables with {len(cte.parent_ctes)} parents"
+        )
+        to_inline: list[CTE] = []
+        force_group = False
+        for parent_cte in cte.parent_ctes:
+            if not parent_cte.is_root_datasource:
+                self.log(f"parent {parent_cte.name} is not root")
+                continue
+            if parent_cte.parent_ctes:
+                self.log(f"parent {parent_cte.name} has parents")
+                continue
+            raw_root = parent_cte.source.datasources[0]
+            if not isinstance(raw_root, Datasource):
+                self.log(f"parent {parent_cte.name} is not datasource")
+                continue
+            root: Datasource = raw_root
+            if not root.can_be_inlined:
+                self.log(f"parent {parent_cte.name} datasource is not inlineable")
+                continue
+            root_outputs = {x.address for x in root.output_concepts}
+            cte_outputs = {x.address for x in parent_cte.output_columns}
+            grain_components = {x.address for x in root.grain.components}
+            if not cte_outputs.issubset(root_outputs):
+                self.log(f"Not all {parent_cte.name} outputs are found on datasource")
+                continue
+            if not grain_components.issubset(cte_outputs):
+                self.log("Not all datasource components in cte outputs, forcing group")
+                force_group = True
+            to_inline.append(parent_cte)
+
+        for replaceable in to_inline:
+
+            result = cte.inline_parent_datasource(replaceable, force_group=force_group)
+            if result:
+                self.log(f"Inlined parent {replaceable.name}")
+            else:
+                self.log(f"Failed to inline {replaceable.name}")
+        return optimized

trilogy/core/optimizations/predicate_pushdown.py

@@ -0,0 +1,105 @@
+from trilogy.core.models import (
+    CTE,
+    Conditional,
+    BooleanOperator,
+    Datasource,
+)
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+def decompose_condition(conditional: Conditional):
+    chunks = []
+    if conditional.operator == BooleanOperator.AND:
+        for val in [conditional.left, conditional.right]:
+            if isinstance(val, Conditional):
+                chunks.extend(decompose_condition(val))
+            else:
+                chunks.append(val)
+    else:
+        chunks.append(conditional)
+    return chunks
+
+
+def is_child_of(a, comparison):
+    if isinstance(comparison, Conditional):
+        return (
+            is_child_of(a, comparison.left) or is_child_of(a, comparison.right)
+        ) and comparison.operator == BooleanOperator.AND
+    return comparison == a
+
+
+class PredicatePushdown(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+
+            return False
+
+        optimized = False
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+        self.log(
+            f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
+        )
+        if isinstance(cte.condition, Conditional):
+            candidates = cte.condition.decompose()
+        else:
+            candidates = [cte.condition]
+        self.log(f"Have {len(candidates)} candidates to try to push down")
+        for candidate in candidates:
+            conditions = {x.address for x in candidate.concept_arguments}
+            for parent_cte in cte.parent_ctes:
+                if is_child_of(cte.condition, parent_cte.condition):
+                    continue
+                materialized = {k for k, v in parent_cte.source_map.items() if v != []}
+                # if it's a root datasource, we can filter on _any_ of the output concepts
+                if parent_cte.is_root_datasource:
+                    extra_check = {
+                        x.address
+                        for x in parent_cte.source.datasources[0].output_concepts
+                    }
+                    if conditions.issubset(extra_check):
+                        for x in conditions:
+                            if x not in materialized:
+                                materialized.add(x)
+                                parent_cte.source_map[x] = [
+                                    parent_cte.source.datasources[0].name
+                                ]
+                if conditions.issubset(materialized):
+                    if all(
+                        [
+                            is_child_of(candidate, child.condition)
+                            for child in inverse_map.get(parent_cte.name, [])
+                        ]
+                    ):
+                        self.log(
+                            f"All concepts are found on {parent_cte.name} and all it's children include same filter; pushing up filter"
+                        )
+                        if parent_cte.condition:
+                            parent_cte.condition = Conditional(
+                                left=parent_cte.condition,
+                                operator=BooleanOperator.AND,
+                                right=candidate,
+                            )
+                        else:
+                            parent_cte.condition = candidate
+                        optimized = True
+                else:
+                    self.log(
+                        f"conditions {conditions} not subset of parent {parent_cte.name} parent has {materialized} "
+                    )
+
+        if all(
+            [
+                is_child_of(cte.condition, parent_cte.condition)
+                for parent_cte in cte.parent_ctes
+            ]
+        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
+            self.log("All parents have same filter, removing filter")
+            cte.condition = None
+            optimized = True
+
+        return optimized

trilogy/dialect/base.py

@@ -44,6 +44,9 @@ from trilogy.core.models import (
     RowsetDerivationStatement,
     ConceptDeclarationStatement,
     ImportStatement,
+    RawSQLStatement,
+    ProcessedRawSQLStatement,
+    NumericType,
 )
 from trilogy.core.query_processor import process_query, process_persist
 from trilogy.dialect.common import render_join
@@ -95,6 +98,7 @@ DATATYPE_MAP = {
     DataType.INTEGER: "int",
     DataType.FLOAT: "float",
     DataType.BOOL: "bool",
+    DataType.NUMERIC: "numeric",
 }
 
 
@@ -332,6 +336,7 @@ class BaseDialect:
             Parenthetical,
             AggregateWrapper,
             MagicConstants,
+            NumericType,
             ListType,
             ListWrapper[int],
             ListWrapper[str],
@@ -437,6 +442,8 @@ class BaseDialect:
             return str(e.value)
         elif isinstance(e, DatePart):
             return str(e.value)
+        elif isinstance(e, NumericType):
+            return f"{self.DATATYPE_MAP[DataType.NUMERIC]}({e.precision},{e.scale})"
         elif isinstance(e, MagicConstants):
             if e == MagicConstants.NULL:
                 return "null"
@@ -558,11 +565,20 @@ class BaseDialect:
             | RowsetDerivationStatement
             | MergeStatement
             | ImportStatement
+            | RawSQLStatement
         ],
         hooks: Optional[List[BaseHook]] = None,
-    ) -> List[ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement]:
+    ) -> List[
+        ProcessedQuery
+        | ProcessedQueryPersist
+        | ProcessedShowStatement
+        | ProcessedRawSQLStatement
+    ]:
         output: List[
-            ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement
+            ProcessedQuery
+            | ProcessedQueryPersist
+            | ProcessedShowStatement
+            | ProcessedRawSQLStatement
         ] = []
         for statement in statements:
             if isinstance(statement, PersistStatement):
@@ -604,6 +620,8 @@ class BaseDialect:
                     )
                 else:
                     raise NotImplementedError(type(statement))
+            elif isinstance(statement, RawSQLStatement):
+                output.append(ProcessedRawSQLStatement(text=statement.text))
             elif isinstance(
                 statement,
                 (
@@ -619,10 +637,18 @@ class BaseDialect:
         return output
 
     def compile_statement(
-        self, query: ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement
+        self,
+        query: (
+            ProcessedQuery
+            | ProcessedQueryPersist
+            | ProcessedShowStatement
+            | ProcessedRawSQLStatement
+        ),
     ) -> str:
         if isinstance(query, ProcessedShowStatement):
             return ";\n".join([str(x) for x in query.output_values])
+        elif isinstance(query, ProcessedRawSQLStatement):
+            return query.text
         select_columns: Dict[str, str] = {}
         cte_output_map = {}
         selected = set()

trilogy/dialect/presto.py

@@ -4,6 +4,7 @@ from jinja2 import Template
 
 from trilogy.core.enums import FunctionType, WindowType
 from trilogy.dialect.base import BaseDialect
+from trilogy.core.models import DataType
 
 
 WINDOW_FUNCTION_MAP: Mapping[WindowType, Callable[[Any, Any, Any], str]] = {}
@@ -26,7 +27,7 @@ FUNCTION_MAP = {
     FunctionType.WEEK: lambda x: f"EXTRACT(WEEK from {x[0]})",
     FunctionType.QUARTER: lambda x: f"EXTRACT(QUARTER from {x[0]})",
     # math
-    FunctionType.DIVIDE: lambda x: f"SAFE_DIVIDE({x[0]},{x[1]})",
+    FunctionType.DIVIDE: lambda x: f"{x[0]}/{x[1]}",
     FunctionType.DATE_ADD: lambda x: f"DATE_ADD('{x[1]}', {x[2]}, {x[0]})",
     FunctionType.CURRENT_DATE: lambda x: "CURRENT_DATE",
     FunctionType.CURRENT_DATETIME: lambda x: "CURRENT_TIMESTAMP",
@@ -80,6 +81,7 @@ class PrestoDialect(BaseDialect):
     }
     QUOTE_CHARACTER = '"'
     SQL_TEMPLATE = SQL_TEMPLATE
+    DATATYPE_MAP = {**BaseDialect.DATATYPE_MAP, DataType.NUMERIC: "DECIMAL"}
 
 
 class TrinoDialect(PrestoDialect):

trilogy/dialect/sql_server.py

@@ -9,6 +9,7 @@ from trilogy.core.models import (
     ProcessedQuery,
     ProcessedQueryPersist,
     ProcessedShowStatement,
+    ProcessedRawSQLStatement,
 )
 from trilogy.dialect.base import BaseDialect
 
@@ -81,7 +82,13 @@ class SqlServerDialect(BaseDialect):
     SQL_TEMPLATE = TSQL_TEMPLATE
 
     def compile_statement(
-        self, query: ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement
+        self,
+        query: (
+            ProcessedQuery
+            | ProcessedQueryPersist
+            | ProcessedShowStatement
+            | ProcessedRawSQLStatement
+        ),
     ) -> str:
         base = super().compile_statement(query)
         if isinstance(base, (ProcessedQuery, ProcessedQueryPersist)):

trilogy/executor.py

@@ -9,6 +9,8 @@ from trilogy.core.models import (
     ProcessedQuery,
     ProcessedShowStatement,
     ProcessedQueryPersist,
+    ProcessedRawSQLStatement,
+    RawSQLStatement,
     MultiSelectStatement,
     SelectStatement,
     PersistStatement,
@@ -112,6 +114,10 @@ class Executor(object):
         )
         return self.execute_query(sql[0])
 
+    @execute_query.register
+    def _(self, query: RawSQLStatement) -> CursorResult:
+        return self.execute_raw_sql(query.text)
+
     @execute_query.register
     def _(self, query: ProcessedShowStatement) -> CursorResult:
         return generate_result_set(
@@ -123,6 +129,10 @@ class Executor(object):
             ],
         )
 
+    @execute_query.register
+    def _(self, query: ProcessedRawSQLStatement) -> CursorResult:
+        return self.execute_raw_sql(query.text)
+
     @execute_query.register
     def _(self, query: ProcessedQuery) -> CursorResult:
         sql = self.generator.compile_statement(query)
@@ -195,7 +205,12 @@ class Executor(object):
 
     def parse_text(
         self, command: str, persist: bool = False
-    ) -> List[ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement]:
+    ) -> List[
+        ProcessedQuery
+        | ProcessedQueryPersist
+        | ProcessedShowStatement
+        | ProcessedRawSQLStatement
+    ]:
         """Process a preql text command"""
         _, parsed = parse_text(command, self.environment)
         generatable = [
@@ -208,6 +223,7 @@ class Executor(object):
                     PersistStatement,
                     MultiSelectStatement,
                     ShowStatement,
+                    RawSQLStatement,
                 ),
             )
         ]
@@ -222,10 +238,13 @@ class Executor(object):
             sql.append(x)
         return sql
 
-    def parse_text_generator(
-        self, command: str, persist: bool = False
-    ) -> Generator[
-        ProcessedQuery | ProcessedQueryPersist | ProcessedShowStatement, None, None
+    def parse_text_generator(self, command: str, persist: bool = False) -> Generator[
+        ProcessedQuery
+        | ProcessedQueryPersist
+        | ProcessedShowStatement
+        | ProcessedRawSQLStatement,
+        None,
+        None,
     ]:
         """Process a preql text command"""
         _, parsed = parse_text(command, self.environment)
@@ -239,6 +258,7 @@ class Executor(object):
                     PersistStatement,
                     MultiSelectStatement,
                     ShowStatement,
+                    RawSQLStatement,
                 ),
             )
         ]

trilogy/parsing/parse_engine.py

@@ -82,6 +82,7 @@ from trilogy.core.models import (
     Parenthetical,
     PersistStatement,
     Query,
+    RawSQLStatement,
     SelectStatement,
     SelectItem,
     WhereClause,
@@ -102,6 +103,7 @@ from trilogy.core.models import (
     RowsetDerivationStatement,
     LooseConceptList,
     list_to_wrapper,
+    NumericType,
 )
 from trilogy.parsing.exceptions import ParseError
 from trilogy.utility import string_to_hash
@@ -308,7 +310,9 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def struct_type(self, meta: Meta, args) -> StructType:
-        final: list[DataType | MapType | ListType | StructType | Concept] = []
+        final: list[
+            DataType | MapType | ListType | StructType | NumericType | Concept
+        ] = []
         for arg in args:
             if not isinstance(arg, (DataType, ListType, StructType)):
                 new = self.environment.concepts.__getitem__(  # type: ignore
@@ -322,12 +326,17 @@ class ParseToObjects(Transformer):
     def list_type(self, args) -> ListType:
         return ListType(type=args[0])
 
-    def data_type(self, args) -> DataType | ListType | StructType:
+    def numeric_type(self, args) -> NumericType:
+        return NumericType(precision=args[0], scale=args[1])
+
+    def data_type(self, args) -> DataType | ListType | StructType | NumericType:
         resolved = args[0]
         if isinstance(resolved, StructType):
             return resolved
         elif isinstance(resolved, ListType):
             return resolved
+        elif isinstance(resolved, NumericType):
+            return resolved
         return DataType(args[0].lower())
 
     def array_comparison(self, args) -> ComparisonOperator:
@@ -585,8 +594,11 @@ class ParseToObjects(Transformer):
         #            namespace=self.environment.namespace,
         return Grain(components=[self.environment.concepts[a] for a in args[0]])
 
+    def MULTILINE_STRING(self, args) -> str:
+        return args[3:-3]
+
     def raw_column_assignment(self, args):
-        return RawColumnExpr(text=args[0][3:-3])
+        return RawColumnExpr(text=args[0])
 
     @v_args(meta=True)
     def datasource(self, meta: Meta, args):
@@ -756,6 +768,10 @@ class ParseToObjects(Transformer):
         self.environment.add_concept(new, meta=meta)
         return merge
 
+    @v_args(meta=True)
+    def rawsql_statement(self, meta: Meta, args) -> RawSQLStatement:
+        return RawSQLStatement(meta=Metadata(line_number=meta.line), text=args[0])
+
     def import_statement(self, args: list[str]) -> ImportStatement:
         alias = args[-1]
         path = args[0].split(".")
@@ -822,7 +838,11 @@ class ParseToObjects(Transformer):
             address=Address(location=address),
             grain=grain,
         )
-        return PersistStatement(select=select, datasource=new_datasource)
+        return PersistStatement(
+            select=select,
+            datasource=new_datasource,
+            meta=Metadata(line_number=meta.line),
+        )
 
     @v_args(meta=True)
     def align_item(self, meta: Meta, args) -> AlignItem:
@@ -864,6 +884,7 @@ class ParseToObjects(Transformer):
             where_clause=where,
             order_by=order_by,
             limit=limit,
+            meta=Metadata(line_number=meta.line),
         )
         for concept in multi.derived_concepts:
             self.environment.add_concept(concept, meta=meta)
@@ -887,7 +908,11 @@ class ParseToObjects(Transformer):
         if not select_items:
             raise ValueError("Malformed select, missing select items")
         output = SelectStatement(
-            selection=select_items, where_clause=where, limit=limit, order_by=order_by
+            selection=select_items,
+            where_clause=where,
+            limit=limit,
+            order_by=order_by,
+            meta=Metadata(line_number=meta.line),
         )
         for item in select_items:
             # we don't know the grain of an aggregate at assignment time
@@ -912,7 +937,7 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def query(self, meta: Meta, args):
-        return Query(text=args[0][3:-3])
+        return Query(text=args[0])
 
     def where(self, args):
         root = args[0]
@@ -1534,6 +1559,8 @@ class ParseToObjects(Transformer):
                 DataType.STRING,
                 DataType.FLOAT,
                 DataType.NUMBER,
+                DataType.NUMERIC,
+                DataType.BOOL,
             },
             arg_count=2,
         )
@@ -1692,6 +1719,7 @@ def parse_text(text: str, environment: Optional[Environment] = None) -> Tuple[
         | SelectStatement
         | PersistStatement
         | ShowStatement
+        | RawSQLStatement
         | None
     ],
 ]:

trilogy/parsing/render.py

@@ -38,6 +38,8 @@ from trilogy.core.models import (
     OrderBy,
     AlignClause,
     AlignItem,
+    RawSQLStatement,
+    NumericType,
 )
 from trilogy.core.enums import Modifier
 
@@ -171,6 +173,10 @@ class Renderer:
     def _(self, arg: DataType):
         return arg.value
 
+    @to_string.register
+    def _(self, arg: "NumericType"):
+        return f"""Numeric({arg.precision},{arg.scale})"""
+
     @to_string.register
     def _(self, arg: ListWrapper):
         return "[" + ", ".join([self.to_string(x) for x in arg]) + "]"
@@ -183,6 +189,10 @@ class Renderer:
     def _(self, arg: "Address"):
         return f"address {arg.location}"
 
+    @to_string.register
+    def _(self, arg: "RawSQLStatement"):
+        return f"raw_sql('''{arg.text}''');"
+
     @to_string.register
     def _(self, arg: "MagicConstants"):
         if arg == MagicConstants.NULL:

trilogy/parsing/trilogy.lark

@@ -9,6 +9,7 @@
     | rowset_derivation_statement
     | import_statement
     | merge_statement
+    | rawsql_statement
     
     _TERMINATOR:  ";"i /\s*/
     
@@ -71,6 +72,9 @@
     // merge statemment
     merge_statement: "merge" IDENTIFIER ("," IDENTIFIER)* ","? 
 
+    // raw sql statement
+    rawsql_statement: "raw_sql"i "(" MULTILINE_STRING ")"
+
     // FUNCTION blocks
     function: raw_function
     function_binding_item: IDENTIFIER ":" data_type
@@ -276,12 +280,14 @@
 
     SHORTHAND_MODIFIER: "~"
 
-    struct_type: "struct" "<" ((data_type | IDENTIFIER) ",")* (data_type | IDENTIFIER) ","? ">" 
+    struct_type: "struct"i "<" ((data_type | IDENTIFIER) ",")* (data_type | IDENTIFIER) ","? ">" 
+
+    list_type: "list"i "<" data_type ">"
 
-    list_type: "list" "<" data_type ">"
+    numeric_type: "numeric"i "(" int_lit "," int_lit ")"
 
 
-    !data_type: "string"i | "number"i | "numeric"i | "map"i | "list"i | "array"i | "any"i | "int"i | "bigint" | "date"i | "datetime"i | "timestamp"i | "float"i | "bool"i | struct_type | list_type
+    !data_type: "string"i | "number"i | "numeric"i | "map"i | "list"i | "array"i | "any"i | "int"i | "bigint"i | "date"i | "datetime"i | "timestamp"i | "float"i | "bool"i | numeric_type | struct_type | list_type
     
     PURPOSE:  "key"i | "metric"i  | CONST
     PROPERTY: "property"i