pytrilogy 0.0.1.111__tar.gz → 0.0.1.113__tar.gz

{pytrilogy-0.0.1.111/pytrilogy.egg-info → pytrilogy-0.0.1.113}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.111
+Version: 0.0.1.113
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.111
+Version: 0.0.1.113
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113}/tests/test_declarations.py

@@ -4,7 +4,7 @@ from trilogy.parser import parse
 def test_declarations():
     declarations = """key namespace.user_id int metadata(description="the description");
     metric namespace.count <- count(namespace.user_id);
-    metric namespace.distinct_count <- count_distinct(namespace.user_id); # the distinct count of user ids
+    metric namespace.distinct_count <- count_distinct(namespace.user_id); #the distinct count of user ids
     """
     env, _ = parse(declarations)
 
@@ -14,3 +14,5 @@ def test_declarations():
         env.concepts["namespace.distinct_count"].metadata.description
         == "the distinct count of user ids"
     )
+
+    assert env.concepts["namespace.user_id"].metadata.description == "the description"

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113}/tests/test_parsing.py

@@ -1,4 +1,4 @@
-from trilogy.core.enums import Purpose
+from trilogy.core.enums import Purpose, ComparisonOperator
 from trilogy.core.models import (
     DataType,
     Parenthetical,
@@ -6,6 +6,7 @@ from trilogy.core.models import (
     ShowStatement,
     SelectStatement,
     Environment,
+    Comparison,
 )
 from trilogy.core.functions import argument_to_purpose, function_args_to_output_purpose
 from trilogy.parsing.parse_engine import (
@@ -210,10 +211,55 @@ rowset test<- select
     ;
 
 select 
-    count(test.name) -> test_name_count;
+    count( test.name ) -> test_name_count;
 """
     )
     # assert output_purpose == Purpose.METRIC
     for name in ["test_name_count"]:
         assert name in env.concepts
         assert env.concepts[name].purpose == Purpose.METRIC
+
+
+def test_between():
+    _, parsed = parse_text(
+        "const order_id <- 4; SELECT order_id  WHERE order_id BETWEEN 3 and 5;"
+    )
+    query: ProcessedQuery = parsed[-1]
+    left = query.where_clause.conditional.left
+    assert isinstance(
+        left,
+        Comparison,
+    ), type(left)
+    assert left.operator == ComparisonOperator.GTE
+    assert left.right == 3
+
+    right = query.where_clause.conditional.right
+    assert isinstance(
+        right,
+        Comparison,
+    ), type(right)
+    assert right.operator == ComparisonOperator.LTE
+    assert right.right == 5
+
+
+def test_the_comments():
+    _, parsed = parse_text(
+        """const
+         # comment here?
+           order_id <- 4; SELECT 
+        # TOOD - add in more columns?
+        order_id   # this is the order id
+        WHERE 
+        # order_id should not be null
+        order_id
+        # in this comp
+          is not 
+        null; # nulls are the worst
+        
+        """
+    )
+    query = parsed[-1]
+    right = query.where_clause.conditional.right
+    assert isinstance(right, MagicConstants), type(right)
+    rendered = BaseDialect().render_expr(right)
+    assert rendered == "null"

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113}/tests/test_select.py

@@ -37,6 +37,7 @@ datasource users (
     env, parsed = parse(declarations)
 
     q1 = """select
+    # a comment
     user_id,
     about_me,
     count(post_id)->post_count

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113}/tests/test_where_clause.py

@@ -150,7 +150,7 @@ select
     order_id,
     total_revenue,
 where 
-(order_id =1 or order_id = 2) and total_revenue>30
+(order_id = 1 or order_id = 2) and total_revenue>30
 ;
 
 

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.1.111"
+__version__ = "0.0.1.113"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113}/trilogy/core/enums.py

@@ -76,6 +76,12 @@ class JoinType(Enum):
 class Ordering(Enum):
     ASCENDING = "asc"
     DESCENDING = "desc"
+    ASC_NULLS_AUTO = "asc nulls auto"
+    ASC_NULLS_FIRST = "asc nulls first"
+    ASC_NULLS_LAST = "asc nulls last"
+    DESC_NULLS_FIRST = "desc nulls first"
+    DESC_NULLS_LAST = "desc nulls last"
+    DESC_NULLS_AUTO = "desc nulls auto"
 
 
 class WindowType(Enum):
@@ -220,6 +226,7 @@ class ComparisonOperator(Enum):
     ILIKE = "ilike"
     CONTAINS = "contains"
     ELSE = "else"
+    BETWEEN = "between"
 
     @classmethod
     def _missing_(cls, value):

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113}/trilogy/core/models.py

@@ -812,6 +812,7 @@ class Function(Namespaced, SelectGrain, BaseModel):
             "Parenthetical",
             CaseWhen,
             "CaseElse",
+            list,
             ListWrapper[int],
             ListWrapper[str],
             ListWrapper[float],
@@ -996,6 +997,7 @@ class WindowItem(Namespaced, SelectGrain, BaseModel):
     content: Concept
     order_by: List["OrderItem"]
     over: List["Concept"] = Field(default_factory=list)
+    index: Optional[int] = None
 
     def with_namespace(self, namespace: str) -> "WindowItem":
         return WindowItem(
@@ -2041,6 +2043,14 @@ class CTE(BaseModel):
         ds_being_inlined = qds_being_inlined.datasources[0]
         if not isinstance(ds_being_inlined, Datasource):
             return False
+        if any(
+            [
+                x.identifier == ds_being_inlined.identifier
+                for x in self.source.datasources
+            ]
+        ):
+            return False
+
         self.source.datasources = [
             ds_being_inlined,
             *[
@@ -2727,6 +2737,14 @@ class Comparison(ConceptArgs, Namespaced, SelectGrain, BaseModel):
             raise SyntaxError(
                 f"Cannot compare {self.left} and {self.right} of different types"
             )
+        if self.operator == ComparisonOperator.BETWEEN:
+            if (
+                not isinstance(self.right, ComparisonOperator)
+                and self.right.operator == BooleanOperator.AND
+            ):
+                raise SyntaxError(
+                    f"Between operator must have two operands with and, not {self.right}"
+                )
 
     def __add__(self, other):
         if not isinstance(other, (Comparison, Conditional, Parenthetical)):
@@ -2802,6 +2820,29 @@ class Comparison(ConceptArgs, Namespaced, SelectGrain, BaseModel):
         output += get_concept_arguments(self.right)
         return output
 
+    @property
+    def row_arguments(self) -> List[Concept]:
+        output = []
+        if isinstance(self.left, ConceptArgs):
+            output += self.left.row_arguments
+        else:
+            output += get_concept_arguments(self.left)
+        if isinstance(self.right, ConceptArgs):
+            output += self.right.row_arguments
+        else:
+            output += get_concept_arguments(self.right)
+        return output
+
+    @property
+    def existence_arguments(self) -> List[Tuple[Concept, ...]]:
+        """Return concepts directly referenced in where clause"""
+        output: List[Tuple[Concept, ...]] = []
+        if isinstance(self.left, ConceptArgs):
+            output += self.left.existence_arguments
+        if isinstance(self.right, ConceptArgs):
+            output += self.right.existence_arguments
+        return output
+
 
 class SubselectComparison(Comparison):
 
@@ -2901,6 +2942,7 @@ class Conditional(ConceptArgs, Namespaced, SelectGrain, BaseModel):
         float,
         list,
         bool,
+        MagicConstants,
         Concept,
         Comparison,
         "Conditional",
@@ -2914,11 +2956,13 @@ class Conditional(ConceptArgs, Namespaced, SelectGrain, BaseModel):
         float,
         list,
         bool,
+        MagicConstants,
         Concept,
         Comparison,
         "Conditional",
         "Parenthetical",
         Function,
+        FilterItem,
     ]
     operator: BooleanOperator
 

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113}/trilogy/core/optimization.py

@@ -12,9 +12,6 @@ from trilogy.constants import logger, CONFIG
 from abc import ABC
 
 
-REGISTERED_RULES: list["OptimizationRule"] = []
-
-
 class OptimizationRule(ABC):
 
     def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
@@ -66,13 +63,15 @@ class InlineDatasource(OptimizationRule):
             to_inline.append(parent_cte)
 
         for replaceable in to_inline:
-            self.log(f"Inlining parent {replaceable.name}")
-            cte.inline_parent_datasource(replaceable, force_group=force_group)
 
+            result = cte.inline_parent_datasource(replaceable, force_group=force_group)
+            if result:
+                self.log(f"Inlined parent {replaceable.name}")
+            else:
+                self.log(f"Failed to inline {replaceable.name}")
         return optimized
 
 
-# This will be used in the future for more complex condition decomposition
 def decompose_condition(conditional: Conditional):
     chunks = []
     if conditional.operator == BooleanOperator.AND:
@@ -154,12 +153,6 @@ class PredicatePushdown(OptimizationRule):
         return optimized
 
 
-if CONFIG.optimizations.datasource_inlining:
-    REGISTERED_RULES.append(InlineDatasource())
-if CONFIG.optimizations.predicate_pushdown:
-    REGISTERED_RULES.append(PredicatePushdown())
-
-
 def filter_irrelevant_ctes(
     input: list[CTE],
     root_cte: CTE,
@@ -233,6 +226,12 @@ def optimize_ctes(
     input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
 ):
     complete = False
+    REGISTERED_RULES: list["OptimizationRule"] = []
+
+    if CONFIG.optimizations.datasource_inlining:
+        REGISTERED_RULES.append(InlineDatasource())
+    if CONFIG.optimizations.predicate_pushdown:
+        REGISTERED_RULES.append(PredicatePushdown())
 
     while not complete:
         actions_taken = False

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113}/trilogy/dialect/base.py

@@ -10,6 +10,7 @@ from trilogy.core.enums import (
     WindowType,
     DatePart,
     PurposeLineage,
+    ComparisonOperator,
 )
 from trilogy.core.models import (
     ListType,
@@ -58,17 +59,23 @@ def INVALID_REFERENCE_STRING(x: Any, callsite: str = ""):
 
 
 def window_factory(string: str, include_concept: bool = False) -> Callable:
-    def render_window(concept: str, window: str, sort: str) -> str:
+    def render_window(
+        concept: str, window: str, sort: str, offset: int | None = None
+    ) -> str:
         if not include_concept:
             concept = ""
+        if offset:
+            base = f"{string}({concept}, {offset})"
+        else:
+            base = f"{string}({concept})"
         if window and sort:
-            return f"{string}({concept}) over (partition by {window} order by {sort} )"
+            return f"{base} over (partition by {window} order by {sort} )"
         elif window:
-            return f"{string}({concept}) over (partition by {window})"
+            return f"{base} over (partition by {window})"
         elif sort:
-            return f"{string}({concept}) over (order by {sort} )"
+            return f"{base} over (order by {sort} )"
         else:
-            return f"{string}({concept}) over ()"
+            return f"{base} over ()"
 
     return render_window
 
@@ -109,10 +116,10 @@ FUNCTION_MAP = {
     FunctionType.INDEX_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
     FunctionType.UNNEST: lambda x: f"unnest({x[0]})",
     # math
-    FunctionType.ADD: lambda x: f"({x[0]} + {x[1]})",
-    FunctionType.SUBTRACT: lambda x: f"({x[0]} - {x[1]})",
-    FunctionType.DIVIDE: lambda x: f"({x[0]} / {x[1]})",
-    FunctionType.MULTIPLY: lambda x: f"({x[0]} * {x[1]})",
+    FunctionType.ADD: lambda x: f"{x[0]} + {x[1]}",
+    FunctionType.SUBTRACT: lambda x: f"{x[0]} - {x[1]}",
+    FunctionType.DIVIDE: lambda x: f"{x[0]} / {x[1]}",
+    FunctionType.MULTIPLY: lambda x: f"{x[0]} * {x[1]}",
     FunctionType.ROUND: lambda x: f"round({x[0]},{x[1]})",
     FunctionType.MOD: lambda x: f"({x[0]} % {x[1]})",
     # aggregate types
@@ -355,17 +362,30 @@ class BaseDialect:
                     lookup = lookup_cte.existence_source_map[e.right.address]
 
                 return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} (select {lookup[0]}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {lookup[0]})"
-            elif isinstance(e.right, (ListWrapper, Parenthetical)):
+            elif isinstance(e.right, (ListWrapper, Parenthetical, list)):
                 return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map)}"
-            elif isinstance(e.right, (str, int, bool, float, list)):
+
+            elif isinstance(
+                e.right,
+                (
+                    str,
+                    int,
+                    bool,
+                    float,
+                ),
+            ):
                 return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} ({self.render_expr(e.right, cte=cte, cte_map=cte_map)})"
             else:
-                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} ({self.render_expr(e.right, cte=cte, cte_map=cte_map)})"
+                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map)}"
         elif isinstance(e, Comparison):
+            if e.operator == ComparisonOperator.BETWEEN:
+                right_comp = e.right
+                assert isinstance(right_comp, Conditional)
+                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(right_comp.left, cte=cte, cte_map=cte_map) and self.render_expr(right_comp.right, cte=cte, cte_map=cte_map)}"
             return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map)}"
         elif isinstance(e, Conditional):
             # conditions need to be nested in parentheses
-            return f"( {self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map)} ) "
+            return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map)}"
         elif isinstance(e, WindowItem):
             rendered_order_components = [
                 f"{self.render_expr(x.expr, cte, cte_map=cte_map)} {x.order.value}"
@@ -375,11 +395,11 @@ class BaseDialect:
                 self.render_expr(x, cte, cte_map=cte_map) for x in e.over
             ]
             return f"{self.WINDOW_FUNCTION_MAP[e.type](concept = self.render_expr(e.content, cte=cte, cte_map=cte_map), window=','.join(rendered_over_components), sort=','.join(rendered_order_components))}"  # noqa: E501
-        elif isinstance(e, FilterItem):
-            return f"CASE WHEN {self.render_expr(e.where.conditional, cte=cte, cte_map=cte_map)} THEN  {self.render_expr(e.content, cte=cte, cte_map=cte_map)} ELSE 0 END"
         elif isinstance(e, Parenthetical):
             # conditions need to be nested in parentheses
-            return f"( {self.render_expr(e.content, cte=cte, cte_map=cte_map)} ) "
+            if isinstance(e.content, list):
+                return f"( {','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e.content])} )"
+            return f"( {self.render_expr(e.content, cte=cte, cte_map=cte_map)} )"
         elif isinstance(e, CaseWhen):
             return f"WHEN {self.render_expr(e.comparison, cte=cte, cte_map=cte_map) } THEN {self.render_expr(e.expr, cte=cte, cte_map=cte_map) }"
         elif isinstance(e, CaseElse):
@@ -412,7 +432,7 @@ class BaseDialect:
         elif isinstance(e, ListWrapper):
             return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}]"
         elif isinstance(e, list):
-            return f"{','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}"
+            return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}]"
         elif isinstance(e, DataType):
             return str(e.value)
         elif isinstance(e, DatePart):

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113}/trilogy/parsing/common.py

@@ -41,7 +41,7 @@ def concept_list_to_keys(concepts: Tuple[Concept, ...]) -> Tuple[Concept, ...]:
 
 
 def constant_to_concept(
-    parent: ListWrapper | int | float | str,
+    parent: ListWrapper | list | int | float | str,
     name: str,
     namespace: str,
     purpose: Purpose | None = None,

{pytrilogy-0.0.1.111 → pytrilogy-0.0.1.113}/trilogy/parsing/parse_engine.py

@@ -114,296 +114,17 @@ from trilogy.parsing.common import (
     arbitrary_to_concept,
 )
 
+CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper)
 
-CONSTANT_TYPES = (int, float, str, bool, ListWrapper)
-
-grammar = r"""
-    !start: ( block | show_statement | comment )*
-    block: statement _TERMINATOR comment?
-    ?statement: concept
-    | datasource
-    | function
-    | multi_select_statement
-    | select_statement
-    | persist_statement
-    | rowset_derivation_statement
-    | import_statement
-    | merge_statement
-    
-    _TERMINATOR:  ";"i /\s*/
-    
-    comment:   /#.*(\n|$)/ |  /\/\/.*\n/  
-    
-    // property display_name string
-    concept_declaration: PURPOSE IDENTIFIER data_type concept_nullable_modifier? metadata?
-    //customer_id.property first_name STRING;
-    //<customer_id,country>.property local_alias STRING
-    concept_property_declaration: PROPERTY (prop_ident | IDENTIFIER) data_type concept_nullable_modifier? metadata?
-    //metric post_length <- len(post_text);
-    concept_derivation:  (PURPOSE | AUTO | PROPERTY ) (prop_ident | IDENTIFIER) "<" "-" expr
-
-    rowset_derivation_statement: ("rowset"i IDENTIFIER "<" "-" (multi_select_statement | select_statement)) | ("with"i IDENTIFIER "as"i (multi_select_statement | select_statement))
-    
-    constant_derivation: CONST IDENTIFIER "<" "-" literal
-    concept_nullable_modifier: "?"
-    concept:  (concept_declaration | concept_derivation | concept_property_declaration | constant_derivation)
-    
-    //concept property
-    prop_ident: "<" (IDENTIFIER ",")* IDENTIFIER ","? ">" "." IDENTIFIER
-
-    // datasource concepts
-    datasource: "datasource" IDENTIFIER  "("  column_assignment_list ")"  grain_clause? (address | query)
-    
-    grain_clause: "grain" "(" column_list ")"
-    
-    address: "address" ADDRESS
-    
-    query: "query" MULTILINE_STRING
-    
-    concept_assignment: IDENTIFIER | (MODIFIER "[" concept_assignment "]" ) | (SHORTHAND_MODIFIER concept_assignment  )
-    
-    column_assignment: ((IDENTIFIER | raw_column_assignment | _static_functions ) ":" concept_assignment) 
-
-    raw_column_assignment: "raw" "(" MULTILINE_STRING ")"
-    
-    column_assignment_list : (column_assignment "," )* column_assignment ","?
-    
-    column_list : (IDENTIFIER "," )* IDENTIFIER ","?
-    
-    import_statement: "import" (IDENTIFIER ".") * IDENTIFIER "as" IDENTIFIER
-
-    // persist_statement
-    persist_statement: "persist"i IDENTIFIER "into"i IDENTIFIER "from"i select_statement grain_clause?
-
-    // select statement
-    select_statement: "select"i select_list  where? comment* order_by? comment* limit? comment*
-
-    // multiple_selects
-    multi_select_statement: select_statement ("merge" select_statement)+ "align"i align_clause  where? comment* order_by? comment* limit? comment*
-
-    align_item: IDENTIFIER ":" IDENTIFIER ("," IDENTIFIER)* ","?
-
-    align_clause: align_item ("," align_item)*  ","?
-
-    // merge statemment
-    merge_statement: "merge" IDENTIFIER ("," IDENTIFIER)* ","? comment*
-
-    // FUNCTION blocks
-    function: raw_function
-    function_binding_item: IDENTIFIER ":" data_type
-    function_binding_list: (function_binding_item ",")* function_binding_item ","?
-    raw_function: "bind" "sql"  IDENTIFIER  "("  function_binding_list ")" "-" ">" data_type "as"i MULTILINE_STRING
-    
-    // user_id where state = Mexico
-    filter_item: "filter"i IDENTIFIER where
-
-    // rank/lag/lead
-    WINDOW_TYPE: ("row_number"i|"rank"i|"lag"i|"lead"i | "sum"i)  /[\s]+/
-    
-    window_item: WINDOW_TYPE (IDENTIFIER | select_transform | comment+ ) window_item_over? window_item_order?
-    
-    window_item_over: ("OVER"i over_list)
-    
-    window_item_order: ("ORDER"i? "BY"i order_list)
-    
-    select_hide_modifier: "--"
-    select_partial_modifier: "~"
-    select_item: (select_hide_modifier | select_partial_modifier)? (IDENTIFIER | select_transform | comment+ )
-    
-    select_list:  ( select_item "," )* select_item ","?
-    
-    //  count(post_id) -> post_count
-    _assignment: ("-" ">") | "as"
-    select_transform : expr _assignment IDENTIFIER metadata?
-    
-    metadata: "metadata" "(" IDENTIFIER "=" _string_lit ")"
-    
-    limit: "LIMIT"i /[0-9]+/
-    
-    !window_order: ("TOP"i | "BOTTOM"i)
-    
-    window: window_order /[0-9]+/
-    
-    window_order_by: "BY"i column_list
-    
-    order_list: (expr ORDERING "," )* expr ORDERING ","?
-    
-    over_list: (IDENTIFIER "," )* IDENTIFIER ","?
-    
-    ORDERING: ("ASC"i | "DESC"i)
-    
-    order_by: "ORDER"i "BY"i order_list
-    
-    //WHERE STATEMENT
-    
-    LOGICAL_OPERATOR: "AND"i | "OR"i
-    
-    conditional: expr LOGICAL_OPERATOR (conditional | expr)
-    
-    where: "WHERE"i (expr | conditional)
-    
-    expr_reference: IDENTIFIER
-
-    !array_comparison: ( ("NOT"i "IN"i) | "IN"i)
-
-    COMPARISON_OPERATOR: (/is[\s]+not/ | "is" |"=" | ">" | "<" | ">=" | "<=" | "!="  )
-    
-    comparison: (expr COMPARISON_OPERATOR expr) 
-
-    subselect_comparison: expr array_comparison expr | (expr array_comparison expr_tuple)
-    
-    expr_tuple: "("  (expr ",")* expr ","?  ")"
-
-    //unnesting is a function
-    unnest: "UNNEST"i "(" expr ")"
-    //indexing into an expression is a function
-    index_access: expr "[" int_lit "]"
-    attr_access: expr  "[" _string_lit "]"
-
-    parenthetical: "(" (conditional | expr) ")"
-    
-    expr:  window_item | filter_item | comparison | subselect_comparison | fgroup |  aggregate_functions | unnest | _string_functions | _math_functions | _generic_functions | _constant_functions| _date_functions |  literal |  expr_reference  | index_access | attr_access |  parenthetical
-    
-    // functions
-    
-    fadd: ("add"i "(" expr "," expr ")" ) | ( expr "+" expr )
-    fsub: ("subtract"i "(" expr "," expr ")" ) | ( expr "-" expr )
-    fmul: ("multiply"i "(" expr "," expr ")" ) | ( expr "*" expr )
-    fdiv: ( "divide"i "(" expr "," expr ")") | ( expr "/" expr )
-    fmod: ( "mod"i "(" expr "," expr ")") | ( expr "%" expr )
-    fround: "round"i "(" expr "," expr ")"
-    fabs: "abs"i "(" expr ")"
-        
-    _math_functions: fadd | fsub | fmul | fdiv | fround | fmod | fabs
-    
-    //generic
-    fcast: "cast"i "(" expr "AS"i data_type ")"
-    concat: ("concat"i "(" (expr ",")* expr ")") | (expr "||" expr)
-    fcoalesce: "coalesce"i "(" (expr ",")* expr ")"
-    fcase_when: "WHEN"i (expr | conditional) "THEN"i expr
-    fcase_else: "ELSE"i expr
-    fcase: "CASE"i (fcase_when)* (fcase_else)? "END"i
-    len: "len"i "(" expr ")"
-    fnot: "NOT"i expr
-
-    _generic_functions: fcast | concat | fcoalesce | fcase | len | fnot
-
-    //constant
-    fcurrent_date: "current_date"i "(" ")"
-    fcurrent_datetime: "current_datetime"i "(" ")"
-
-    _constant_functions: fcurrent_date | fcurrent_datetime
-    
-    //string
-    like: "like"i "(" expr "," _string_lit ")"
-    ilike: "ilike"i "(" expr "," _string_lit ")"
-    alt_like: expr "like"i expr
-    upper: "upper"i "(" expr ")"
-    lower: "lower"i "(" expr ")"    
-    fsplit: "split"i "(" expr "," _string_lit ")"
-    fstrpos: "strpos"i "(" expr "," expr ")"
-    fsubstring: "substring"i "(" expr "," expr "," expr ")"
-    
-    _string_functions: like | ilike | upper | lower | fsplit | fstrpos | fsubstring | alt_like
-    
-    // special aggregate
-    fgroup: "group"i "(" expr ")" aggregate_over?
-    //aggregates
-    count: "count"i "(" expr ")"
-    count_distinct: "count_distinct"i "(" expr ")"
-    sum: "sum"i "(" expr ")"
-    avg: "avg"i "(" expr ")"
-    max: "max"i "(" expr ")"
-    min: "min"i "(" expr ")"
-    
-    //aggregates can force a grain
-    aggregate_all: "*"
-    aggregate_over: ("BY"i (aggregate_all | over_list))
-    aggregate_functions: (count | count_distinct | sum | avg | max | min) aggregate_over?
-
-    // date functions
-    fdate: "date"i "(" expr ")"
-    fdatetime: "datetime"i "(" expr ")"
-    ftimestamp: "timestamp"i "(" expr ")"
-    
-    fsecond: "second"i "(" expr ")"
-    fminute: "minute"i "(" expr ")"
-    fhour: "hour"i "(" expr ")"
-    fday: "day"i "(" expr ")"
-    fday_of_week: "day_of_week"i "(" expr ")"
-    fweek: "week"i "(" expr ")"
-    fmonth: "month"i "(" expr ")"
-    fquarter: "quarter"i  "(" expr ")"
-    fyear: "year"i "(" expr ")"
-    
-    DATE_PART: "DAY"i | "WEEK"i | "MONTH"i | "QUARTER"i | "YEAR"i | "MINUTE"i | "HOUR"i | "SECOND"i
-    fdate_trunc: "date_trunc"i "(" expr "," DATE_PART ")"
-    fdate_part: "date_part"i "(" expr "," DATE_PART ")"
-    fdate_add: "date_add"i "(" expr "," DATE_PART "," int_lit ")"
-    fdate_diff: "date_diff"i "(" expr "," expr "," DATE_PART ")"
-    
-    _date_functions: fdate | fdate_add | fdate_diff | fdatetime | ftimestamp | fsecond | fminute | fhour | fday | fday_of_week | fweek | fmonth | fquarter | fyear | fdate_part | fdate_trunc
-    
-    _static_functions: _string_functions | _math_functions | _generic_functions | _constant_functions| _date_functions
-
-    // base language constructs
-    IDENTIFIER: /[a-zA-Z_][a-zA-Z0-9_\\-\\.\-]*/
-    ADDRESS: /[a-zA-Z_][a-zA-Z0-9_\\-\\.\-\*]*/ | /`[a-zA-Z_][a-zA-Z0-9_\\-\\.\-\*]*`/
-
-    MULTILINE_STRING: /\'{3}(.*?)\'{3}/s
-    
-    DOUBLE_STRING_CHARS: /(?:(?!\${)([^"\\]|\\.))+/+ // any character except "
-    SINGLE_STRING_CHARS: /(?:(?!\${)([^'\\]|\\.))+/+ // any character except '
-    _single_quote: "'" ( SINGLE_STRING_CHARS )* "'" 
-    _double_quote: "\"" ( DOUBLE_STRING_CHARS )* "\"" 
-    _string_lit: _single_quote | _double_quote
-
-    MINUS: "-"
-
-    int_lit: MINUS? /[0-9]+/
-    
-    float_lit: /[0-9]*\.[0-9]+/
-
-    array_lit: "[" (literal ",")* literal ","? "]"()
-    
-    !bool_lit: "True"i | "False"i
-
-    !null_lit: "null"i
-    
-    literal: _string_lit | int_lit | float_lit | bool_lit | null_lit | array_lit
-
-    MODIFIER: "Optional"i | "Partial"i | "Nullable"i
-
-    SHORTHAND_MODIFIER: "~"
-
-    struct_type: "struct" "<" ((data_type | IDENTIFIER) ",")* (data_type | IDENTIFIER) ","? ">" 
-
-    list_type: "list" "<" data_type ">"
-
-
-    !data_type: "string"i | "number"i | "numeric"i | "map"i | "list"i | "array"i | "any"i | "int"i | "bigint" | "date"i | "datetime"i | "timestamp"i | "float"i | "bool"i | struct_type | list_type
-    
-    PURPOSE:  "key"i | "metric"i | "const"i | "constant"i
-    PROPERTY: "property"i
-    CONST: "const"i | "constant"i
-    AUTO: "AUTO"i 
-
-    // meta functions
-    CONCEPTS: "CONCEPTS"i
-    DATASOURCES: "DATASOURCES"i
-
-    show_category: CONCEPTS | DATASOURCES
-
-    show_statement: "show"i ( show_category | select_statement | persist_statement) _TERMINATOR
-
-    %import common.WS_INLINE -> _WHITESPACE
-    %import common.WS
-    %ignore WS
-"""  # noqa: E501
-
-PARSER = Lark(
-    grammar, start="start", propagate_positions=True, g_regex_flags=IGNORECASE
-)
+with open(join(dirname(__file__), "trilogy.lark"), "r") as f:
+    PARSER = Lark(
+        f.read(),
+        start="start",
+        propagate_positions=True,
+        g_regex_flags=IGNORECASE,
+        parser="lalr",
+        cache=True,
+    )
 
 
 def parse_concept_reference(
@@ -567,6 +288,9 @@ class ParseToObjects(Transformer):
     def IDENTIFIER(self, args) -> str:
         return args.value
 
+    def concept_lit(self, args) -> Concept:
+        return self.environment.concepts.__getitem__(args[0])
+
     def ADDRESS(self, args) -> str:
         return args.value
 
@@ -622,19 +346,23 @@ class ParseToObjects(Transformer):
     def column_assignment(self, meta: Meta, args):
         # TODO -> deal with conceptual modifiers
         modifiers = []
-        concept = args[1]
+        alias = args[0]
+        concept_list = args[1]
         # recursively collect modifiers
-        while len(concept) > 1:
-            modifiers.append(concept[0])
-            concept = concept[1]
+        if len(concept_list) > 1:
+            modifiers += concept_list[:-1]
+        concept = concept_list[-1]
         resolved = self.environment.concepts.__getitem__(  # type: ignore
-            key=concept[0], line_no=meta.line
+            key=concept, line_no=meta.line
         )
-        return ColumnAssignment(alias=args[0], modifiers=modifiers, concept=resolved)
+        return ColumnAssignment(alias=alias, modifiers=modifiers, concept=resolved)
 
     def _TERMINATOR(self, args):
         return None
 
+    def _static_functions(self, args):
+        return args[0]
+
     def MODIFIER(self, args) -> Modifier:
         return Modifier(args.value)
 
@@ -893,6 +621,9 @@ class ParseToObjects(Transformer):
         assert len(args) == 1
         return Comment(text=args[0].value)
 
+    def PARSE_COMMENT(self, args):
+        return Comment(text=args.value)
+
     @v_args(meta=True)
     def select_transform(self, meta, args) -> ConceptTransform:
 
@@ -962,7 +693,7 @@ class ParseToObjects(Transformer):
         if isinstance(content, ConceptTransform):
             return SelectItem(content=content, modifiers=modifiers)
         return SelectItem(
-            content=self.environment.concepts.__getitem__(content, meta.line),
+            content=content,
             modifiers=modifiers,
         )
 
@@ -972,8 +703,12 @@ class ParseToObjects(Transformer):
     def limit(self, args):
         return Limit(count=int(args[0].value))
 
-    def ORDERING(self, args):
-        return Ordering(args.lower())
+    def ordering(self, args: list[str]):
+        base = args[0].lower()
+        if len(args) > 1:
+            null_sort = args[-1]
+            return Ordering(" ".join([base, "nulls", null_sort.lower()]))
+        return Ordering(base)
 
     def order_list(self, args):
 
@@ -1001,7 +736,7 @@ class ParseToObjects(Transformer):
         return OrderBy(items=args[0])
 
     def over_list(self, args):
-        return [self.environment.concepts[x] for x in args]
+        return [x for x in args]
 
     @v_args(meta=True)
     def merge_statement(self, meta: Meta, args) -> MergeStatement:
@@ -1044,9 +779,7 @@ class ParseToObjects(Transformer):
                 # add the parsed objects of the import in
                 self.parsed = {**self.parsed, **nparser.parsed}
             except Exception as e:
-                raise ImportError(
-                    f"Unable to import file {dirname(target)}, parsing error: {e}"
-                )
+                raise ImportError(f"Unable to import file {target}, parsing error: {e}")
 
         for _, concept in nparser.environment.concepts.items():
             self.environment.add_concept(concept.with_namespace(alias))
@@ -1241,6 +974,19 @@ class ParseToObjects(Transformer):
             raise SyntaxError
         return Comparison(left=args[0], right=args[2], operator=args[1])
 
+    def between_comparison(self, args) -> Conditional:
+        left_bound = args[1]
+        right_bound = args[2]
+        return Conditional(
+            left=Comparison(
+                left=args[0], right=left_bound, operator=ComparisonOperator.GTE
+            ),
+            right=Comparison(
+                left=args[0], right=right_bound, operator=ComparisonOperator.LTE
+            ),
+            operator=BooleanOperator.AND,
+        )
+
     @v_args(meta=True)
     def subselect_comparison(self, meta: Meta, args) -> SubselectComparison:
         right = args[2]
@@ -1297,13 +1043,23 @@ class ParseToObjects(Transformer):
         type = args[0]
         order_by = []
         over = []
-        for item in args[2:]:
-            if isinstance(item, WindowItemOrder):
+        index = None
+        concept: Concept | None = None
+        for item in args:
+            if isinstance(item, int):
+                index = item
+            elif isinstance(item, WindowItemOrder):
                 order_by = item.contents
             elif isinstance(item, WindowItemOver):
                 over = item.contents
-        concept = self.environment.concepts[args[1]]
-        return WindowItem(type=type, content=concept, over=over, order_by=order_by)
+            elif isinstance(item, str):
+                concept = self.environment.concepts[item]
+            elif isinstance(item, Concept):
+                concept = item
+        assert concept
+        return WindowItem(
+            type=type, content=concept, over=over, order_by=order_by, index=index
+        )
 
     def filter_item(self, args) -> FilterItem:
         where: WhereClause
@@ -1501,6 +1257,9 @@ class ParseToObjects(Transformer):
         )
         return SubString(args)
 
+    def logical_operator(self, args):
+        return BooleanOperator(args[0].value.lower())
+
     @v_args(meta=True)
     def lower(self, meta, args):
         args = self.process_function_args(args, meta=meta)
@@ -1893,12 +1652,12 @@ class ParseToObjects(Transformer):
     @v_args(meta=True)
     def fcurrent_date(self, meta, args):
         args = self.process_function_args(args, meta=meta)
-        return CurrentDate(args)
+        return CurrentDate([])
 
     @v_args(meta=True)
     def fcurrent_datetime(self, meta, args):
         args = self.process_function_args(args, meta=meta)
-        return CurrentDatetime(args)
+        return CurrentDatetime([])
 
     @v_args(meta=True)
     def fnot(self, meta, args):
@@ -1914,10 +1673,14 @@ def unpack_visit_error(e: VisitError):
     elif isinstance(e.orig_exc, (UndefinedConceptException, ImportError)):
         raise e.orig_exc
     elif isinstance(e.orig_exc, (ValidationError, TypeError)):
-        raise InvalidSyntaxException(str(e.orig_exc))
+        raise InvalidSyntaxException(str(e.orig_exc) + str(e.rule) + str(e.obj))
     raise e
 
 
+def parse_text_raw(text: str, environment: Optional[Environment] = None):
+    PARSER.parse(text)
+
+
 def parse_text(text: str, environment: Optional[Environment] = None) -> Tuple[
     Environment,
     List[