pytrilogy 0.0.3.113__py3-none-any.whl → 0.0.3.116__py3-none-any.whl

trilogy/dialect/base.py

@@ -176,6 +176,20 @@ def struct_arg(args):
     return [f"{x[1]}: {x[0]}" for x in zip(args[::2], args[1::2])]
 
 
+def hash_from_args(val, hash_type):
+    hash_type = hash_type[1:-1]
+    if hash_type.lower() == "md5":
+        return f"md5({val})"
+    elif hash_type.lower() == "sha1":
+        return f"sha1({val})"
+    elif hash_type.lower() == "sha256":
+        return f"sha256({val})"
+    elif hash_type.lower() == "sha512":
+        return f"sha512({val})"
+    else:
+        raise ValueError(f"Unsupported hash type: {hash_type}")
+
+
 FUNCTION_MAP = {
     # generic types
     FunctionType.ALIAS: lambda x: f"{x[0]}",
@@ -194,6 +208,13 @@ FUNCTION_MAP = {
     FunctionType.INDEX_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
     FunctionType.MAP_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
     FunctionType.UNNEST: lambda x: f"unnest({x[0]})",
+    FunctionType.DATE_SPINE: lambda x: f"""unnest(
+        generate_series(
+            {x[0]},
+            {x[1]},
+            INTERVAL '1 day'
+        )
+    )""",
     FunctionType.RECURSE_EDGE: lambda x: f"CASE WHEN {x[1]} IS NULL THEN {x[0]} ELSE {x[1]} END",
     FunctionType.ATTR_ACCESS: lambda x: f"""{x[0]}.{x[1].replace("'", "")}""",
     FunctionType.STRUCT: lambda x: f"{{{', '.join(struct_arg(x))}}}",
@@ -213,6 +234,9 @@ FUNCTION_MAP = {
     FunctionType.ARRAY_TO_STRING: lambda args: (
         f"array_to_string({args[0]}, {args[1]})"
     ),
+    FunctionType.ARRAY_FILTER: lambda args: (
+        f"array_filter({args[0]}, {args[1]} -> {args[2]})"
+    ),
     # math
     FunctionType.ADD: lambda x: " + ".join(x),
     FunctionType.ABS: lambda x: f"abs({x[0]})",
@@ -237,6 +261,7 @@ FUNCTION_MAP = {
     FunctionType.AVG: lambda x: f"avg({x[0]})",
     FunctionType.MAX: lambda x: f"max({x[0]})",
     FunctionType.MIN: lambda x: f"min({x[0]})",
+    FunctionType.ANY: lambda x: f"any_value({x[0]})",
     # string types
     FunctionType.LIKE: lambda x: f" {x[0]} like {x[1]} ",
     FunctionType.UPPER: lambda x: f"UPPER({x[0]}) ",
@@ -249,6 +274,7 @@ FUNCTION_MAP = {
     FunctionType.REGEXP_REPLACE: lambda x: f"REGEXP_REPLACE({x[0]},{x[1]}, {x[2]})",
     FunctionType.TRIM: lambda x: f"TRIM({x[0]})",
     FunctionType.REPLACE: lambda x: f"REPLACE({x[0]},{x[1]},{x[2]})",
+    FunctionType.HASH: lambda x: hash_from_args(x[0], x[1]),
     # FunctionType.NOT_LIKE: lambda x: f" CASE WHEN {x[0]} like {x[1]} THEN 0 ELSE 1 END",
     # date types
     FunctionType.DATE_TRUNCATE: lambda x: f"date_trunc({x[0]},{x[1]})",
@@ -285,6 +311,7 @@ FUNCTION_GRAIN_MATCH_MAP = {
     FunctionType.AVG: lambda args: f"{args[0]}",
     FunctionType.MAX: lambda args: f"{args[0]}",
     FunctionType.MIN: lambda args: f"{args[0]}",
+    FunctionType.ANY: lambda args: f"{args[0]}",
 }
 
 
@@ -472,7 +499,20 @@ class BaseDialect:
             elif isinstance(c.lineage, BuildRowsetItem):
                 rval = f"{self.render_concept_sql(c.lineage.content, cte=cte, alias=False, raise_invalid=raise_invalid)}"
             elif isinstance(c.lineage, BuildMultiSelectLineage):
-                rval = f"{self.render_concept_sql(c.lineage.find_source(c, cte), cte=cte, alias=False, raise_invalid=raise_invalid)}"
+                if c.address in c.lineage.calculated_derivations:
+                    assert c.lineage.derive is not None
+                    for x in c.lineage.derive.items:
+                        if x.address == c.address:
+                            rval = self.render_expr(
+                                x.expr,
+                                cte=cte,
+                                raise_invalid=raise_invalid,
+                            )
+                            break
+                else:
+                    rval = f"{self.render_concept_sql(c.lineage.find_source(c, cte), cte=cte, alias=False, raise_invalid=raise_invalid)}"
+            elif isinstance(c.lineage, BuildComparison):
+                rval = f"{self.render_expr(c.lineage.left, cte=cte, raise_invalid=raise_invalid)} {c.lineage.operator.value} {self.render_expr(c.lineage.right, cte=cte, raise_invalid=raise_invalid)}"
             elif isinstance(c.lineage, AGGREGATE_ITEMS):
                 args = [
                     self.render_expr(v, cte)  # , alias=False)
@@ -804,7 +844,7 @@ class BaseDialect:
             if self.rendering.parameters:
                 if e.concept.namespace == DEFAULT_NAMESPACE:
                     return f":{e.concept.name}"
-                return f":{e.concept.address}"
+                return f":{e.concept.address.replace('.', '_')}"
             elif e.concept.lineage:
                 return self.render_expr(e.concept.lineage, cte=cte, cte_map=cte_map)
             return f"{self.QUOTE_CHARACTER}{e.concept.address}{self.QUOTE_CHARACTER}"

trilogy/executor.py

@@ -397,7 +397,7 @@ class Executor(object):
                 if v.safe_address == param or v.address == param
             ]
         if not matched:
-            raise SyntaxError(f"No concept found for parameter {param}")
+            raise SyntaxError(f"No concept found for parameter {param};")
 
         concept: Concept = matched.pop()
         return self._concept_to_value(concept, local_concepts=local_concepts)

trilogy/parsing/common.py

@@ -3,9 +3,7 @@ from typing import Iterable, List, Sequence, Tuple
 
 from lark.tree import Meta
 
-from trilogy.constants import (
-    VIRTUAL_CONCEPT_PREFIX,
-)
+from trilogy.constants import DEFAULT_NAMESPACE, VIRTUAL_CONCEPT_PREFIX
 from trilogy.core.constants import ALL_ROWS_CONCEPT
 from trilogy.core.enums import (
     ConceptSource,
@@ -64,10 +62,12 @@ ARBITRARY_INPUTS = (
     | Parenthetical
     | ListWrapper
     | MapWrapper
+    | Comparison
     | int
     | float
     | str
     | date
+    | bool
 )
 
 
@@ -91,7 +91,7 @@ def process_function_arg(
         # to simplify anonymous function handling
         if (
             arg.operator not in FunctionClass.AGGREGATE_FUNCTIONS.value
-            and arg.operator != FunctionType.UNNEST
+            and arg.operator not in FunctionClass.ONE_TO_MANY.value
         ):
             return arg
         id_hash = string_to_hash(str(arg))
@@ -311,13 +311,18 @@ def concept_is_relevant(
     if concept.purpose in (Purpose.METRIC,):
         if all([c in others for c in concept.grain.components]):
             return False
+    if (
+        concept.derivation in (Derivation.BASIC,)
+        and isinstance(concept.lineage, Function)
+        and concept.lineage.operator == FunctionType.DATE_SPINE
+    ):
+        return True
     if concept.derivation in (Derivation.BASIC,) and isinstance(
         concept.lineage, (Function, CaseWhen)
     ):
         relevant = False
         for arg in concept.lineage.arguments:
             relevant = atom_is_relevant(arg, others, environment) or relevant
-
         return relevant
     if concept.derivation in (Derivation.BASIC,) and isinstance(
         concept.lineage, Parenthetical
@@ -394,6 +399,10 @@ def _get_relevant_parent_concepts(arg) -> tuple[list[ConceptRef], bool]:
         return [x.reference for x in arg.by], True
     elif isinstance(arg, FunctionCallWrapper):
         return get_relevant_parent_concepts(arg.content)
+    elif isinstance(arg, Comparison):
+        left, lflag = get_relevant_parent_concepts(arg.left)
+        right, rflag = get_relevant_parent_concepts(arg.right)
+        return left + right, lflag or rflag
     return get_concept_arguments(arg), False
 
 
@@ -529,7 +538,7 @@ def function_to_concept(
     elif parent.operator == FunctionType.UNION:
         derivation = Derivation.UNION
         granularity = Granularity.MULTI_ROW
-    elif parent.operator == FunctionType.UNNEST:
+    elif parent.operator in FunctionClass.ONE_TO_MANY.value:
         derivation = Derivation.UNNEST
         granularity = Granularity.MULTI_ROW
     elif parent.operator == FunctionType.RECURSE_EDGE:
@@ -770,6 +779,27 @@ def align_item_to_concept(
     return new
 
 
+def derive_item_to_concept(
+    parent: ARBITRARY_INPUTS,
+    name: str,
+    lineage: MultiSelectLineage,
+    namespace: str | None = None,
+) -> Concept:
+    datatype = arg_to_datatype(parent)
+    grain = Grain()
+    new = Concept(
+        name=name,
+        datatype=datatype,
+        purpose=Purpose.PROPERTY,
+        lineage=lineage,
+        grain=grain,
+        namespace=namespace or DEFAULT_NAMESPACE,
+        granularity=Granularity.MULTI_ROW,
+        derivation=Derivation.MULTISELECT,
+    )
+    return new
+
+
 def rowset_concept(
     orig_address: ConceptRef,
     environment: Environment,
@@ -851,20 +881,7 @@ def rowset_to_concepts(rowset: RowsetDerivationStatement, environment: Environme
 
 
 def generate_concept_name(
-    parent: (
-        AggregateWrapper
-        | FunctionCallWrapper
-        | WindowItem
-        | FilterItem
-        | Function
-        | ListWrapper
-        | MapWrapper
-        | Parenthetical
-        | int
-        | float
-        | str
-        | date
-    ),
+    parent: ARBITRARY_INPUTS,
 ) -> str:
     """Generate a name for a concept based on its parent type and content."""
     if isinstance(parent, AggregateWrapper):
@@ -885,6 +902,8 @@ def generate_concept_name(
         return f"{VIRTUAL_CONCEPT_PREFIX}_paren_{string_to_hash(str(parent))}"
     elif isinstance(parent, FunctionCallWrapper):
         return f"{VIRTUAL_CONCEPT_PREFIX}_{parent.name}_{string_to_hash(str(parent))}"
+    elif isinstance(parent, Comparison):
+        return f"{VIRTUAL_CONCEPT_PREFIX}_comp_{string_to_hash(str(parent))}"
     else:  # ListWrapper, MapWrapper, or primitive types
         return f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"
 
@@ -909,6 +928,82 @@ def parenthetical_to_concept(
     )
 
 
+def comparison_to_concept(
+    parent: Comparison,
+    name: str,
+    namespace: str,
+    environment: Environment,
+    metadata: Metadata | None = None,
+):
+    fmetadata = metadata or Metadata()
+
+    pkeys: List[Concept] = []
+    namespace = namespace or environment.namespace
+    is_metric = False
+    ref_args, is_metric = get_relevant_parent_concepts(parent)
+    concrete_args = [environment.concepts[c.address] for c in ref_args]
+    pkeys += [
+        x
+        for x in concrete_args
+        if not x.derivation == Derivation.CONSTANT
+        and not (x.derivation == Derivation.AGGREGATE and not x.grain.components)
+    ]
+    grain: Grain | None = Grain()
+    for x in pkeys:
+        grain += x.grain
+    if parent.operator in FunctionClass.ONE_TO_MANY.value:
+        # if the function will create more rows, we don't know what grain this is at
+        grain = None
+    modifiers = get_upstream_modifiers(pkeys, environment)
+    key_grain: list[str] = []
+    for x in pkeys:
+        # metrics will group to keys, so do not do key traversal
+        if is_metric:
+            key_grain.append(x.address)
+        # otherwse, for row ops, assume keys are transitive
+        elif x.keys:
+            key_grain += [*x.keys]
+        else:
+            key_grain.append(x.address)
+    keys = set(key_grain)
+    if is_metric:
+        purpose = Purpose.METRIC
+    elif not pkeys:
+        purpose = Purpose.CONSTANT
+    else:
+        purpose = Purpose.PROPERTY
+    fmetadata = metadata or Metadata()
+
+    if grain is not None:
+        r = Concept(
+            name=name,
+            datatype=parent.output_datatype,
+            purpose=purpose,
+            lineage=parent,
+            namespace=namespace,
+            keys=keys,
+            modifiers=modifiers,
+            grain=grain,
+            metadata=fmetadata,
+            derivation=Derivation.BASIC,
+            granularity=Granularity.MULTI_ROW,
+        )
+        return r
+
+    return Concept(
+        name=name,
+        datatype=parent.output_datatype,
+        purpose=purpose,
+        lineage=parent,
+        namespace=namespace,
+        keys=keys,
+        modifiers=modifiers,
+        metadata=fmetadata,
+        derivation=Derivation.BASIC,
+        granularity=Granularity.MULTI_ROW,
+    )
+
+
 def arbitrary_to_concept(
     parent: ARBITRARY_INPUTS,
     environment: Environment,
@@ -968,5 +1063,7 @@ def arbitrary_to_concept(
         return constant_to_concept(parent, name, namespace, metadata)
     elif isinstance(parent, Parenthetical):
         return parenthetical_to_concept(parent, name, namespace, environment, metadata)
+    elif isinstance(parent, Comparison):
+        return comparison_to_concept(parent, name, namespace, environment, metadata)
     else:
         return constant_to_concept(parent, name, namespace, metadata)

trilogy/parsing/parse_engine.py

@@ -62,6 +62,8 @@ from trilogy.core.models.author import (
     Conditional,
     CustomFunctionFactory,
     CustomType,
+    DeriveClause,
+    DeriveItem,
     Expr,
     FilterItem,
     Function,
@@ -69,6 +71,7 @@ from trilogy.core.models.author import (
     Grain,
     HavingClause,
     Metadata,
+    MultiSelectLineage,
     OrderBy,
     OrderItem,
     Parenthetical,
@@ -135,6 +138,7 @@ from trilogy.parsing.common import (
     align_item_to_concept,
     arbitrary_to_concept,
     constant_to_concept,
+    derive_item_to_concept,
     process_function_args,
     rowset_to_concepts,
 )
@@ -603,6 +607,9 @@ class ParseToObjects(Transformer):
     def PROPERTY(self, args):
         return Purpose.PROPERTY
 
+    def HASH_TYPE(self, args):
+        return args.value
+
     @v_args(meta=True)
     def prop_ident(self, meta: Meta, args) -> Tuple[List[Concept], str]:
         return [self.environment.concepts[grain] for grain in args[:-1]], args[-1]
@@ -707,7 +714,14 @@ class ParseToObjects(Transformer):
 
         if isinstance(
             source_value,
-            (FilterItem, WindowItem, AggregateWrapper, Function, FunctionCallWrapper),
+            (
+                FilterItem,
+                WindowItem,
+                AggregateWrapper,
+                Function,
+                FunctionCallWrapper,
+                Comparison,
+            ),
         ):
             concept = arbitrary_to_concept(
                 source_value,
@@ -1275,6 +1289,17 @@ class ParseToObjects(Transformer):
     def align_clause(self, meta: Meta, args) -> AlignClause:
         return AlignClause(items=args)
 
+    @v_args(meta=True)
+    def derive_item(self, meta: Meta, args) -> DeriveItem:
+        return DeriveItem(
+            expr=args[0], name=args[1], namespace=self.environment.namespace
+        )
+
+    @v_args(meta=True)
+    def derive_clause(self, meta: Meta, args) -> DeriveClause:
+
+        return DeriveClause(items=args)
+
     @v_args(meta=True)
     def multi_select_statement(self, meta: Meta, args) -> MultiSelectStatement:
 
@@ -1284,6 +1309,7 @@ class ParseToObjects(Transformer):
         order_by: OrderBy | None = None
         where: WhereClause | None = None
         having: HavingClause | None = None
+        derive: DeriveClause | None = None
         for arg in args:
             if isinstance(arg, SelectStatement):
                 selects.append(arg)
@@ -1297,11 +1323,24 @@ class ParseToObjects(Transformer):
                 having = arg
             elif isinstance(arg, AlignClause):
                 align = arg
+            elif isinstance(arg, DeriveClause):
+                derive = arg
 
         assert align
         assert align is not None
 
         derived_concepts = []
+        new_selects = [x.as_lineage(self.environment) for x in selects]
+        lineage = MultiSelectLineage(
+            selects=new_selects,
+            align=align,
+            derive=derive,
+            namespace=self.environment.namespace,
+            where_clause=where,
+            having_clause=having,
+            limit=limit,
+            hidden_components=set(y for x in new_selects for y in x.hidden_components),
+        )
         for x in align.items:
             concept = align_item_to_concept(
                 x,
@@ -1314,6 +1353,19 @@ class ParseToObjects(Transformer):
             )
             derived_concepts.append(concept)
             self.environment.add_concept(concept, meta=meta)
+        if derive:
+            for derived in derive.items:
+                derivation = derived.expr
+                name = derived.name
+                if not isinstance(derivation, (Function, Comparison, WindowItem)):
+                    raise SyntaxError(
+                        f"Invalid derive expression {derivation} in {meta.line}, must be a function or conditional"
+                    )
+                concept = derive_item_to_concept(
+                    derivation, name, lineage, self.environment.namespace
+                )
+                derived_concepts.append(concept)
+                self.environment.add_concept(concept, meta=meta)
         multi = MultiSelectStatement(
             selects=selects,
             align=align,
@@ -1323,6 +1375,7 @@ class ParseToObjects(Transformer):
             limit=limit,
             meta=Metadata(line_number=meta.line),
             derived_concepts=derived_concepts,
+            derive=derive,
         )
         return multi
 
@@ -1819,6 +1872,10 @@ class ParseToObjects(Transformer):
     def array_agg(self, meta, args):
         return self.function_factory.create_function(args, FunctionType.ARRAY_AGG, meta)
 
+    @v_args(meta=True)
+    def any(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.ANY, meta)
+
     @v_args(meta=True)
     def avg(self, meta, args):
         return self.function_factory.create_function(args, FunctionType.AVG, meta)
@@ -1879,6 +1936,10 @@ class ParseToObjects(Transformer):
     def ftrim(self, meta, args):
         return self.function_factory.create_function(args, FunctionType.TRIM, meta)
 
+    @v_args(meta=True)
+    def fhash(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.HASH, meta)
+
     @v_args(meta=True)
     def fsubstring(self, meta, args):
         return self.function_factory.create_function(args, FunctionType.SUBSTRING, meta)
@@ -2022,6 +2083,12 @@ class ParseToObjects(Transformer):
             )
         return self.function_factory.create_function(args, FunctionType.CAST, meta)
 
+    @v_args(meta=True)
+    def fdate_spine(self, meta, args) -> Function:
+        return self.function_factory.create_function(
+            args, FunctionType.DATE_SPINE, meta
+        )
+
     # utility functions
     @v_args(meta=True)
     def fcast(self, meta, args) -> Function:
@@ -2191,6 +2258,33 @@ class ParseToObjects(Transformer):
             meta,
         )
 
+    @v_args(meta=True)
+    def farray_filter(self, meta, args) -> Function:
+        factory: CustomFunctionFactory = args[1]
+        if not len(factory.function_arguments) == 1:
+            raise InvalidSyntaxException(
+                "Array filter function must have exactly one argument;"
+            )
+        array_type = arg_to_datatype(args[0])
+        if not isinstance(array_type, ArrayType):
+            raise InvalidSyntaxException(
+                f"Array filter function must be applied to an array, not {array_type}"
+            )
+        return self.function_factory.create_function(
+            [
+                args[0],
+                factory.function_arguments[0],
+                factory(
+                    ArgBinding(
+                        name=factory.function_arguments[0].name,
+                        datatype=array_type.value_data_type,
+                    )
+                ),
+            ],
+            FunctionType.ARRAY_FILTER,
+            meta,
+        )
+
 
 def unpack_visit_error(e: VisitError, text: str | None = None):
     """This is required to get exceptions from imports, which would
@@ -2227,6 +2321,7 @@ ERROR_CODES: dict[int, str] = {
     101: "Using FROM keyword? Trilogy does not have a FROM clause (Datasource resolution is automatic).",
     # 200 codes relate to required explicit syntax (we could loosen these?)
     201: 'Missing alias? Alias must be specified with "AS" - e.g. `SELECT x+1 AS y`',
+    202: "Missing closing semicolon? Statements must be terminated with a semicolon `;`.",
     210: "Missing order direction? Order by must be explicit about direction - specify `asc` or `desc`.",
 }
 
@@ -2307,7 +2402,7 @@ def parse_text(
         )
 
     def _handle_unexpected_token(e: UnexpectedToken, text: str) -> None:
-        """Handle UnexpectedToken errors with specific logic."""
+        """Handle UnexpectedToken errors to make friendlier error messages."""
         # Handle ordering direction error
         pos = e.pos_in_stream or 0
         if e.expected == {"ORDERING_DIRECTION"}:
@@ -2319,12 +2414,27 @@ def parse_text(
         )
         if parsed_tokens == ["FROM"]:
             raise _create_syntax_error(101, pos, text)
-
-        # Attempt recovery for aliasing
+        # check if they are missing a semicolon
+        try:
+            e.interactive_parser.feed_token(Token("_TERMINATOR", ";"))
+            state = e.interactive_parser.lexer_thread.state
+            if state and state.last_token:
+                new_pos = state.last_token.end_pos or pos
+            else:
+                new_pos = pos
+            raise _create_syntax_error(202, new_pos, text)
+        except UnexpectedToken:
+            pass
+        # check if they forgot an as
         try:
             e.interactive_parser.feed_token(Token("AS", "AS"))
+            state = e.interactive_parser.lexer_thread.state
+            if state and state.last_token:
+                new_pos = state.last_token.end_pos or pos
+            else:
+                new_pos = pos
             e.interactive_parser.feed_token(Token("IDENTIFIER", e.token.value))
-            raise _create_syntax_error(201, pos, text)
+            raise _create_syntax_error(201, new_pos, text)
         except UnexpectedToken:
             pass
 

trilogy/parsing/render.py

@@ -662,7 +662,8 @@ class Renderer:
                     pair_strings.append(self.indent_lines(pair_line))
             inputs = ",\n".join(pair_strings)
             return f"struct(\n{inputs}\n{self.indent_context.current_indent})"
-
+        if arg.operator == FunctionType.ALIAS:
+            return f"{self.to_string(arg.arguments[0])}"
         inputs = ",".join(args)
         return f"{arg.operator.value}({inputs})"
 

trilogy/parsing/trilogy.lark

@@ -74,12 +74,16 @@
     select_statement: where? "select"i select_list where?  having?  order_by?  limit? 
 
     // multiple_selects
-    multi_select_statement: select_statement ("merge" select_statement)+ "align"i align_clause  where? order_by?  limit? 
+    multi_select_statement: select_statement ("merge" select_statement)+ "align"i align_clause ("derive" derive_clause)?  where? order_by?  limit? 
 
     align_item: IDENTIFIER ":" IDENTIFIER ("," IDENTIFIER)*  ","?
 
     align_clause: align_item ("AND"i align_item)*  "AND"i?
 
+    derive_item: expr "->" IDENTIFIER
+
+    derive_clause: derive_item ("," derive_item)* ","?
+
     merge_statement: "merge"i  WILDCARD_IDENTIFIER "into"i SHORTHAND_MODIFIER? WILDCARD_IDENTIFIER
 
     // raw sql statement
@@ -113,7 +117,7 @@
     filter_item: _filter_base | _filter_alt
 
     // rank/lag/lead
-    WINDOW_TYPE: ("row_number"i|"rank"i|"lag"i|"lead"i | "sum"i | "avg"i | "max"i | "min"i )  /[\s]+/
+    WINDOW_TYPE: ("row_number"i|"rank"i|"lag"i|"lead"i | "sum"i | "avg"i | "max"i | "min"i | "count"i  )  /[\s]+/
 
     window_item_over: ("OVER"i over_list)
    
@@ -308,8 +312,11 @@
     fregexp_contains: _REGEXP_CONTAINS expr "," expr  ")"
     _REGEXP_REPLACE.1: "regexp_replace("
     fregexp_replace: _REGEXP_REPLACE expr "," expr "," expr   ")"
+    _HASH.1: "hash("
+    HASH_TYPE: "md5"i | "sha1"i | "sha256"i | "sha512"i
+    fhash: _HASH expr "," HASH_TYPE ")"
 
-    _string_functions: like | ilike | upper | flower | fsplit | fstrpos | fsubstring | fcontains | ftrim | freplace | fregexp_extract | fregexp_contains | fregexp_replace
+    _string_functions: like | ilike | upper | flower | fsplit | fstrpos | fsubstring | fcontains | ftrim | freplace | fregexp_extract | fregexp_contains | fregexp_replace | fhash
 
     //array_functions
     _ARRAY_SUM.1: "array_sum("i
@@ -323,8 +330,10 @@
     _ARRAY_TRANSFORM.1: "array_transform("i
     transform_lambda: "@" IDENTIFIER
     farray_transform: _ARRAY_TRANSFORM expr ","  transform_lambda   ")"
+    _ARRAY_FILTER.1: "array_filter("i
+    farray_filter: _ARRAY_FILTER expr "," transform_lambda ")"
 
-    _array_functions: farray_sum | farray_distinct | farray_sort | farray_transform | farray_to_string
+    _array_functions: farray_sum | farray_distinct | farray_sort | farray_transform | farray_to_string | farray_filter
 
     //map_functions
     _MAP_KEYS.1: "map_keys("i
@@ -358,11 +367,13 @@
     min: _MIN expr ")"
     _ARRAY_AGG.1: "array_agg("i
     array_agg: _ARRAY_AGG expr ")"
+    _ANY.1: "any("i
+    any: _ANY expr ")"
     
     //aggregates can force a grain
     aggregate_all: "*"
     aggregate_over: ("BY"i (aggregate_all | over_list))
-    aggregate_functions: (count | count_distinct | sum | avg | max | min | array_agg) aggregate_over?
+    aggregate_functions: (count | count_distinct | sum | avg | max | min | array_agg | any) aggregate_over?
 
     // date functions
     _DATE.1: "date("i
@@ -405,8 +416,10 @@
     fdate_sub: _DATE_SUB expr "," DATE_PART "," expr ")"
     _DATE_DIFF.1: "date_diff("i
     fdate_diff: _DATE_DIFF expr "," expr "," DATE_PART ")"
+    _DATE_SPINE.1: "date_spine("i
+    fdate_spine: _DATE_SPINE expr "," expr  ")"
     
-    _date_functions: fdate | fdate_add | fdate_sub | fdate_diff | fdatetime | ftimestamp | fsecond | fminute | fhour | fday |fday_name | fday_of_week | fweek | fmonth | fmonth_name | fquarter | fyear | fdate_part | fdate_trunc
+    _date_functions: fdate | fdate_add | fdate_sub | fdate_diff | fdatetime | ftimestamp | fsecond | fminute | fhour | fday |fday_name | fday_of_week | fweek | fmonth | fmonth_name | fquarter | fyear | fdate_part | fdate_trunc | fdate_spine
     
     _static_functions:  _string_functions | _math_functions | _array_functions | _map_functions
 
@@ -434,7 +447,7 @@
     
     float_lit: /\-?[0-9]*\.[0-9]+/
 
-    array_lit: "[" (literal ",")* literal ","? "]"()
+    array_lit: "[" (expr ",")* expr ","? "]"()
 
     tuple_lit: "(" (literal ",")* literal ","? ")"