pytrilogy 0.0.2.12__py3-none-any.whl → 0.0.2.14__py3-none-any.whl

trilogy/core/processing/nodes/select_node_v2.py

@@ -43,6 +43,7 @@ class SelectNode(StrategyNode):
         parents: List["StrategyNode"] | None = None,
         depth: int = 0,
         partial_concepts: List[Concept] | None = None,
+        nullable_concepts: List[Concept] | None = None,
         accept_partial: bool = False,
         grain: Optional[Grain] = None,
         force_group: bool | None = False,
@@ -58,6 +59,7 @@ class SelectNode(StrategyNode):
             parents=parents,
             depth=depth,
             partial_concepts=partial_concepts,
+            nullable_concepts=nullable_concepts,
             force_group=force_group,
             grain=grain,
             conditions=conditions,
@@ -115,6 +117,7 @@ class SelectNode(StrategyNode):
             partial_concepts=[
                 c.concept for c in datasource.columns if not c.is_complete
             ],
+            nullable_concepts=[c.concept for c in datasource.columns if c.is_nullable],
             source_type=SourceType.DIRECT_SELECT,
             condition=self.conditions,
             # select nodes should never group
@@ -183,6 +186,7 @@ class SelectNode(StrategyNode):
             parents=self.parents,
             whole_grain=self.whole_grain,
             partial_concepts=list(self.partial_concepts),
+            nullable_concepts=list(self.nullable_concepts),
             accept_partial=self.accept_partial,
             grain=self.grain,
             force_group=self.force_group,

trilogy/core/processing/utility.py

@@ -18,9 +18,11 @@ from trilogy.core.models import (
     WindowItem,
     AggregateWrapper,
     DataType,
+    ConceptPair,
+    UnnestJoin,
 )
 
-from trilogy.core.enums import Purpose, Granularity, BooleanOperator
+from trilogy.core.enums import Purpose, Granularity, BooleanOperator, Modifier
 from trilogy.core.constants import CONSTANT_DATASET
 from enum import Enum
 from trilogy.utility import unique
@@ -243,8 +245,23 @@ def get_node_joins(
             local_concepts = [
                 c for c in local_concepts if c.granularity != Granularity.SINGLE_ROW
             ]
-        else:
+        elif any(
+            [
+                c.address in [x.address for x in identifier_map[right].partial_concepts]
+                for c in local_concepts
+            ]
+        ) or any(
+            [
+                c.address in [x.address for x in identifier_map[left].nullable_concepts]
+                for c in local_concepts
+            ]
+        ):
             join_type = JoinType.LEFT_OUTER
+            local_concepts = [
+                c for c in local_concepts if c.granularity != Granularity.SINGLE_ROW
+            ]
+        else:
+            join_type = JoinType.INNER
             # remove any constants if other join keys exist
             local_concepts = [
                 c for c in local_concepts if c.granularity != Granularity.SINGLE_ROW
@@ -287,7 +304,18 @@ def get_node_joins(
                     )
             narg = (left_arg, right_arg)
             if narg not in join_tuples:
-                join_tuples.append((left_arg, right_arg))
+                modifiers = set()
+                if left_arg.address in [
+                    x.address for x in left_datasource.nullable_concepts
+                ] and right_arg.address in [
+                    x.address for x in right_datasource.nullable_concepts
+                ]:
+                    modifiers.add(Modifier.NULLABLE)
+                join_tuples.append(
+                    ConceptPair(
+                        left=left_arg, right=right_arg, modifiers=list(modifiers)
+                    )
+                )
         final_joins_pre.append(
             BaseJoin(
                 left_datasource=identifier_map[left],
@@ -412,3 +440,63 @@ def decompose_condition(
     else:
         chunks.append(conditional)
     return chunks
+
+
+def find_nullable_concepts(
+    source_map: Dict[str, set[Datasource | QueryDatasource | UnnestJoin]],
+    datasources: List[Datasource | QueryDatasource],
+    joins: List[BaseJoin | UnnestJoin],
+) -> List[str]:
+    """give a set of datasources and joins, find the concepts
+    that may contain nulls in the output set
+    """
+    nullable_datasources = set()
+    datasource_map = {
+        x.identifier: x
+        for x in datasources
+        if isinstance(x, (Datasource, QueryDatasource))
+    }
+    for join in joins:
+        is_on_nullable_condition = False
+        if not isinstance(join, BaseJoin):
+            continue
+        if not join.concept_pairs:
+            continue
+        for pair in join.concept_pairs:
+            if pair.right.address in [
+                y.address
+                for y in datasource_map[
+                    join.right_datasource.identifier
+                ].nullable_concepts
+            ]:
+                is_on_nullable_condition = True
+                break
+            if pair.left.address in [
+                y.address
+                for y in datasource_map[
+                    join.left_datasource.identifier
+                ].nullable_concepts
+            ]:
+                is_on_nullable_condition = True
+                break
+        if is_on_nullable_condition:
+            nullable_datasources.add(datasource_map[join.right_datasource.identifier])
+    final_nullable = set()
+
+    for k, v in source_map.items():
+        local_nullable = [
+            x for x in datasources if k in [v.address for v in x.nullable_concepts]
+        ]
+        if all(
+            [
+                k in [v.address for v in x.nullable_concepts]
+                for x in datasources
+                if k in [z.address for z in x.output_concepts]
+            ]
+        ):
+            final_nullable.add(k)
+        all_ds = set([ds for ds in local_nullable]).union(nullable_datasources)
+        if nullable_datasources:
+            if set(v).issubset(all_ds):
+                final_nullable.add(k)
+    return list(sorted(final_nullable))

trilogy/core/query_processor.py

@@ -35,7 +35,6 @@ from trilogy.core.ergonomics import CTE_NAMES
 from trilogy.core.optimization import optimize_ctes
 from math import ceil
 from collections import defaultdict
-from random import shuffle
 
 LOGGER_PREFIX = "[QUERY BUILD]"
 
@@ -128,8 +127,6 @@ def generate_source_map(
                 if qdk in output_address:
                     source_map[qdk].append(cte.name)
             # now do a pass that accepts partials
-            # TODO: move this into a second loop by first creationg all sub sources
-            # then loop through this
             for cte in matches:
                 if qdk not in source_map:
                     source_map[qdk] = [cte.name]
@@ -180,7 +177,6 @@ def generate_cte_name(full_name: str, name_map: dict[str, str]) -> str:
             int = ceil(idx / len(CTE_NAMES))
             suffix = f"_{int}"
         valid = [x for x in CTE_NAMES if x + suffix not in name_map.values()]
-        shuffle(valid)
         lookup = valid[0]
         new_name = f"{lookup}{suffix}"
         name_map[full_name] = new_name
@@ -196,8 +192,6 @@ def resolve_cte_base_name_and_alias_v2(
     raw_joins: List[Join | InstantiatedUnnestJoin],
 ) -> Tuple[str | None, str | None]:
     joins: List[Join] = [join for join in raw_joins if isinstance(join, Join)]
-    # INFO     trilogy:query_processor.py:263 Finished building source map for civet with 3 parents, have {'local.relevant_customers': ['fowl', 'fowl'],
-    # 'customer.demographics.gender': ['mandrill'], 'customer.id': ['mandrill'], 'customer.demographics.id': ['mandrill'], 'customer.id_9268029262289908': [], 'customer.demographics.gender_1513806568509111': []}, query_datasource had non-empty keys ['local.relevant_customers', 'customer.demographics.gender', 'customer.id', 'customer.demographics.id'] and existence had non-empty keys []
     if (
         len(source.datasources) == 1
         and isinstance(source.datasources[0], Datasource)
@@ -301,6 +295,7 @@ def datasource_to_ctes(
         parent_ctes=parents,
         condition=query_datasource.condition,
         partial_concepts=query_datasource.partial_concepts,
+        nullable_concepts=query_datasource.nullable_concepts,
         join_derived_concepts=query_datasource.join_derived_concepts,
         hidden_concepts=query_datasource.hidden_concepts,
         base_name_override=base_name,
@@ -334,12 +329,13 @@ def append_existence_check(
         for subselect in where.existence_arguments:
             if not subselect:
                 continue
-            logger.info(
-                f"{LOGGER_PREFIX} fetching existance clause inputs {[str(c) for c in subselect]}"
-            )
+
             eds = source_query_concepts(
                 [*subselect], environment=environment, g=graph, history=history
             )
+            logger.info(
+                f"{LOGGER_PREFIX} fetching existence clause inputs {[str(c) for c in subselect]}"
+            )
             node.add_parents([eds])
             node.add_existence_concepts([*subselect])
 
@@ -384,9 +380,7 @@ def get_query_node(
     if nest_where and statement.where_clause:
         if not all_aggregate:
             ods.conditions = statement.where_clause.conditional
-        ods.output_concepts = statement.output_components
-        # ods.hidden_concepts = where_delta
-        ods.rebuild_cache()
+        ods.set_output_concepts(statement.output_components)
         append_existence_check(ods, environment, graph, history)
         ds = GroupNode(
             output_concepts=statement.output_components,

trilogy/dialect/common.py

@@ -3,9 +3,9 @@ from trilogy.core.enums import UnnestMode, Modifier
 from typing import Optional, Callable
 
 
-def null_wrapper(lval: str, rval: str, concept: Concept) -> str:
-    if concept.modifiers and Modifier.NULLABLE in concept.modifiers:
-        return f"(({lval} is null and {rval} is null) or ({lval} = {rval}))"
+def null_wrapper(lval: str, rval: str, modifiers: list[Modifier]) -> str:
+    if Modifier.NULLABLE in modifiers:
+        return f"({lval} = {rval} or ({lval} is null and {rval} is null))"
     return f"{lval} = {rval}"
 
 
@@ -48,7 +48,7 @@ def render_join(
         null_wrapper(
             f"{left_name}.{quote_character}{join.left_cte.get_alias(key.concept) if isinstance(join.left_cte, Datasource) else key.concept.safe_address}{quote_character}",
             f"{right_name}.{quote_character}{join.right_cte.get_alias(key.concept) if isinstance(join.right_cte, Datasource) else key.concept.safe_address}{quote_character}",
-            key.concept,
+            modifiers=key.concept.modifiers or [],
         )
         for key in join.joinkeys
     ]
@@ -56,11 +56,13 @@ def render_join(
         base_joinkeys.extend(
             [
                 null_wrapper(
-                    f"{left_name}.{quote_character}{join.left_cte.get_alias(left_concept) if isinstance(join.left_cte, Datasource) else left_concept.safe_address}{quote_character}",
-                    f"{right_name}.{quote_character}{join.right_cte.get_alias(right_concept) if isinstance(join.right_cte, Datasource) else right_concept.safe_address}{quote_character}",
-                    left_concept,
+                    f"{left_name}.{quote_character}{join.left_cte.get_alias(pair.left) if isinstance(join.left_cte, Datasource) else pair.left.safe_address}{quote_character}",
+                    f"{right_name}.{quote_character}{join.right_cte.get_alias(pair.right) if isinstance(join.right_cte, Datasource) else pair.right.safe_address}{quote_character}",
+                    modifiers=pair.modifiers
+                    + (pair.left.modifiers or [])
+                    + (pair.right.modifiers or []),
                 )
-                for left_concept, right_concept in join.joinkey_pairs
+                for pair in join.joinkey_pairs
             ]
         )
     if not base_joinkeys:

trilogy/executor.py

@@ -300,10 +300,16 @@ class Executor(object):
                 self.environment.add_datasource(x.datasource)
             yield x
 
-    def execute_raw_sql(self, command: str) -> CursorResult:
+    def execute_raw_sql(
+        self, command: str, variables: dict | None = None
+    ) -> CursorResult:
         """Run a command against the raw underlying
         execution engine"""
-        return self.connection.execute(text(command))
+        if variables:
+            return self.connection.execute(text(command), variables)
+        return self.connection.execute(
+            text(command),
+        )
 
     def execute_text(self, command: str) -> List[CursorResult]:
         """Run a preql text command"""

trilogy/parsing/common.py

@@ -25,6 +25,15 @@ from trilogy.core.enums import PurposeLineage
 from trilogy.constants import (
     VIRTUAL_CONCEPT_PREFIX,
 )
+from trilogy.core.enums import Modifier
+
+
+def get_upstream_modifiers(keys: List[Concept]) -> list[Modifier]:
+    modifiers = set()
+    for pkey in keys:
+        if pkey.modifiers:
+            modifiers.update(pkey.modifiers)
+    return list(modifiers)
 
 
 def process_function_args(
@@ -50,7 +59,7 @@ def process_function_args(
             id_hash = string_to_hash(str(arg))
             concept = function_to_concept(
                 arg,
-                name=f"{VIRTUAL_CONCEPT_PREFIX}_{id_hash}",
+                name=f"{VIRTUAL_CONCEPT_PREFIX}_{arg.operator.value}_{id_hash}",
                 namespace=environment.namespace,
             )
             # to satisfy mypy, concept will always have metadata
@@ -125,7 +134,7 @@ def constant_to_concept(
 
 
 def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
-    pkeys = []
+    pkeys: List[Concept] = []
     for x in parent.arguments:
         pkeys += [
             x
@@ -135,7 +144,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
     grain = Grain()
     for x in pkeys:
         grain += x.grain
-
+    modifiers = get_upstream_modifiers(pkeys)
     key_grain = []
     for x in pkeys:
         if x.keys:
@@ -155,6 +164,7 @@ def function_to_concept(parent: Function, name: str, namespace: str) -> Concept:
         namespace=namespace,
         grain=grain,
         keys=keys,
+        modifiers=modifiers,
     )
 
 
@@ -166,6 +176,7 @@ def filter_item_to_concept(
     metadata: Metadata | None = None,
 ) -> Concept:
     fmetadata = metadata or Metadata()
+    modifiers = get_upstream_modifiers(parent.content.concept_arguments)
     return Concept(
         name=name,
         datatype=parent.content.datatype,
@@ -184,6 +195,7 @@ def filter_item_to_concept(
             if parent.content.purpose == Purpose.PROPERTY
             else Grain()
         ),
+        modifiers=modifiers,
     )
 
 
@@ -202,6 +214,7 @@ def window_item_to_concept(
             grain += [item.expr.output]
     else:
         grain = parent.over + [parent.content.output]
+    modifiers = get_upstream_modifiers(parent.content.concept_arguments)
     return Concept(
         name=name,
         datatype=parent.content.datatype,
@@ -212,6 +225,7 @@ def window_item_to_concept(
         grain=Grain(components=grain),
         namespace=namespace,
         keys=keys,
+        modifiers=modifiers,
     )
 
 
@@ -229,6 +243,7 @@ def agg_wrapper_to_concept(
     # at that grain
     fmetadata = metadata or Metadata()
     aggfunction = parent.function
+    modifiers = get_upstream_modifiers(parent.concept_arguments)
     out = Concept(
         name=name,
         datatype=aggfunction.output_datatype,
@@ -238,6 +253,7 @@ def agg_wrapper_to_concept(
         grain=Grain(components=parent.by) if parent.by else Grain(),
         namespace=namespace,
         keys=tuple(parent.by) if parent.by else keys,
+        modifiers=modifiers,
     )
     return out
 
@@ -255,20 +271,34 @@ def arbitrary_to_concept(
         | str
     ),
     namespace: str,
-    name: str,
+    name: str | None = None,
     metadata: Metadata | None = None,
     purpose: Purpose | None = None,
 ) -> Concept:
 
     if isinstance(parent, AggregateWrapper):
+        if not name:
+            name = (
+                f"_agg_{parent.function.operator.value}_{string_to_hash(str(parent))}"
+            )
         return agg_wrapper_to_concept(parent, namespace, name, metadata, purpose)
     elif isinstance(parent, WindowItem):
+        if not name:
+            name = f"_window_{parent.type.value}_{string_to_hash(str(parent))}"
         return window_item_to_concept(parent, name, namespace, purpose, metadata)
     elif isinstance(parent, FilterItem):
+        if not name:
+            name = f"_filter_{parent.content.name}_{string_to_hash(str(parent))}"
         return filter_item_to_concept(parent, name, namespace, purpose, metadata)
     elif isinstance(parent, Function):
+        if not name:
+            name = f"_func_{parent.operator.value}_{string_to_hash(str(parent))}"
         return function_to_concept(parent, name, namespace)
     elif isinstance(parent, ListWrapper):
+        if not name:
+            name = f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"
         return constant_to_concept(parent, name, namespace, purpose, metadata)
     else:
+        if not name:
+            name = f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(parent))}"
         return constant_to_concept(parent, name, namespace, purpose, metadata)

trilogy/parsing/parse_engine.py

@@ -16,7 +16,6 @@ from trilogy.core.internal import INTERNAL_NAMESPACE, ALL_ROWS_CONCEPT
 from trilogy.constants import (
     DEFAULT_NAMESPACE,
     NULL_VALUE,
-    VIRTUAL_CONCEPT_PREFIX,
     MagicConstants,
 )
 from trilogy.core.enums import (
@@ -93,6 +92,7 @@ from trilogy.core.models import (
     WindowItemOver,
     RawColumnExpr,
     arg_to_datatype,
+    merge_datatypes,
     ListWrapper,
     MapWrapper,
     MapType,
@@ -109,7 +109,6 @@ from trilogy.core.models import (
     HavingClause,
 )
 from trilogy.parsing.exceptions import ParseError
-from trilogy.utility import string_to_hash
 from trilogy.parsing.common import (
     agg_wrapper_to_concept,
     window_item_to_concept,
@@ -739,8 +738,8 @@ class ParseToObjects(Transformer):
                 x = arbitrary_to_concept(
                     x,
                     namespace=namespace,
-                    name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(x))}",
                 )
+                self.environment.add_concept(x)
             return x
 
         return [
@@ -781,6 +780,11 @@ class ParseToObjects(Transformer):
     def rawsql_statement(self, meta: Meta, args) -> RawSQLStatement:
         return RawSQLStatement(meta=Metadata(line_number=meta.line), text=args[0])
 
+    def resolve_import_address(self, address) -> str:
+        with open(address, "r", encoding="utf-8") as f:
+            text = f.read()
+        return text
+
     def import_statement(self, args: list[str]) -> ImportStatement:
         alias = args[-1]
         path = args[0].split(".")
@@ -790,8 +794,7 @@ class ParseToObjects(Transformer):
             nparser = self.parsed[target]
         else:
             try:
-                with open(target, "r", encoding="utf-8") as f:
-                    text = f.read()
+                text = self.resolve_import_address(target)
                 nparser = ParseToObjects(
                     visit_tokens=True,
                     text=text,
@@ -1093,7 +1096,6 @@ class ParseToObjects(Transformer):
             left = arbitrary_to_concept(
                 args[0],
                 namespace=self.environment.namespace,
-                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(args[0]))}",
             )
             self.environment.add_concept(left)
         else:
@@ -1102,7 +1104,6 @@ class ParseToObjects(Transformer):
             right = arbitrary_to_concept(
                 args[2],
                 namespace=self.environment.namespace,
-                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(args[2]))}",
             )
             self.environment.add_concept(right)
         else:
@@ -1137,7 +1138,6 @@ class ParseToObjects(Transformer):
             right = arbitrary_to_concept(
                 right,
                 namespace=self.environment.namespace,
-                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(right))}",
             )
             self.environment.add_concept(right, meta=meta)
         return SubselectComparison(
@@ -1186,8 +1186,9 @@ class ParseToObjects(Transformer):
     def window_item_order(self, args):
         return WindowItemOrder(contents=args[0])
 
-    def window_item(self, args) -> WindowItem:
-        type = args[0]
+    @v_args(meta=True)
+    def window_item(self, meta, args) -> WindowItem:
+        type: WindowType = args[0]
         order_by = []
         over = []
         index = None
@@ -1203,6 +1204,14 @@ class ParseToObjects(Transformer):
                 concept = self.environment.concepts[item]
             elif isinstance(item, Concept):
                 concept = item
+            elif isinstance(item, WindowType):
+                type = item
+            else:
+                concept = arbitrary_to_concept(
+                    item,
+                    namespace=self.environment.namespace,
+                )
+                self.environment.add_concept(concept, meta=meta)
         assert concept
         return WindowItem(
             type=type, content=concept, over=over, order_by=order_by, index=index
@@ -1697,8 +1706,7 @@ class ParseToObjects(Transformer):
     @v_args(meta=True)
     def fadd(self, meta, args) -> Function:
         args = process_function_args(args, meta=meta, environment=self.environment)
-        output_datatype = arg_to_datatype(args[0])
-        # TODO: check for valid transforms?
+        output_datatype = merge_datatypes([arg_to_datatype(x) for x in args])
         return Function(
             operator=FunctionType.ADD,
             arguments=args,
@@ -1711,7 +1719,7 @@ class ParseToObjects(Transformer):
     @v_args(meta=True)
     def fsub(self, meta, args) -> Function:
         args = process_function_args(args, meta=meta, environment=self.environment)
-        output_datatype = arg_to_datatype(args[0])
+        output_datatype = merge_datatypes([arg_to_datatype(x) for x in args])
         return Function(
             operator=FunctionType.SUBTRACT,
             arguments=args,
@@ -1724,7 +1732,7 @@ class ParseToObjects(Transformer):
     @v_args(meta=True)
     def fmul(self, meta, args) -> Function:
         args = process_function_args(args, meta=meta, environment=self.environment)
-        output_datatype = arg_to_datatype(args[0])
+        output_datatype = merge_datatypes([arg_to_datatype(x) for x in args])
         return Function(
             operator=FunctionType.MULTIPLY,
             arguments=args,
@@ -1736,8 +1744,8 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def fdiv(self, meta: Meta, args):
-        output_datatype = arg_to_datatype(args[0])
         args = process_function_args(args, meta=meta, environment=self.environment)
+        output_datatype = merge_datatypes([arg_to_datatype(x) for x in args])
         return Function(
             operator=FunctionType.DIVIDE,
             arguments=args,
@@ -1749,12 +1757,11 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def fmod(self, meta: Meta, args):
-        output_datatype = arg_to_datatype(args[0])
         args = process_function_args(args, meta=meta, environment=self.environment)
         return Function(
             operator=FunctionType.MOD,
             arguments=args,
-            output_datatype=output_datatype,
+            output_datatype=DataType.INTEGER,
             output_purpose=function_args_to_output_purpose(args),
             valid_inputs=[
                 {DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
@@ -1781,12 +1788,15 @@ class ParseToObjects(Transformer):
 
     def fcase(self, args: List[Union[CaseWhen, CaseElse]]):
         datatypes = set()
+        mapz = dict()
         for arg in args:
             output_datatype = arg_to_datatype(arg.expr)
-            datatypes.add(output_datatype)
+            if output_datatype != DataType.NULL:
+                datatypes.add(output_datatype)
+            mapz[str(arg.expr)] = output_datatype
         if not len(datatypes) == 1:
             raise SyntaxError(
-                f"All case expressions must have the same output datatype, got {datatypes}"
+                f"All case expressions must have the same output datatype, got {datatypes} from {mapz}"
             )
         return Function(
             operator=FunctionType.CASE,
@@ -1830,6 +1840,8 @@ def unpack_visit_error(e: VisitError):
         unpack_visit_error(e.orig_exc)
     elif isinstance(e.orig_exc, (UndefinedConceptException, ImportError)):
         raise e.orig_exc
+    elif isinstance(e.orig_exc, SyntaxError):
+        raise InvalidSyntaxException(str(e.orig_exc) + str(e.rule) + str(e.obj))
     elif isinstance(e.orig_exc, (ValidationError, TypeError)):
         raise InvalidSyntaxException(str(e.orig_exc) + str(e.rule) + str(e.obj))
     raise e

trilogy/parsing/trilogy.lark

@@ -43,7 +43,7 @@
     
     query: "query" MULTILINE_STRING
     
-    concept_assignment: SHORTHAND_MODIFIER? IDENTIFIER 
+    concept_assignment: SHORTHAND_MODIFIER* IDENTIFIER 
     
     //column_assignment
     //figure out if we want static
@@ -90,9 +90,9 @@
 
 
     // rank/lag/lead
-    WINDOW_TYPE: ("row_number"i|"rank"i|"lag"i|"lead"i | "sum"i)  /[\s]+/
+    WINDOW_TYPE: ("row_number"i|"rank"i|"lag"i|"lead"i | "sum"i | "avg"i | "max"i | "min"i )  /[\s]+/
     
-    window_item: WINDOW_TYPE int_lit? concept_lit  window_item_over? window_item_order?
+    window_item: WINDOW_TYPE int_lit? expr  window_item_over? window_item_order?
     
     window_item_over: ("OVER"i over_list)
     
@@ -150,7 +150,7 @@
 
     subselect_comparison: expr array_comparison (literal | _constant_functions | _string_functions | concept_lit | filter_item | window_item | unnest | fgroup |  expr_tuple | parenthetical )
     
-    expr_tuple: "("  expr ("," expr)+ ","?  ")"
+    expr_tuple: ("("  expr ("," expr)+ ","?  ")") | ("("  expr "," ")")
 
     parenthetical: "(" expr ")"
 
@@ -296,7 +296,7 @@
 
     MODIFIER: "Optional"i | "Partial"i | "Nullable"i
 
-    SHORTHAND_MODIFIER: "~"
+    SHORTHAND_MODIFIER: "~" | "?"
 
     struct_type: "struct"i "<" ((data_type | IDENTIFIER) ",")* (data_type | IDENTIFIER) ","? ">"