pytrilogy 0.0.2.57__py3-none-any.whl → 0.0.3.0__py3-none-any.whl

trilogy/parsing/parse_engine.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 from datetime import date, datetime
+from enum import Enum
 from os.path import dirname, join
 from pathlib import Path
 from re import IGNORECASE
@@ -26,8 +27,9 @@ from trilogy.core.enums import (
     ComparisonOperator,
     ConceptSource,
     DatePart,
+    Derivation,
     FunctionType,
-    InfiniteFunctionArgs,
+    Granularity,
     IOType,
     Modifier,
     Ordering,
@@ -38,97 +40,91 @@ from trilogy.core.enums import (
 )
 from trilogy.core.exceptions import InvalidSyntaxException, UndefinedConceptException
 from trilogy.core.functions import (
-    Abs,
-    AttrAccess,
-    Bool,
-    Coalesce,
-    Count,
-    CountDistinct,
     CurrentDate,
     CurrentDatetime,
-    Group,
-    IndexAccess,
-    IsNull,
-    MapAccess,
-    Max,
-    Min,
-    Split,
-    StrPos,
-    SubString,
-    Unnest,
-    function_args_to_output_purpose,
+    FunctionFactory,
 )
 from trilogy.core.internal import ALL_ROWS_CONCEPT, INTERNAL_NAMESPACE
-from trilogy.core.models import (
-    Address,
+from trilogy.core.models.author import (
     AggregateWrapper,
     AlignClause,
     AlignItem,
     CaseElse,
     CaseWhen,
-    ColumnAssignment,
     Comment,
     Comparison,
     Concept,
-    ConceptDeclarationStatement,
-    ConceptDerivation,
-    ConceptTransform,
+    ConceptRef,
     Conditional,
-    CopyStatement,
-    Datasource,
-    DataType,
-    Environment,
-    EnvironmentConceptDict,
+    Expr,
     FilterItem,
     Function,
     Grain,
     HavingClause,
-    ImportStatement,
-    Limit,
-    ListType,
-    ListWrapper,
-    MapType,
-    MapWrapper,
-    MergeStatementV2,
     Metadata,
-    MultiSelectStatement,
-    NumericType,
     OrderBy,
     OrderItem,
     Parenthetical,
-    PersistStatement,
-    Query,
-    RawColumnExpr,
-    RawSQLStatement,
-    RowsetDerivationStatement,
-    SelectItem,
-    SelectStatement,
-    ShowStatement,
-    StructType,
     SubselectComparison,
-    TupleWrapper,
     WhereClause,
     Window,
     WindowItem,
     WindowItemOrder,
     WindowItemOver,
+)
+from trilogy.core.models.core import (
+    DataType,
+    ListType,
+    ListWrapper,
+    MapType,
+    MapWrapper,
+    NumericType,
+    StructType,
+    TupleWrapper,
     arg_to_datatype,
     dict_to_map_wrapper,
     list_to_wrapper,
-    merge_datatypes,
     tuple_to_wrapper,
 )
+from trilogy.core.models.datasource import (
+    Address,
+    ColumnAssignment,
+    Datasource,
+    Query,
+    RawColumnExpr,
+)
+from trilogy.core.models.environment import Environment, Import
+from trilogy.core.statements.author import (
+    ConceptDeclarationStatement,
+    ConceptDerivationStatement,
+    ConceptTransform,
+    CopyStatement,
+    ImportStatement,
+    Limit,
+    MergeStatementV2,
+    MultiSelectStatement,
+    PersistStatement,
+    RawSQLStatement,
+    RowsetDerivationStatement,
+    SelectItem,
+    SelectStatement,
+    ShowStatement,
+)
 from trilogy.parsing.common import (
-    agg_wrapper_to_concept,
+    align_item_to_concept,
     arbitrary_to_concept,
     constant_to_concept,
-    filter_item_to_concept,
-    function_to_concept,
     process_function_args,
-    window_item_to_concept,
+    rowset_to_concepts,
 )
 from trilogy.parsing.exceptions import ParseError
 
+
+class ParsePass(Enum):
+    INITIAL = 1
+    VALIDATION = 2
+
+
 CONSTANT_TYPES = (int, float, str, bool, list, ListWrapper, MapWrapper)
 
 SELF_LABEL = "root"
@@ -188,27 +184,19 @@ def expr_to_boolean(
 
 
 def unwrap_transformation(
-    input: Union[
-        FilterItem,
-        WindowItem,
-        Concept,
-        Function,
-        AggregateWrapper,
-        int,
-        str,
-        float,
-        bool,
-    ],
+    input: Expr,
+    environment: Environment,
 ) -> Function | FilterItem | WindowItem | AggregateWrapper:
     if isinstance(input, Function):
         return input
     elif isinstance(input, AggregateWrapper):
         return input
-    elif isinstance(input, Concept):
+    elif isinstance(input, ConceptRef):
+        concept = environment.concepts[input.address]
         return Function(
             operator=FunctionType.ALIAS,
-            output_datatype=input.datatype,
-            output_purpose=input.purpose,
+            output_datatype=concept.datatype,
+            output_purpose=concept.purpose,
             arguments=[input],
         )
     elif isinstance(input, FilterItem):
@@ -216,7 +204,7 @@ def unwrap_transformation(
     elif isinstance(input, WindowItem):
         return input
     elif isinstance(input, Parenthetical):
-        return unwrap_transformation(input.content)
+        return unwrap_transformation(input.content, environment)
     else:
         return Function(
             operator=FunctionType.CONSTANT,
@@ -247,7 +235,8 @@ class ParseToObjects(Transformer):
         )
         # we do a second pass to pick up circular dependencies
         # after initial parsing
-        self.pass_count = 1
+        self.parse_pass = ParsePass.INITIAL
+        self.function_factory = FunctionFactory(self.environment)
 
     def set_text(self, text: str):
         self.text_lookup[self.token_address] = text
@@ -258,15 +247,15 @@ class ParseToObjects(Transformer):
         return results
 
     def prepare_parse(self):
-        self.pass_count = 1
+        self.parse_pass = ParsePass.INITIAL
         self.environment.concepts.fail_on_missing = False
         for _, v in self.parsed.items():
             v.prepare_parse()
 
     def hydrate_missing(self):
-        self.pass_count = 2
+        self.parse_pass = ParsePass.VALIDATION
         for k, v in self.parsed.items():
-            if v.pass_count == 2:
+            if v.parse_pass == ParsePass.VALIDATION:
                 continue
             v.hydrate_missing()
         reparsed = self.transform(self.tokens[self.token_address])
@@ -312,10 +301,23 @@ class ParseToObjects(Transformer):
     def QUOTED_IDENTIFIER(self, args) -> str:
         return args.value[1:-1]
 
+    # @v_args(meta=True)
+    # def concept_lit(self, meta: Meta, args) -> ConceptRef:
+    #     address = args[0]
+    #     return self.environment.concepts.__getitem__(address, meta.line)
+    #     return ConceptRef(address=address, line_no=meta.line)
     @v_args(meta=True)
-    def concept_lit(self, meta: Meta, args) -> Concept:
+    def concept_lit(self, meta: Meta, args) -> ConceptRef:
         address = args[0]
-        return self.environment.concepts.__getitem__(address, meta.line)
+        if "." not in address and self.environment.namespace == DEFAULT_NAMESPACE:
+            address = f"{DEFAULT_NAMESPACE}.{address}"
+        mapping = self.environment.concepts[address]
+        datatype = mapping.output_datatype
+        return ConceptRef(
+            address=mapping.address,
+            metadata=Metadata(line_number=meta.line),
+            datatype=datatype,
+        )
 
     def ADDRESS(self, args) -> Address:
         return Address(location=args.value, quoted=False)
@@ -399,7 +401,9 @@ class ParseToObjects(Transformer):
         resolved = self.environment.concepts.__getitem__(  # type: ignore
             key=concept, line_no=meta.line, file=self.token_address
         )
-        return ColumnAssignment(alias=alias, modifiers=modifiers, concept=resolved)
+        return ColumnAssignment(
+            alias=alias, modifiers=modifiers, concept=resolved.reference
+        )
 
     def _TERMINATOR(self, args):
         return None
@@ -489,14 +493,16 @@ class ParseToObjects(Transformer):
             metadata=metadata,
             namespace=namespace,
             modifiers=modifiers,
+            derivation=Derivation.ROOT,
+            granularity=Granularity.MULTI_ROW,
         )
         if concept.metadata:
             concept.metadata.line_number = meta.line
-        self.environment.add_concept(concept, meta=meta, force=True)
+        self.environment.add_concept(concept, meta=meta)
         return ConceptDeclarationStatement(concept=concept)
 
     @v_args(meta=True)
-    def concept_derivation(self, meta: Meta, args) -> ConceptDerivation:
+    def concept_derivation(self, meta: Meta, args) -> ConceptDerivationStatement:
         if len(args) > 3:
             metadata = args[3]
         else:
@@ -539,7 +545,7 @@ class ParseToObjects(Transformer):
             if concept.metadata:
                 concept.metadata.line_number = meta.line
             self.environment.add_concept(concept, meta=meta)
-            return ConceptDerivation(concept=concept)
+            return ConceptDerivationStatement(concept=concept)
 
         elif isinstance(source_value, CONSTANT_TYPES):
             concept = constant_to_concept(
@@ -552,7 +558,7 @@ class ParseToObjects(Transformer):
             if concept.metadata:
                 concept.metadata.line_number = meta.line
             self.environment.add_concept(concept, meta=meta)
-            return ConceptDerivation(concept=concept)
+            return ConceptDerivationStatement(concept=concept)
 
         raise SyntaxError(
             f"Received invalid type {type(args[2])} {args[2]} as input to select"
@@ -570,12 +576,11 @@ class ParseToObjects(Transformer):
             select=select,
             namespace=self.environment.namespace or DEFAULT_NAMESPACE,
         )
-        for new_concept in output.derived_concepts:
+        for new_concept in rowset_to_concepts(output, self.environment):
             if new_concept.metadata:
                 new_concept.metadata.line_number = meta.line
             # output.select.local_concepts[new_concept.address] = new_concept
             self.environment.add_concept(new_concept)
-
         return output
 
     @v_args(meta=True)
@@ -672,15 +677,14 @@ class ParseToObjects(Transformer):
             where=where,
             non_partial_for=non_partial_for,
         )
-        for column in columns:
-            column.concept = column.concept.with_grain(datasource.grain)
         if datasource.where:
             for x in datasource.where.concept_arguments:
                 if x.address not in datasource.output_concepts:
                     raise ValueError(
                         f"Datasource {name} where condition depends on concept {x.address} that does not exist on the datasource, line {meta.line}."
                     )
-        self.environment.add_datasource(datasource, meta=meta)
+        if self.parse_pass == ParsePass.VALIDATION:
+            self.environment.add_datasource(datasource, meta=meta)
         return datasource
 
     @v_args(meta=True)
@@ -694,39 +698,20 @@ class ParseToObjects(Transformer):
     @v_args(meta=True)
     def select_transform(self, meta: Meta, args) -> ConceptTransform:
         output: str = args[1]
-        transformation = unwrap_transformation(args[0])
+        transformation = unwrap_transformation(args[0], self.environment)
         lookup, namespace, output, parent = parse_concept_reference(
             output, self.environment
         )
 
         metadata = Metadata(line_number=meta.line, concept_source=ConceptSource.SELECT)
 
-        if isinstance(transformation, AggregateWrapper):
-            concept = agg_wrapper_to_concept(
-                transformation, namespace=namespace, name=output, metadata=metadata
-            )
-        elif isinstance(transformation, WindowItem):
-            concept = window_item_to_concept(
-                transformation, namespace=namespace, name=output, metadata=metadata
-            )
-        elif isinstance(transformation, FilterItem):
-            concept = filter_item_to_concept(
-                transformation, namespace=namespace, name=output, metadata=metadata
-            )
-        elif isinstance(transformation, CONSTANT_TYPES):
-            concept = constant_to_concept(
-                transformation, namespace=namespace, name=output, metadata=metadata
-            )
-        elif isinstance(transformation, Function):
-            concept = function_to_concept(
-                transformation,
-                namespace=namespace,
-                name=output,
-                metadata=metadata,
-                environment=self.environment,
-            )
-        else:
-            raise SyntaxError("Invalid transformation")
+        concept = arbitrary_to_concept(
+            transformation,
+            environment=self.environment,
+            namespace=namespace,
+            name=output,
+            metadata=metadata,
+        )
 
         return ConceptTransform(function=transformation, output=concept)
 
@@ -775,20 +760,10 @@ class ParseToObjects(Transformer):
             return Ordering(" ".join([base, "nulls", null_sort.lower()]))
         return Ordering(base)
 
-    def order_list(self, args):
-        def handle_order_item(x, namespace: str):
-            if not isinstance(x, Concept):
-                x = arbitrary_to_concept(
-                    x, namespace=namespace, environment=self.environment
-                )
-            return x
-
+    def order_list(self, args) -> List[OrderItem]:
         return [
             OrderItem(
-                expr=handle_order_item(
-                    x,
-                    self.environment.namespace,
-                ),
+                expr=x,
                 order=y,
             )
             for x, y in zip(args[::2], args[1::2])
@@ -801,7 +776,7 @@ class ParseToObjects(Transformer):
         return [x for x in args]
 
     @v_args(meta=True)
-    def merge_statement(self, meta: Meta, args) -> MergeStatementV2:
+    def merge_statement(self, meta: Meta, args) -> MergeStatementV2 | None:
         modifiers = []
         cargs: list[str] = []
         source_wildcard = None
@@ -817,12 +792,12 @@ class ParseToObjects(Transformer):
                 raise ValueError("Invalid merge, source is wildcard, target is not")
             source_wildcard = source[:-2]
             target_wildcard = target[:-2]
-            sources = [
+            sources: list[Concept] = [
                 v
                 for k, v in self.environment.concepts.items()
                 if v.namespace == source_wildcard
             ]
-            targets = {}
+            targets: dict[str, Concept] = {}
             for x in sources:
                 target = target_wildcard + "." + x.name
                 if target in self.environment.concepts:
@@ -831,19 +806,22 @@ class ParseToObjects(Transformer):
         else:
             sources = [self.environment.concepts[source]]
             targets = {sources[0].address: self.environment.concepts[target]}
-        new = MergeStatementV2(
-            sources=sources,
-            targets=targets,
-            modifiers=modifiers,
-            source_wildcard=source_wildcard,
-            target_wildcard=target_wildcard,
-        )
-        for source_c in new.sources:
-            self.environment.merge_concept(
-                source_c, targets[source_c.address], modifiers
+
+        if self.parse_pass == ParsePass.VALIDATION:
+            new = MergeStatementV2(
+                sources=sources,
+                targets=targets,
+                modifiers=modifiers,
+                source_wildcard=source_wildcard,
+                target_wildcard=target_wildcard,
             )
+            for source_c in new.sources:
+                self.environment.merge_concept(
+                    source_c, targets[source_c.address], modifiers
+                )
 
-        return new
+            return new
+        return None
 
     @v_args(meta=True)
     def rawsql_statement(self, meta: Meta, args) -> RawSQLStatement:
@@ -919,7 +897,9 @@ class ParseToObjects(Transformer):
                 ) from e
 
         imps = ImportStatement(alias=alias, path=Path(args[0]))
-        self.environment.add_import(alias, new_env, imps)
+        self.environment.add_import(
+            alias, new_env, Import(alias=alias, path=Path(args[0]))
+        )
         return imps
 
     @v_args(meta=True)
@@ -931,7 +911,7 @@ class ParseToObjects(Transformer):
         return ShowStatement(content=args[0])
 
     @v_args(meta=True)
-    def persist_statement(self, meta: Meta, args) -> PersistStatement:
+    def persist_statement(self, meta: Meta, args) -> PersistStatement | None:
         identifier: str = args[0]
         address: str = args[1]
         select: SelectStatement = args[2]
@@ -939,29 +919,31 @@ class ParseToObjects(Transformer):
             grain: Grain | None = args[3]
         else:
             grain = None
-
-        new_datasource = select.to_datasource(
-            namespace=(
-                self.environment.namespace
-                if self.environment.namespace
-                else DEFAULT_NAMESPACE
-            ),
-            name=identifier,
-            address=Address(location=address),
-            grain=grain,
-        )
-        return PersistStatement(
-            select=select,
-            datasource=new_datasource,
-            meta=Metadata(line_number=meta.line),
-        )
+        if self.parse_pass == ParsePass.VALIDATION:
+            new_datasource = select.to_datasource(
+                namespace=(
+                    self.environment.namespace
+                    if self.environment.namespace
+                    else DEFAULT_NAMESPACE
+                ),
+                name=identifier,
+                address=Address(location=address),
+                grain=grain,
+                environment=self.environment,
+            )
+            return PersistStatement(
+                select=select,
+                datasource=new_datasource,
+                meta=Metadata(line_number=meta.line),
+            )
+        return None
 
     @v_args(meta=True)
     def align_item(self, meta: Meta, args) -> AlignItem:
         return AlignItem(
             alias=args[0],
             namespace=self.environment.namespace,
-            concepts=[self.environment.concepts[arg] for arg in args[1:]],
+            concepts=[self.environment.concepts[arg].reference for arg in args[1:]],
         )
 
     @v_args(meta=True)
@@ -970,11 +952,13 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def multi_select_statement(self, meta: Meta, args) -> MultiSelectStatement:
+
         selects: list[SelectStatement] = []
         align: AlignClause | None = None
         limit: int | None = None
         order_by: OrderBy | None = None
         where: WhereClause | None = None
+        having: HavingClause | None = None
         for arg in args:
             if isinstance(arg, SelectStatement):
                 selects.append(arg)
@@ -984,15 +968,27 @@ class ParseToObjects(Transformer):
                 order_by = arg
             elif isinstance(arg, WhereClause):
                 where = arg
+            elif isinstance(arg, HavingClause):
+                having = arg
             elif isinstance(arg, AlignClause):
                 align = arg
 
         assert align
         assert align is not None
-        base_local: EnvironmentConceptDict = selects[0].local_concepts
-        for select in selects[1:]:
-            for k, v in select.local_concepts.items():
-                base_local[k] = v
+
+        derived_concepts = []
+        for x in align.items:
+            concept = align_item_to_concept(
+                x,
+                align,
+                selects,
+                where=where,
+                having=having,
+                limit=limit,
+                environment=self.environment,
+            )
+            derived_concepts.append(concept)
+            self.environment.add_concept(concept, meta=meta)
         multi = MultiSelectStatement(
             selects=selects,
             align=align,
@@ -1001,10 +997,8 @@ class ParseToObjects(Transformer):
             order_by=order_by,
             limit=limit,
             meta=Metadata(line_number=meta.line),
-            local_concepts=base_local,
+            derived_concepts=derived_concepts,
         )
-        for concept in multi.derived_concepts:
-            self.environment.add_concept(concept, meta=meta)
         return multi
 
     @v_args(meta=True)
@@ -1114,15 +1108,10 @@ class ParseToObjects(Transformer):
             return ""
         return args[0]
 
-    def struct_lit(self, args):
-        zipped = dict(zip(args[::2], args[1::2]))
-        types = [arg_to_datatype(x) for x in args[1::2]]
-        return Function(
-            operator=FunctionType.STRUCT,
-            output_datatype=StructType(fields=types, fields_map=zipped),
-            output_purpose=function_args_to_output_purpose(args),
-            arguments=args,
-            arg_count=-1,
+    @v_args(meta=True)
+    def struct_lit(self, meta, args):
+        return self.function_factory.create_function(
+            args, operator=FunctionType.STRUCT, meta=meta
         )
 
     def map_lit(self, args):
@@ -1137,19 +1126,21 @@ class ParseToObjects(Transformer):
         if args[1] == ComparisonOperator.IN:
             raise SyntaxError
         if isinstance(args[0], AggregateWrapper):
-            left = arbitrary_to_concept(
+            left_c = arbitrary_to_concept(
                 args[0],
                 environment=self.environment,
             )
-            self.environment.add_concept(left)
+            self.environment.add_concept(left_c)
+            left = left_c.reference
         else:
             left = args[0]
         if isinstance(args[2], AggregateWrapper):
-            right = arbitrary_to_concept(
+            right_c = arbitrary_to_concept(
                 args[2],
                 environment=self.environment,
             )
-            self.environment.add_concept(right)
+            self.environment.add_concept(right_c)
+            right = right_c.reference
         else:
             right = args[2]
         return Comparison(left=left, right=right, operator=args[1])
@@ -1185,8 +1176,9 @@ class ParseToObjects(Transformer):
         ):
             right = right.content
         if isinstance(right, (Function, FilterItem, WindowItem, AggregateWrapper)):
-            right = arbitrary_to_concept(right, environment=self.environment)
-            self.environment.add_concept(right, meta=meta)
+            right_concept = arbitrary_to_concept(right, environment=self.environment)
+            self.environment.add_concept(right_concept, meta=meta)
+            right = right_concept.reference
         return SubselectComparison(
             left=args[0],
             right=right,
@@ -1233,6 +1225,12 @@ class ParseToObjects(Transformer):
     def window_item_order(self, args):
         return WindowItemOrder(contents=args[0])
 
+    def logical_operator(self, args):
+        return BooleanOperator(args[0].value.lower())
+
+    def DATE_PART(self, args):
+        return DatePart(args.value)
+
     @v_args(meta=True)
     def window_item(self, meta, args) -> WindowItem:
         type: WindowType = args[0]
@@ -1249,8 +1247,8 @@ class ParseToObjects(Transformer):
                 over = item.contents
             elif isinstance(item, str):
                 concept = self.environment.concepts[item]
-            elif isinstance(item, Concept):
-                concept = item
+            elif isinstance(item, ConceptRef):
+                concept = self.environment.concepts[item.address]
             elif isinstance(item, WindowType):
                 type = item
             else:
@@ -1258,7 +1256,11 @@ class ParseToObjects(Transformer):
                 self.environment.add_concept(concept, meta=meta)
         assert concept
         return WindowItem(
-            type=type, content=concept, over=over, order_by=order_by, index=index
+            type=type,
+            content=concept.reference,
+            over=over,
+            order_by=order_by,
+            index=index,
         )
 
     def filter_item(self, args) -> FilterItem:
@@ -1268,7 +1270,7 @@ class ParseToObjects(Transformer):
             where = raw
         else:
             where = WhereClause(conditional=raw)
-        concept = self.environment.concepts[string_concept]
+        concept = self.environment.concepts[string_concept].reference
         return FilterItem(content=concept, where=where)
 
     # BEGIN FUNCTIONS
@@ -1285,7 +1287,12 @@ class ParseToObjects(Transformer):
         return args[0]
 
     def aggregate_all(self, args):
-        return [self.environment.concepts[f"{INTERNAL_NAMESPACE}.{ALL_ROWS_CONCEPT}"]]
+        return [
+            ConceptRef(
+                address=f"{INTERNAL_NAMESPACE}.{ALL_ROWS_CONCEPT}",
+                datatype=DataType.INTEGER,
+            )
+        ]
 
     def aggregate_functions(self, args):
         if len(args) == 2:
@@ -1295,452 +1302,190 @@ class ParseToObjects(Transformer):
     @v_args(meta=True)
     def index_access(self, meta, args):
         args = process_function_args(args, meta=meta, environment=self.environment)
-        if args[0].datatype == DataType.MAP or isinstance(args[0].datatype, MapType):
-            return MapAccess(args)
-        return IndexAccess(args)
+        base = args[0]
+        if base.datatype == DataType.MAP or isinstance(base.datatype, MapType):
+            return self.function_factory.create_function(
+                args, FunctionType.MAP_ACCESS, meta
+            )
+        return self.function_factory.create_function(
+            args, FunctionType.INDEX_ACCESS, meta
+        )
 
     @v_args(meta=True)
     def map_key_access(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return MapAccess(args)
+        return self.function_factory.create_function(
+            args, FunctionType.MAP_ACCESS, meta
+        )
 
     @v_args(meta=True)
     def attr_access(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return AttrAccess(args)
+        return self.function_factory.create_function(
+            args, FunctionType.ATTR_ACCESS, meta
+        )
 
     @v_args(meta=True)
     def fcoalesce(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Coalesce(args)
+        return self.function_factory.create_function(args, FunctionType.COALESCE, meta)
 
     @v_args(meta=True)
     def unnest(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Unnest(args)
+        return self.function_factory.create_function(args, FunctionType.UNNEST, meta)
 
     @v_args(meta=True)
     def count(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Count(args)
+        return self.function_factory.create_function(args, FunctionType.COUNT, meta)
 
     @v_args(meta=True)
     def fgroup(self, meta, args):
         if len(args) == 2:
-            fargs = [args[0]] + args[1]
+            fargs = [args[0]] + list(args[1])
         else:
             fargs = [args[0]]
-        args = process_function_args(fargs, meta=meta, environment=self.environment)
-        return Group(args)
+        return self.function_factory.create_function(fargs, FunctionType.GROUP, meta)
 
     @v_args(meta=True)
     def fabs(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Abs(args)
+        return self.function_factory.create_function(args, FunctionType.ABS, meta)
 
     @v_args(meta=True)
     def count_distinct(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return CountDistinct(args)
+        return self.function_factory.create_function(
+            args, FunctionType.COUNT_DISTINCT, meta
+        )
 
     @v_args(meta=True)
     def sum(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.SUM,
-            arguments=args,
-            output_datatype=args[0].datatype,
-            output_purpose=Purpose.METRIC,
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.SUM, meta)
 
     @v_args(meta=True)
     def avg(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        arg = args[0]
-
-        return Function(
-            operator=FunctionType.AVG,
-            arguments=args,
-            output_datatype=arg.datatype,
-            output_purpose=Purpose.METRIC,
-            valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.AVG, meta)
 
     @v_args(meta=True)
     def max(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Max(args)
+        return self.function_factory.create_function(args, FunctionType.MAX, meta)
 
     @v_args(meta=True)
     def min(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Min(args)
+        return self.function_factory.create_function(args, FunctionType.MIN, meta)
 
     @v_args(meta=True)
     def len(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.LENGTH,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.STRING, DataType.ARRAY, DataType.MAP},
-            # output_grain=args[0].grain,
-        )
+        return self.function_factory.create_function(args, FunctionType.LENGTH, meta)
 
     @v_args(meta=True)
     def fsplit(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Split(args)
+        return self.function_factory.create_function(args, FunctionType.SPLIT, meta)
 
     @v_args(meta=True)
     def concat(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.CONCAT,
-            arguments=args,
-            output_datatype=DataType.STRING,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.STRING},
-            arg_count=-1,
-            # output_grain=args[0].grain,
-        )
+        return self.function_factory.create_function(args, FunctionType.CONCAT, meta)
 
     @v_args(meta=True)
     def union(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        output_datatype = merge_datatypes([arg_to_datatype(x) for x in args])
-        return Function(
-            operator=FunctionType.UNION,
-            arguments=args,
-            output_datatype=output_datatype,
-            output_purpose=Purpose.KEY,
-            valid_inputs={*DataType},
-            arg_count=-1,
-        )
+        return self.function_factory.create_function(args, FunctionType.UNION, meta)
 
     @v_args(meta=True)
     def like(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.LIKE,
-            arguments=args,
-            output_datatype=DataType.BOOL,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.STRING},
-            arg_count=2,
-        )
+        return self.function_factory.create_function(args, FunctionType.LIKE, meta)
 
     @v_args(meta=True)
     def alt_like(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.LIKE,
-            arguments=args,
-            output_datatype=DataType.BOOL,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.STRING},
-            arg_count=2,
-        )
+        return self.function_factory.create_function(args, FunctionType.LIKE, meta)
 
     @v_args(meta=True)
     def ilike(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.ILIKE,
-            arguments=args,
-            output_datatype=DataType.BOOL,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.STRING},
-            arg_count=2,
-        )
+        return self.function_factory.create_function(args, FunctionType.LIKE, meta)
 
     @v_args(meta=True)
     def upper(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.UPPER,
-            arguments=args,
-            output_datatype=DataType.STRING,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.STRING},
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.UPPER, meta)
 
     @v_args(meta=True)
     def fstrpos(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return StrPos(args)
+        return self.function_factory.create_function(args, FunctionType.STRPOS, meta)
 
     @v_args(meta=True)
     def fsubstring(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return SubString(args)
-
-    def logical_operator(self, args):
-        return BooleanOperator(args[0].value.lower())
+        return self.function_factory.create_function(args, FunctionType.SUBSTRING, meta)
 
     @v_args(meta=True)
     def lower(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.LOWER,
-            arguments=args,
-            output_datatype=DataType.STRING,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.STRING},
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.LOWER, meta)
 
     # date functions
     @v_args(meta=True)
     def fdate(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.DATE,
-            arguments=args,
-            output_datatype=DataType.DATE,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={
-                DataType.DATE,
-                DataType.TIMESTAMP,
-                DataType.DATETIME,
-                DataType.STRING,
-            },
-            arg_count=1,
-        )
-
-    def DATE_PART(self, args):
-        return DatePart(args.value)
+        return self.function_factory.create_function(args, FunctionType.DATE, meta)
 
     @v_args(meta=True)
     def fdate_trunc(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.DATE_TRUNCATE,
-            arguments=args,
-            output_datatype=DataType.DATE,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs=[
-                {
-                    DataType.DATE,
-                    DataType.TIMESTAMP,
-                    DataType.DATETIME,
-                    DataType.STRING,
-                },
-                {DataType.DATE_PART},
-            ],
-            arg_count=2,
+        return self.function_factory.create_function(
+            args, FunctionType.DATE_TRUNCATE, meta
         )
 
     @v_args(meta=True)
     def fdate_part(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.DATE_PART,
-            arguments=args,
-            output_datatype=DataType.DATE,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs=[
-                {
-                    DataType.DATE,
-                    DataType.TIMESTAMP,
-                    DataType.DATETIME,
-                    DataType.STRING,
-                },
-                {DataType.DATE_PART},
-            ],
-            arg_count=2,
-        )
+        return self.function_factory.create_function(args, FunctionType.DATE_PART, meta)
 
     @v_args(meta=True)
     def fdate_add(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.DATE_ADD,
-            arguments=args,
-            output_datatype=DataType.DATE,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs=[
-                {
-                    DataType.DATE,
-                    DataType.TIMESTAMP,
-                    DataType.DATETIME,
-                    DataType.STRING,
-                },
-                {DataType.DATE_PART},
-                {DataType.INTEGER},
-            ],
-            arg_count=3,
-        )
+        return self.function_factory.create_function(args, FunctionType.DATE_ADD, meta)
 
     @v_args(meta=True)
     def fdate_diff(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        purpose = function_args_to_output_purpose(args)
-        return Function(
-            operator=FunctionType.DATE_DIFF,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=purpose,
-            valid_inputs=[
-                {
-                    DataType.DATE,
-                    DataType.TIMESTAMP,
-                    DataType.DATETIME,
-                },
-                {
-                    DataType.DATE,
-                    DataType.TIMESTAMP,
-                    DataType.DATETIME,
-                },
-                {DataType.DATE_PART},
-            ],
-            arg_count=3,
-        )
+        return self.function_factory.create_function(args, FunctionType.DATE_DIFF, meta)
 
     @v_args(meta=True)
     def fdatetime(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.DATETIME,
-            arguments=args,
-            output_datatype=DataType.DATETIME,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={
-                DataType.DATE,
-                DataType.TIMESTAMP,
-                DataType.DATETIME,
-                DataType.STRING,
-            },
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.DATETIME, meta)
 
     @v_args(meta=True)
     def ftimestamp(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.TIMESTAMP,
-            arguments=args,
-            output_datatype=DataType.TIMESTAMP,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs=[{DataType.TIMESTAMP, DataType.STRING}],
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.TIMESTAMP, meta)
 
     @v_args(meta=True)
     def fsecond(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.SECOND,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.TIMESTAMP, DataType.DATETIME},
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.SECOND, meta)
 
     @v_args(meta=True)
     def fminute(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.MINUTE,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.TIMESTAMP, DataType.DATETIME},
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.MINUTE, meta)
 
     @v_args(meta=True)
     def fhour(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.HOUR,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.TIMESTAMP, DataType.DATETIME},
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.HOUR, meta)
 
     @v_args(meta=True)
     def fday(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.DAY,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.DATE, DataType.TIMESTAMP, DataType.DATETIME},
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.DAY, meta)
 
     @v_args(meta=True)
     def fday_of_week(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.DAY_OF_WEEK,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.DATE, DataType.TIMESTAMP, DataType.DATETIME},
-            arg_count=1,
+        return self.function_factory.create_function(
+            args, FunctionType.DAY_OF_WEEK, meta
         )
 
     @v_args(meta=True)
     def fweek(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.WEEK,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.DATE, DataType.TIMESTAMP, DataType.DATETIME},
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.WEEK, meta)
 
     @v_args(meta=True)
     def fmonth(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.MONTH,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.DATE, DataType.TIMESTAMP, DataType.DATETIME},
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.MONTH, meta)
 
     @v_args(meta=True)
     def fquarter(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.QUARTER,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.DATE, DataType.TIMESTAMP, DataType.DATETIME},
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.QUARTER, meta)
 
     @v_args(meta=True)
     def fyear(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.YEAR,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=Purpose.PROPERTY,
-            valid_inputs={DataType.DATE, DataType.TIMESTAMP, DataType.DATETIME},
-            arg_count=1,
-        )
+        return self.function_factory.create_function(args, FunctionType.YEAR, meta)
 
     # utility functions
     @v_args(meta=True)
     def fcast(self, meta, args) -> Function:
+        # if it's casting a constant, we'll process that directly
         args = process_function_args(args, meta=meta, environment=self.environment)
         if isinstance(args[0], str):
             processed: date | datetime | int | float | bool | str
@@ -1760,134 +1505,39 @@ class ParseToObjects(Transformer):
                 processed = args[0]
             else:
                 raise SyntaxError(f"Invalid cast type {args[1]}")
-            return Function(
-                operator=FunctionType.CONSTANT,
-                output_datatype=args[1],
-                output_purpose=Purpose.CONSTANT,
-                arguments=[processed],
+            return self.function_factory.create_function(
+                [processed], FunctionType.CONSTANT, meta
             )
-        output_datatype = args[1]
-        return Function(
-            operator=FunctionType.CAST,
-            arguments=args,
-            output_datatype=output_datatype,
-            output_purpose=function_args_to_output_purpose(args),
-            valid_inputs={
-                DataType.INTEGER,
-                DataType.STRING,
-                DataType.FLOAT,
-                DataType.NUMBER,
-                DataType.NUMERIC,
-                DataType.BOOL,
-            },
-            arg_count=2,
-        )
+        return self.function_factory.create_function(args, FunctionType.CAST, meta)
 
     # math functions
     @v_args(meta=True)
     def fadd(self, meta, args) -> Function:
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        output_datatype = merge_datatypes([arg_to_datatype(x) for x in args])
-        return Function(
-            operator=FunctionType.ADD,
-            arguments=args,
-            output_datatype=output_datatype,
-            output_purpose=function_args_to_output_purpose(args),
-            valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
-            arg_count=-1,
-        )
+        return self.function_factory.create_function(args, FunctionType.ADD, meta)
 
     @v_args(meta=True)
     def fsub(self, meta, args) -> Function:
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        output_datatype = merge_datatypes([arg_to_datatype(x) for x in args])
-        return Function(
-            operator=FunctionType.SUBTRACT,
-            arguments=args,
-            output_datatype=output_datatype,
-            output_purpose=function_args_to_output_purpose(args),
-            valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
-            arg_count=-1,
-        )
+        return self.function_factory.create_function(args, FunctionType.SUBTRACT, meta)
 
     @v_args(meta=True)
     def fmul(self, meta, args) -> Function:
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        output_datatype = merge_datatypes([arg_to_datatype(x) for x in args])
-        return Function(
-            operator=FunctionType.MULTIPLY,
-            arguments=args,
-            output_datatype=output_datatype,
-            output_purpose=function_args_to_output_purpose(args),
-            valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
-            arg_count=-1,
-        )
+        return self.function_factory.create_function(args, FunctionType.MULTIPLY, meta)
 
     @v_args(meta=True)
-    def fdiv(self, meta: Meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        # 2024-11-18 - this is a bit of a hack, but division always returns a float
-        # output_datatype = merge_datatypes([arg_to_datatype(x) for x in args])
-        return Function(
-            operator=FunctionType.DIVIDE,
-            arguments=args,
-            output_datatype=DataType.FLOAT,  # division always returns a float
-            output_purpose=function_args_to_output_purpose(args),
-            valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
-            arg_count=-1,
-        )
+    def fdiv(self, meta: Meta, args) -> Function:
+        return self.function_factory.create_function(args, FunctionType.DIVIDE, meta)
 
     @v_args(meta=True)
-    def fmod(self, meta: Meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Function(
-            operator=FunctionType.MOD,
-            arguments=args,
-            output_datatype=DataType.INTEGER,
-            output_purpose=function_args_to_output_purpose(args),
-            valid_inputs=[
-                {DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
-                {DataType.INTEGER},
-            ],
-            arg_count=2,
-        )
+    def fmod(self, meta: Meta, args) -> Function:
+        return self.function_factory.create_function(args, FunctionType.MOD, meta)
 
     @v_args(meta=True)
     def fround(self, meta, args) -> Function:
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        output_datatype = arg_to_datatype(args[0])
-        return Function(
-            operator=FunctionType.ROUND,
-            arguments=args,
-            output_datatype=output_datatype,
-            output_purpose=function_args_to_output_purpose(args),
-            valid_inputs=[
-                {DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
-                {DataType.INTEGER},
-            ],
-            arg_count=2,
-        )
+        return self.function_factory.create_function(args, FunctionType.ROUND, meta)
 
-    def fcase(self, args: List[Union[CaseWhen, CaseElse]]):
-        datatypes = set()
-        mapz = dict()
-        for arg in args:
-            output_datatype = arg_to_datatype(arg.expr)
-            if output_datatype != DataType.NULL:
-                datatypes.add(output_datatype)
-            mapz[str(arg.expr)] = output_datatype
-        if not len(datatypes) == 1:
-            raise SyntaxError(
-                f"All case expressions must have the same output datatype, got {datatypes} from {mapz}"
-            )
-        return Function(
-            operator=FunctionType.CASE,
-            arguments=args,
-            output_datatype=datatypes.pop(),
-            output_purpose=Purpose.PROPERTY,
-            # valid_inputs=[{DataType.INTEGER, DataType.FLOAT, DataType.NUMBER}, {DataType.INTEGER}],
-            arg_count=InfiniteFunctionArgs,
-        )
+    @v_args(meta=True)
+    def fcase(self, meta, args: List[Union[CaseWhen, CaseElse]]) -> Function:
+        return self.function_factory.create_function(args, FunctionType.CASE, meta)
 
     @v_args(meta=True)
     def fcase_when(self, meta, args) -> CaseWhen:
@@ -1902,23 +1552,19 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def fcurrent_date(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
         return CurrentDate([])
 
     @v_args(meta=True)
     def fcurrent_datetime(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
         return CurrentDatetime([])
 
     @v_args(meta=True)
     def fnot(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return IsNull(args)
+        return self.function_factory.create_function(args, FunctionType.IS_NULL, meta)
 
     @v_args(meta=True)
     def fbool(self, meta, args):
-        args = process_function_args(args, meta=meta, environment=self.environment)
-        return Bool(args)
+        return self.function_factory.create_function(args, FunctionType.BOOL, meta)
 
 
 def unpack_visit_error(e: VisitError):