pytrilogy 0.3.138__cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

trilogy/parsing/parse_engine.py

@@ -0,0 +1,2813 @@
+from dataclasses import dataclass
+from datetime import date, datetime
+from enum import Enum
+from logging import getLogger
+from os.path import dirname, join
+from pathlib import Path
+from re import IGNORECASE
+from typing import Any, List, Optional, Tuple, Union
+
+from lark import Lark, ParseTree, Token, Transformer, Tree, v_args
+from lark.exceptions import (
+    UnexpectedCharacters,
+    UnexpectedEOF,
+    UnexpectedInput,
+    UnexpectedToken,
+    VisitError,
+)
+from lark.tree import Meta
+from pydantic import ValidationError
+
+from trilogy.constants import (
+    CONFIG,
+    DEFAULT_NAMESPACE,
+    NULL_VALUE,
+    MagicConstants,
+    Parsing,
+)
+from trilogy.core.enums import (
+    BooleanOperator,
+    ComparisonOperator,
+    ConceptSource,
+    CreateMode,
+    DatasourceState,
+    DatePart,
+    Derivation,
+    FunctionType,
+    Granularity,
+    IOType,
+    Modifier,
+    Ordering,
+    PersistMode,
+    PublishAction,
+    Purpose,
+    ShowCategory,
+    ValidationScope,
+    WindowOrder,
+    WindowType,
+)
+from trilogy.core.exceptions import (
+    InvalidSyntaxException,
+    MissingParameterException,
+    UndefinedConceptException,
+)
+from trilogy.core.functions import (
+    CurrentDate,
+    FunctionFactory,
+)
+from trilogy.core.internal import ALL_ROWS_CONCEPT, INTERNAL_NAMESPACE
+from trilogy.core.models.author import (
+    AggregateWrapper,
+    AlignClause,
+    AlignItem,
+    ArgBinding,
+    CaseElse,
+    CaseWhen,
+    Comment,
+    Comparison,
+    Concept,
+    ConceptRef,
+    Conditional,
+    CustomFunctionFactory,
+    CustomType,
+    DeriveClause,
+    DeriveItem,
+    Expr,
+    FilterItem,
+    Function,
+    FunctionCallWrapper,
+    Grain,
+    HavingClause,
+    Metadata,
+    MultiSelectLineage,
+    OrderBy,
+    OrderItem,
+    Parenthetical,
+    RowsetItem,
+    SubselectComparison,
+    UndefinedConceptFull,
+    WhereClause,
+    Window,
+    WindowItem,
+    WindowItemOrder,
+    WindowItemOver,
+)
+from trilogy.core.models.core import (
+    ArrayType,
+    DataType,
+    DataTyped,
+    ListWrapper,
+    MapType,
+    MapWrapper,
+    NumericType,
+    StructComponent,
+    StructType,
+    TraitDataType,
+    TupleWrapper,
+    arg_to_datatype,
+    dict_to_map_wrapper,
+    is_compatible_datatype,
+    list_to_wrapper,
+    tuple_to_wrapper,
+)
+from trilogy.core.models.datasource import (
+    Address,
+    ColumnAssignment,
+    Datasource,
+    Query,
+    RawColumnExpr,
+)
+from trilogy.core.models.environment import (
+    DictImportResolver,
+    Environment,
+    FileSystemImportResolver,
+    Import,
+)
+from trilogy.core.statements.author import (
+    ConceptDeclarationStatement,
+    ConceptDerivationStatement,
+    ConceptTransform,
+    CopyStatement,
+    CreateStatement,
+    FunctionDeclaration,
+    ImportStatement,
+    Limit,
+    MergeStatementV2,
+    MockStatement,
+    MultiSelectStatement,
+    PersistStatement,
+    PublishStatement,
+    RawSQLStatement,
+    RowsetDerivationStatement,
+    SelectItem,
+    SelectStatement,
+    ShowStatement,
+    TypeDeclaration,
+    ValidateStatement,
+)
+from trilogy.parsing.common import (
+    align_item_to_concept,
+    arbitrary_to_concept,
+    constant_to_concept,
+    derive_item_to_concept,
+    process_function_args,
+    rowset_to_concepts,
+)
+from trilogy.parsing.exceptions import NameShadowError, ParseError
+
+perf_logger = getLogger("trilogy.parse.performance")
+
+
+class ParsePass(Enum):
+    INITIAL = 1
+    VALIDATION = 2
+
+
+CONSTANT_TYPES = (int, float, str, bool, ListWrapper, TupleWrapper, MapWrapper)
+
+SELF_LABEL = "root"
+
+MAX_PARSE_DEPTH = 10
+
+SUPPORTED_INCREMENTAL_TYPES: set[DataType] = set([DataType.DATE, DataType.TIMESTAMP])
+
+STDLIB_ROOT = Path(__file__).parent.parent
+
+
+@dataclass
+class WholeGrainWrapper:
+    where: WhereClause
+
+
+@dataclass
+class FunctionBindingType:
+    type: DataType | TraitDataType | None = None
+
+
+@dataclass
+class DropOn:
+    functions: List[FunctionType]
+
+
+@dataclass
+class AddOn:
+    functions: List[FunctionType]
+
+
+@dataclass
+class DatasourcePartitionClause:
+    columns: List[ConceptRef]
+
+
+@dataclass
+class DatasourceIncrementalClause:
+    columns: List[ConceptRef]
+
+
+with open(join(dirname(__file__), "trilogy.lark"), "r") as f:
+    PARSER = Lark(
+        f.read(),
+        start="start",
+        propagate_positions=True,
+        g_regex_flags=IGNORECASE,
+        parser="lalr",
+        cache=True,
+    )
+
+
+def parse_concept_reference(
+    name: str, environment: Environment, purpose: Optional[Purpose] = None
+) -> Tuple[str, str, str, str | None]:
+    parent = None
+    if "." in name:
+        if purpose == Purpose.PROPERTY:
+            parent, name = name.rsplit(".", 1)
+            namespace = environment.concepts[parent].namespace or DEFAULT_NAMESPACE
+            lookup = f"{namespace}.{name}"
+        else:
+            namespace, name = name.rsplit(".", 1)
+            lookup = f"{namespace}.{name}"
+    else:
+        namespace = environment.namespace or DEFAULT_NAMESPACE
+        lookup = name
+    return lookup, namespace, name, parent
+
+
+def expr_to_boolean(
+    root,
+    function_factory: FunctionFactory,
+) -> Union[Comparison, SubselectComparison, Conditional]:
+    if not isinstance(root, (Comparison, SubselectComparison, Conditional)):
+        if arg_to_datatype(root) == DataType.BOOL:
+            root = Comparison(left=root, right=True, operator=ComparisonOperator.EQ)
+        elif arg_to_datatype(root) == DataType.INTEGER:
+            root = Comparison(
+                left=function_factory.create_function(
+                    [root],
+                    FunctionType.BOOL,
+                ),
+                right=True,
+                operator=ComparisonOperator.EQ,
+            )
+        else:
+            root = Comparison(
+                left=root, right=NULL_VALUE, operator=ComparisonOperator.IS_NOT
+            )
+
+    return root
+
+
+def unwrap_transformation(
+    input: Expr,
+    environment: Environment,
+) -> (
+    Function
+    | FilterItem
+    | WindowItem
+    | AggregateWrapper
+    | FunctionCallWrapper
+    | Parenthetical
+):
+    if isinstance(input, Function):
+        return input
+    elif isinstance(input, AggregateWrapper):
+        return input
+    elif isinstance(input, ConceptRef):
+        concept = environment.concepts[input.address]
+        return Function(
+            operator=FunctionType.ALIAS,
+            output_datatype=concept.datatype,
+            output_purpose=concept.purpose,
+            arguments=[input],
+        )
+    elif isinstance(input, FilterItem):
+        return input
+    elif isinstance(input, WindowItem):
+        return input
+    elif isinstance(input, FunctionCallWrapper):
+        return input
+    elif isinstance(input, Parenthetical):
+        return input
+    else:
+        return Function.model_construct(
+            operator=FunctionType.CONSTANT,
+            output_datatype=arg_to_datatype(input),
+            output_purpose=Purpose.CONSTANT,
+            arguments=[input],
+        )
+
+
+def rehydrate_lineage(
+    lineage: Any, environment: Environment, function_factory: FunctionFactory
+) -> Any:
+    """Fix datatype propagation. This is a hack to fix the fact that we don't know the datatypes of functions until we've parsed all concepts"""
+    if isinstance(lineage, Function):
+        rehydrated = [
+            rehydrate_lineage(x, environment, function_factory)
+            for x in lineage.arguments
+        ]
+        return function_factory.create_function(
+            rehydrated,
+            operator=lineage.operator,
+        )
+    elif isinstance(lineage, Parenthetical):
+        lineage.content = rehydrate_lineage(
+            lineage.content, environment, function_factory
+        )
+        return lineage
+    elif isinstance(lineage, WindowItem):
+        # this is temporarily guaranteed until we do some upstream work
+        assert isinstance(lineage.content, ConceptRef)
+        lineage.content.datatype = environment.concepts[
+            lineage.content.address
+        ].datatype
+        return lineage
+    elif isinstance(lineage, AggregateWrapper):
+        lineage.function = rehydrate_lineage(
+            lineage.function, environment, function_factory
+        )
+        return lineage
+    elif isinstance(lineage, RowsetItem):
+        lineage.content.datatype = environment.concepts[
+            lineage.content.address
+        ].datatype
+        return lineage
+    else:
+        return lineage
+
+
+def rehydrate_concept_lineage(
+    concept: Concept, environment: Environment, function_factory: FunctionFactory
+) -> Concept:
+    concept.lineage = rehydrate_lineage(concept.lineage, environment, function_factory)
+    if isinstance(concept.lineage, DataTyped):
+        concept.datatype = concept.lineage.output_datatype
+    return concept
+
+
+class ParseToObjects(Transformer):
+    def __init__(
+        self,
+        environment: Environment,
+        parse_address: str | None = None,
+        token_address: Path | str | None = None,
+        parsed: dict[str, "ParseToObjects"] | None = None,
+        tokens: dict[Path | str, ParseTree] | None = None,
+        text_lookup: dict[Path | str, str] | None = None,
+        environment_lookup: dict[str, Environment] | None = None,
+        import_keys: list[str] | None = None,
+        parse_config: Parsing | None = None,
+    ):
+        Transformer.__init__(self, True)
+        self.environment: Environment = environment
+        self.parse_address: str = parse_address or SELF_LABEL
+        self.token_address: Path | str = token_address or SELF_LABEL
+        self.parsed: dict[str, ParseToObjects] = parsed if parsed is not None else {}
+        self.tokens: dict[Path | str, ParseTree] = tokens if tokens is not None else {}
+        self.environments: dict[str, Environment] = environment_lookup or {}
+        self.text_lookup: dict[Path | str, str] = (
+            text_lookup if text_lookup is not None else {}
+        )
+        # we do a second pass to pick up circular dependencies
+        # after initial parsing
+        self.parse_pass = ParsePass.INITIAL
+        self.function_factory = FunctionFactory(self.environment)
+        self.import_keys: list[str] = import_keys or ["root"]
+        self.parse_config: Parsing = parse_config or CONFIG.parsing
+
+    def set_text(self, text: str):
+        self.text_lookup[self.token_address] = text
+
+    def transform(self, tree: Tree):
+        results = super().transform(tree)
+        self.tokens[self.token_address] = tree
+        return results
+
+    def prepare_parse(self):
+        self.parse_pass = ParsePass.INITIAL
+        self.environment.concepts.fail_on_missing = False
+        for _, v in self.parsed.items():
+            v.prepare_parse()
+
+    def run_second_parse_pass(self, force: bool = False):
+        if self.token_address not in self.tokens:
+            return []
+        self.parse_pass = ParsePass.VALIDATION
+        for _, v in list(self.parsed.items()):
+            if v.parse_pass == ParsePass.VALIDATION:
+                continue
+            v.run_second_parse_pass()
+        reparsed = self.transform(self.tokens[self.token_address])
+        self.environment.concepts.undefined = {}
+        passed = False
+        passes = 0
+        # output datatypes for functions may have been wrong
+        # as they were derived from not fully understood upstream types
+        # so loop through to recreate function lineage until all datatypes are known
+
+        while not passed:
+            new_passed = True
+            for x, y in self.environment.concepts.items():
+                if y.datatype == DataType.UNKNOWN and y.lineage:
+                    self.environment.concepts[x] = rehydrate_concept_lineage(
+                        y, self.environment, self.function_factory
+                    )
+                    new_passed = False
+            passes += 1
+            if passes > MAX_PARSE_DEPTH:
+                break
+            passed = new_passed
+
+        return reparsed
+
+    def start(self, args):
+        return args
+
+    def LINE_SEPARATOR(self, args):
+        return MagicConstants.LINE_SEPARATOR
+
+    def block(self, args):
+        output = args[0]
+        if isinstance(output, ConceptDeclarationStatement):
+            if len(args) > 1 and args[1] != MagicConstants.LINE_SEPARATOR:
+                comments = [x for x in args[1:] if isinstance(x, Comment)]
+                merged = "\n".join([x.text.split("#")[1].rstrip() for x in comments])
+                output.concept.metadata.description = merged
+        # this is a bad plan for now;
+        # because a comment after an import statement is very common
+        # and it's not intuitive that it modifies the import description
+        # if isinstance(output, ImportStatement):
+        #     if len(args) > 1 and isinstance(args[1], Comment):
+        #         comment = args[1].text.split("#")[1].strip()
+        #         namespace = output.alias
+        #         for _, v in self.environment.concepts.items():
+        #             if v.namespace == namespace:
+        #                 if v.metadata.description:
+        #                     v.metadata.description = (
+        #                         f"{comment}: {v.metadata.description}"
+        #                     )
+        #                 else:
+        #                     v.metadata.description = comment
+
+        return args[0]
+
+    def metadata(self, args):
+        pairs = {key: val for key, val in zip(args[::2], args[1::2])}
+        return Metadata(**pairs)
+
+    def IDENTIFIER(self, args) -> str:
+        return args.value
+
+    def ORDER_IDENTIFIER(self, args) -> ConceptRef:
+        return self.environment.concepts[args.value.strip()].reference
+
+    def WILDCARD_IDENTIFIER(self, args) -> str:
+        return args.value
+
+    def QUOTED_IDENTIFIER(self, args) -> str:
+        return args.value[1:-1]
+
+    @v_args(meta=True)
+    def concept_lit(self, meta: Meta, args) -> ConceptRef:
+        address = args[0]
+        if "." not in address and self.environment.namespace == DEFAULT_NAMESPACE:
+            address = f"{DEFAULT_NAMESPACE}.{address}"
+        mapping = self.environment.concepts[address]
+        datatype = mapping.output_datatype
+        return ConceptRef(
+            # this is load-bearing to handle pseudonyms
+            address=mapping.address,
+            metadata=Metadata(line_number=meta.line),
+            datatype=datatype,
+        )
+
+    def ADDRESS(self, args) -> Address:
+        return Address(location=args.value, quoted=False)
+
+    def QUOTED_ADDRESS(self, args) -> Address:
+        return Address(location=args.value[1:-1], quoted=True)
+
+    def STRING_CHARS(self, args) -> str:
+        return args.value
+
+    def SINGLE_STRING_CHARS(self, args) -> str:
+        return args.value
+
+    def DOUBLE_STRING_CHARS(self, args) -> str:
+        return args.value
+
+    def MINUS(self, args) -> str:
+        return "-"
+
+    @v_args(meta=True)
+    def struct_component(self, meta: Meta, args) -> StructComponent:
+        modifiers = []
+        for arg in args:
+            if isinstance(arg, Modifier):
+                modifiers.append(arg)
+        return StructComponent(name=args[0], type=args[1], modifiers=modifiers)
+
+    @v_args(meta=True)
+    def struct_type(self, meta: Meta, args) -> StructType:
+        final: list[
+            DataType
+            | MapType
+            | ArrayType
+            | NumericType
+            | StructType
+            | StructComponent
+            | Concept
+        ] = []
+        for arg in args:
+            if isinstance(arg, StructComponent):
+                final.append(arg)
+            else:
+                new = self.environment.concepts.__getitem__(  # type: ignore
+                    key=arg, line_no=meta.line
+                )
+                final.append(new)
+
+        return StructType(
+            fields=final,
+            fields_map={
+                x.name: x for x in final if isinstance(x, (Concept, StructComponent))
+            },
+        )
+
+    def list_type(self, args) -> ArrayType:
+        content = args[0]
+        if isinstance(content, str):
+            content = self.environment.concepts[content]
+        return ArrayType(type=content)
+
+    def numeric_type(self, args) -> NumericType:
+        return NumericType(precision=args[0], scale=args[1])
+
+    def map_type(self, args) -> MapType:
+        key = args[0]
+        value = args[1]
+        if isinstance(key, str):
+            key = self.environment.concepts[key]
+        elif isinstance(value, str):
+            value = self.environment.concepts[value]
+        return MapType(key_type=key, value_type=value)
+
+    @v_args(meta=True)
+    def data_type(
+        self, meta: Meta, args
+    ) -> DataType | TraitDataType | ArrayType | StructType | MapType | NumericType:
+        resolved = args[0]
+        traits = args[2:]
+        base: DataType | TraitDataType | ArrayType | StructType | MapType | NumericType
+        if isinstance(resolved, StructType):
+            base = resolved
+        elif isinstance(resolved, ArrayType):
+            base = resolved
+        elif isinstance(resolved, NumericType):
+            base = resolved
+        elif isinstance(resolved, MapType):
+            base = resolved
+        else:
+            base = DataType(args[0].lower())
+        if traits:
+            for trait in traits:
+                if trait not in self.environment.data_types:
+                    raise ParseError(
+                        f"Invalid trait (type) {trait} for {base}, line {meta.line}."
+                    )
+                matched = self.environment.data_types[trait]
+                if not is_compatible_datatype(matched.type, base):
+                    raise ParseError(
+                        f"Invalid trait (type) {trait} for {base}, line {meta.line}. Trait expects type {matched.type}, has {base}"
+                    )
+            return TraitDataType(type=base, traits=traits)
+
+        return base
+
+    def array_comparison(self, args) -> ComparisonOperator:
+        return ComparisonOperator([x.value.lower() for x in args])
+
+    def COMPARISON_OPERATOR(self, args) -> ComparisonOperator:
+        return ComparisonOperator(args.strip())
+
+    def LOGICAL_OPERATOR(self, args) -> BooleanOperator:
+        return BooleanOperator(args.lower())
+
+    def concept_assignment(self, args):
+        return args
+
+    @v_args(meta=True)
+    def column_assignment(self, meta: Meta, args):
+        modifiers = []
+        if len(args) == 2:
+            alias = args[0]
+            concept_list = args[1]
+        else:
+            alias = args[0][-1]
+            concept_list = args[0]
+        # recursively collect modifiers
+        if len(concept_list) > 1:
+            modifiers += concept_list[:-1]
+        concept = concept_list[-1]
+        resolved = self.environment.concepts.__getitem__(  # type: ignore
+            key=concept, line_no=meta.line, file=self.token_address
+        )
+        return ColumnAssignment(
+            alias=alias, modifiers=modifiers, concept=resolved.reference
+        )
+
+    def _TERMINATOR(self, args):
+        return None
+
+    def _static_functions(self, args):
+        return args[0]
+
+    def MODIFIER(self, args) -> Modifier:
+        return Modifier(args.value)
+
+    def SHORTHAND_MODIFIER(self, args) -> Modifier:
+        return Modifier(args.value)
+
+    def PURPOSE(self, args) -> Purpose:
+        return Purpose(args.value)
+
+    def AUTO(self, args) -> Purpose:
+        return Purpose.AUTO
+
+    def CONST(self, args) -> Purpose:
+        return Purpose.CONSTANT
+
+    def CONSTANT(self, args) -> Purpose:
+        return Purpose.CONSTANT
+
+    def PROPERTY(self, args):
+        return Purpose.PROPERTY
+
+    def HASH_TYPE(self, args):
+        return args.value
+
+    @v_args(meta=True)
+    def prop_ident(self, meta: Meta, args) -> Tuple[List[Concept], str]:
+        return [self.environment.concepts[grain] for grain in args[:-1]], args[-1]
+
+    @v_args(meta=True)
+    def concept_property_declaration(self, meta: Meta, args) -> Concept:
+        unique = False
+        if not args[0] == Purpose.PROPERTY:
+            unique = True
+            args = args[1:]
+        metadata = Metadata()
+        modifiers = []
+        for arg in args:
+            if isinstance(arg, Metadata):
+                metadata = arg
+            if isinstance(arg, Modifier):
+                modifiers.append(arg)
+
+        declaration = args[1]
+        if isinstance(declaration, (tuple)):
+            parents, name = declaration
+            if "." in name:
+                namespace, name = name.split(".", 1)
+            else:
+                namespace = self.environment.namespace or DEFAULT_NAMESPACE
+        else:
+            if "." not in declaration:
+                raise ParseError(
+                    f"Property declaration {args[1]} must be fully qualified with a parent key"
+                )
+            grain, name = declaration.rsplit(".", 1)
+            parent = self.environment.concepts[grain]
+            parents = [parent]
+            namespace = parent.namespace
+        concept = Concept(
+            name=name,
+            datatype=args[2],
+            purpose=Purpose.PROPERTY if not unique else Purpose.UNIQUE_PROPERTY,
+            metadata=metadata,
+            grain=Grain(components={x.address for x in parents}),
+            namespace=namespace,
+            keys=set([x.address for x in parents]),
+            modifiers=modifiers,
+        )
+
+        self.environment.add_concept(concept, meta)
+        return concept
+
+    @v_args(meta=True)
+    def concept_declaration(self, meta: Meta, args) -> ConceptDeclarationStatement:
+        metadata = Metadata()
+        modifiers = []
+        purpose = args[0]
+        datatype = args[2]
+        for arg in args:
+            if isinstance(arg, Metadata):
+                metadata = arg
+            if isinstance(arg, Modifier):
+                modifiers.append(arg)
+        name = args[1]
+        _, namespace, name, _ = parse_concept_reference(name, self.environment)
+        if purpose == Purpose.PARAMETER:
+            value = self.environment.parameters.get(name, None)
+            if not value:
+                raise MissingParameterException(
+                    f'This script requires parameter "{name}" to be set in environment.'
+                )
+            if datatype == DataType.INTEGER:
+                value = int(value)
+            elif datatype == DataType.FLOAT:
+                value = float(value)
+            elif datatype == DataType.BOOL:
+                value = bool(value)
+            elif datatype == DataType.STRING:
+                value = str(value)
+            elif datatype == DataType.DATE:
+                if isinstance(value, date):
+                    value = value
+                else:
+                    value = date.fromisoformat(value)
+            elif datatype == DataType.DATETIME:
+                if isinstance(value, datetime):
+                    value = value
+                else:
+                    value = datetime.fromisoformat(value)
+            else:
+                raise ParseError(
+                    f"Unsupported datatype {datatype} for parameter {name}."
+                )
+            rval = self.constant_derivation(
+                meta, [Purpose.CONSTANT, name, value, metadata]
+            )
+            return rval
+
+        concept = Concept(
+            name=name,
+            datatype=datatype,
+            purpose=purpose,
+            metadata=metadata,
+            namespace=namespace,
+            modifiers=modifiers,
+            derivation=Derivation.ROOT,
+            granularity=Granularity.MULTI_ROW,
+        )
+        if concept.metadata:
+            concept.metadata.line_number = meta.line
+        self.environment.add_concept(concept, meta=meta)
+        return ConceptDeclarationStatement(concept=concept)
+
+    @v_args(meta=True)
+    def concept_derivation(self, meta: Meta, args) -> ConceptDerivationStatement:
+
+        if len(args) > 3:
+            metadata = args[3]
+        else:
+            metadata = None
+        purpose = args[0]
+        raw_name = args[1]
+        # abc.def.property pattern
+        if isinstance(raw_name, str):
+            lookup, namespace, name, parent_concept = parse_concept_reference(
+                raw_name, self.environment, purpose
+            )
+        # <abc.def,zef.gf>.property pattern
+        else:
+            keys, name = raw_name
+            keys = [x.address for x in keys]
+            namespaces = set([x.rsplit(".", 1)[0] for x in keys])
+            if not len(namespaces) == 1:
+                namespace = self.environment.namespace or DEFAULT_NAMESPACE
+            else:
+                namespace = namespaces.pop()
+        source_value = args[2]
+        # we need to strip off every parenthetical to see what is being assigned.
+        while isinstance(source_value, Parenthetical):
+            source_value = source_value.content
+
+        if isinstance(
+            source_value,
+            (
+                FilterItem,
+                WindowItem,
+                AggregateWrapper,
+                Function,
+                FunctionCallWrapper,
+                Comparison,
+            ),
+        ):
+            concept = arbitrary_to_concept(
+                source_value,
+                name=name,
+                namespace=namespace,
+                environment=self.environment,
+                metadata=metadata,
+            )
+
+            # let constant purposes exist to support round-tripping
+            # as a build concept may end up with a constant based on constant inlining happening recursively
+            if purpose == Purpose.KEY and concept.purpose != Purpose.KEY:
+                concept.purpose = Purpose.KEY
+            elif (
+                purpose
+                and purpose != Purpose.AUTO
+                and concept.purpose != purpose
+                and purpose != Purpose.CONSTANT
+            ):
+                raise SyntaxError(
+                    f'Concept {name} purpose {concept.purpose} does not match declared purpose {purpose}. Suggest defaulting to "auto"'
+                )
+
+            if concept.metadata:
+                concept.metadata.line_number = meta.line
+            self.environment.add_concept(concept, meta=meta)
+            return ConceptDerivationStatement(concept=concept)
+
+        elif isinstance(source_value, CONSTANT_TYPES):
+            concept = constant_to_concept(
+                source_value,
+                name=name,
+                namespace=namespace,
+                metadata=metadata,
+            )
+            if concept.metadata:
+                concept.metadata.line_number = meta.line
+            self.environment.add_concept(concept, meta=meta)
+            return ConceptDerivationStatement(concept=concept)
+
+        raise SyntaxError(
+            f"Received invalid type {type(args[2])} {args[2]} as input to concept derivation: `{self.text_lookup[self.token_address][meta.start_pos:meta.end_pos]}`"
+        )
+
+    @v_args(meta=True)
+    def rowset_derivation_statement(
+        self, meta: Meta, args
+    ) -> RowsetDerivationStatement:
+        name = args[0]
+        select: SelectStatement | MultiSelectStatement = args[1]
+        output = RowsetDerivationStatement(
+            name=name,
+            select=select,
+            namespace=self.environment.namespace or DEFAULT_NAMESPACE,
+        )
+
+        for new_concept in rowset_to_concepts(output, self.environment):
+            if new_concept.metadata:
+                new_concept.metadata.line_number = meta.line
+            self.environment.add_concept(new_concept, force=True)
+
+        self.environment.add_rowset(
+            output.name, output.select.as_lineage(self.environment)
+        )
+        return output
+
+    @v_args(meta=True)
+    def constant_derivation(
+        self, meta: Meta, args: tuple[Purpose, str, Any, Optional[Metadata]]
+    ) -> Concept:
+
+        if len(args) > 3:
+            metadata = args[3]
+        else:
+            metadata = None
+        name = args[1]
+        constant: Union[str, float, int, bool, MapWrapper, ListWrapper] = args[2]
+        lookup, namespace, name, parent = parse_concept_reference(
+            name, self.environment
+        )
+        concept = Concept(
+            name=name,
+            datatype=arg_to_datatype(constant),
+            purpose=Purpose.CONSTANT,
+            metadata=Metadata(line_number=meta.line) if not metadata else metadata,
+            lineage=Function(
+                operator=FunctionType.CONSTANT,
+                output_datatype=arg_to_datatype(constant),
+                output_purpose=Purpose.CONSTANT,
+                arguments=[constant],
+            ),
+            grain=Grain(components=set()),
+            namespace=namespace,
+            granularity=Granularity.SINGLE_ROW,
+        )
+        if concept.metadata:
+            concept.metadata.line_number = meta.line
+        self.environment.add_concept(concept, meta)
+        return concept
+
+    @v_args(meta=True)
+    def concept(self, meta: Meta, args) -> ConceptDeclarationStatement:
+        if isinstance(args[0], Concept):
+            concept: Concept = args[0]
+        else:
+            concept = args[0].concept
+        if concept.metadata:
+            concept.metadata.line_number = meta.line
+        return ConceptDeclarationStatement(concept=concept)
+
+    def column_assignment_list(self, args):
+        return args
+
+    def column_list(self, args) -> List:
+        return args
+
+    def grain_clause(self, args) -> Grain:
+        return Grain(
+            components=set([self.environment.concepts[a].address for a in args[0]])
+        )
+
+    @v_args(meta=True)
+    def aggregate_by(self, meta: Meta, args):
+        base = args[0]
+        b_concept = base.value.split(" ")[-1]
+        args = [self.environment.concepts[a] for a in [b_concept] + args[1:]]
+        return self.function_factory.create_function(args, FunctionType.GROUP, meta)
+
+    def whole_grain_clause(self, args) -> WholeGrainWrapper:
+        return WholeGrainWrapper(where=args[0])
+
+    def MULTILINE_STRING(self, args) -> str:
+        return args[3:-3]
+
+    def raw_column_assignment(self, args):
+        return RawColumnExpr(text=args[1])
+
+    def DATASOURCE_STATUS(self, args) -> DatasourceState:
+        return DatasourceState(args.value.lower())
+
+    @v_args(meta=True)
+    def datasource_status_clause(self, meta: Meta, args):
+        return args[1]
+
+    @v_args(meta=True)
+    def datasource_partition_clause(self, meta: Meta, args):
+        return DatasourcePartitionClause([ConceptRef(address=arg) for arg in args[0]])
+
+    @v_args(meta=True)
+    def datasource_increment_clause(self, meta: Meta, args):
+        return DatasourceIncrementalClause([ConceptRef(address=arg) for arg in args[0]])
+
+    @v_args(meta=True)
+    def datasource(self, meta: Meta, args):
+        name = args[0]
+        columns: List[ColumnAssignment] = args[1]
+        grain: Optional[Grain] = None
+        address: Optional[Address] = None
+        where: Optional[WhereClause] = None
+        non_partial_for: Optional[WhereClause] = None
+        incremental_by: List[ConceptRef] = []
+        partition_by: List[ConceptRef] = []
+        datasource_status: DatasourceState = DatasourceState.PUBLISHED
+        for val in args[1:]:
+            if isinstance(val, Address):
+                address = val
+            elif isinstance(val, Grain):
+                grain = val
+            elif isinstance(val, WholeGrainWrapper):
+                non_partial_for = val.where
+            elif isinstance(val, Query):
+                address = Address(location=f"({val.text})", is_query=True)
+            elif isinstance(val, WhereClause):
+                where = val
+            elif isinstance(val, DatasourceState):
+                datasource_status = val
+            elif isinstance(val, DatasourceIncrementalClause):
+                incremental_by = val.columns
+            elif isinstance(val, DatasourcePartitionClause):
+                partition_by = val.columns
+        if not address:
+            raise ValueError(
+                "Malformed datasource, missing address or query declaration"
+            )
+
+        datasource = Datasource(
+            name=name,
+            columns=columns,
+            # grain will be set by default from args
+            # TODO: move to factory
+            grain=grain,  # type: ignore
+            address=address,
+            namespace=self.environment.namespace,
+            where=where,
+            non_partial_for=non_partial_for,
+            status=datasource_status,
+            incremental_by=incremental_by,
+            partition_by=partition_by,
+        )
+        if datasource.where:
+            for x in datasource.where.concept_arguments:
+                if x.address not in datasource.output_concepts:
+                    raise ValueError(
+                        f"Datasource {name} where condition depends on concept {x.address} that does not exist on the datasource, line {meta.line}."
+                    )
+        if self.parse_pass == ParsePass.VALIDATION:
+            self.environment.add_datasource(datasource, meta=meta)
+            # if we have any foreign keys on the datasource, we can
+            # at this point optimize them to properties if they do not have other usage.
+            for column in columns:
+                # skip partial for now
+                if not grain:
+                    continue
+                if column.concept.address in grain.components:
+                    continue
+                target_c = self.environment.concepts[column.concept.address]
+                if target_c.purpose != Purpose.KEY:
+                    continue
+
+                key_inputs = grain.components
+                eligible = True
+                for key in key_inputs:
+                    # never overwrite a key with a dependency on a property
+                    # for example - binding a datasource with a grain of <x>.fun should
+                    # never override the grain of x to <fun>
+                    if column.concept.address in (
+                        self.environment.concepts[key].keys or set()
+                    ):
+                        eligible = False
+                if not eligible:
+                    continue
+                keys = [self.environment.concepts[grain] for grain in key_inputs]
+                # target_c.purpose = Purpose.PROPERTY
+                target_c.keys = set([x.address for x in keys])
+                # target_c.grain = Grain(components={x.address for x in keys})
+
+        return datasource
+
+    @v_args(meta=True)
+    def comment(self, meta: Meta, args):
+        assert len(args) == 1
+        return Comment(text=args[0].value)
+
+    def PARSE_COMMENT(self, args):
+        return Comment(text=args.value.rstrip())
+
+    @v_args(meta=True)
+    def select_transform(self, meta: Meta, args) -> ConceptTransform:
+        output: str = args[1]
+        transformation = unwrap_transformation(args[0], self.environment)
+        lookup, namespace, output, parent = parse_concept_reference(
+            output, self.environment
+        )
+
+        metadata = Metadata(line_number=meta.line, concept_source=ConceptSource.SELECT)
+        concept = arbitrary_to_concept(
+            transformation,
+            environment=self.environment,
+            namespace=namespace,
+            name=output,
+            metadata=metadata,
+        )
+        return ConceptTransform(function=transformation, output=concept)
+
+    @v_args(meta=True)
+    def concept_nullable_modifier(self, meta: Meta, args) -> Modifier:
+        return Modifier.NULLABLE
+
+    @v_args(meta=True)
+    def select_hide_modifier(self, meta: Meta, args) -> Modifier:
+        return Modifier.HIDDEN
+
+    @v_args(meta=True)
+    def select_partial_modifier(self, meta: Meta, args) -> Modifier:
+        return Modifier.PARTIAL
+
+    @v_args(meta=True)
+    def select_item(self, meta: Meta, args) -> Optional[SelectItem]:
+        modifiers = [arg for arg in args if isinstance(arg, Modifier)]
+        args = [arg for arg in args if not isinstance(arg, (Modifier, Comment))]
+
+        if not args:
+            return None
+        if len(args) != 1:
+            raise ParseError(
+                "Malformed select statement"
+                f" {args} {self.text_lookup[self.parse_address][meta.start_pos:meta.end_pos]}"
+            )
+        content = args[0]
+        if isinstance(content, ConceptTransform):
+            return SelectItem(content=content, modifiers=modifiers)
+        return SelectItem(
+            content=content,
+            modifiers=modifiers,
+        )
+
+    def select_list(self, args):
+        return [arg for arg in args if arg]
+
+    def limit(self, args):
+        return Limit(count=int(args[0].value))
+
+    def ordering(self, args: list[str]):
+        base = args[0].lower()
+        if len(args) > 1:
+            null_sort = args[-1]
+            return Ordering(" ".join([base, "nulls", null_sort.lower()]))
+        return Ordering(base)
+
+    def order_list(self, args) -> List[OrderItem]:
+        return [
+            OrderItem(
+                expr=x,
+                order=y,
+            )
+            for x, y in zip(args[::2], args[1::2])
+        ]
+
+    def order_by(self, args):
+        return OrderBy(items=args[0])
+
+    def over_component(self, args):
+        return ConceptRef(address=args[0].value.lstrip(",").strip())
+
+    def over_list(self, args):
+        return [x for x in args]
+
+    def PUBLISH_ACTION(self, args) -> PublishAction:
+        action = args.value.lower()
+        if action == "publish":
+            return PublishAction.PUBLISH
+        elif action == "unpublish":
+            return PublishAction.UNPUBLISH
+        else:
+            raise SyntaxError(f"Unknown publish action: {action}")
+
+    @v_args(meta=True)
+    def publish_statement(self, meta: Meta, args) -> PublishStatement:
+        targets = []
+        scope = ValidationScope.DATASOURCES
+        publish_action = PublishAction.PUBLISH
+        for arg in args:
+            if isinstance(arg, str):
+                targets.append(arg)
+            elif isinstance(arg, PublishAction):
+                publish_action = arg
+            elif isinstance(arg, ValidationScope):
+                scope = arg
+                if arg != ValidationScope.DATASOURCES:
+                    raise SyntaxError(
+                        f"Publishing is only supported for Datasources, got {arg} on line {meta.line}"
+                    )
+        return PublishStatement(
+            scope=scope,
+            targets=targets,
+            action=publish_action,
+        )
+
+    def create_modifier_clause(self, args):
+        token = args[0]
+        if token.type == "CREATE_IF_NOT_EXISTS":
+            return CreateMode.CREATE_IF_NOT_EXISTS
+        elif token.type == "CREATE_OR_REPLACE":
+            return CreateMode.CREATE_OR_REPLACE
+
+    @v_args(meta=True)
+    def create_statement(self, meta: Meta, args) -> CreateStatement:
+        targets = []
+        scope = ValidationScope.DATASOURCES
+        create_mode = CreateMode.CREATE
+        for arg in args:
+            if isinstance(arg, str):
+                targets.append(arg)
+            elif isinstance(arg, ValidationScope):
+                scope = arg
+                if arg != ValidationScope.DATASOURCES:
+                    raise SyntaxError(
+                        f"Creating is only supported for Datasources, got {arg} on line {meta.line}"
+                    )
+            elif isinstance(arg, CreateMode):
+                create_mode = arg
+
+        return CreateStatement(scope=scope, targets=targets, create_mode=create_mode)
+
+    def VALIDATE_SCOPE(self, args) -> ValidationScope:
+        base: str = args.lower()
+        if not base.endswith("s"):
+            base += "s"
+        return ValidationScope(base)
+
+    @v_args(meta=True)
+    def validate_statement(self, meta: Meta, args) -> ValidateStatement:
+        if len(args) > 1:
+            scope = args[0]
+            targets = args[1:]
+        elif len(args) == 0:
+            scope = ValidationScope.ALL
+            targets = None
+        else:
+            scope = args[0]
+            targets = None
+        return ValidateStatement(
+            scope=scope,
+            targets=targets,
+        )
+
+    @v_args(meta=True)
+    def mock_statement(self, meta: Meta, args) -> MockStatement:
+        return MockStatement(scope=args[0], targets=args[1:])
+
+    @v_args(meta=True)
+    def merge_statement(self, meta: Meta, args) -> MergeStatementV2 | None:
+        modifiers = []
+        cargs: list[str] = []
+        source_wildcard = None
+        target_wildcard = None
+        for arg in args:
+            if isinstance(arg, Modifier):
+                modifiers.append(arg)
+            else:
+                cargs.append(arg)
+        source, target = cargs
+        if source.endswith(".*"):
+            if not target.endswith(".*"):
+                raise ValueError("Invalid merge, source is wildcard, target is not")
+            source_wildcard = source[:-2]
+            target_wildcard = target[:-2]
+            sources: list[Concept] = [
+                v
+                for k, v in self.environment.concepts.items()
+                if v.namespace == source_wildcard
+            ]
+            targets: dict[str, Concept] = {}
+            for x in sources:
+                target = target_wildcard + "." + x.name
+                if target in self.environment.concepts:
+                    targets[x.address] = self.environment.concepts[target]
+            sources = [x for x in sources if x.address in targets]
+        else:
+            sources = [self.environment.concepts[source]]
+            targets = {sources[0].address: self.environment.concepts[target]}
+
+        if self.parse_pass == ParsePass.VALIDATION:
+            for source_c in sources:
+                if isinstance(source_c, UndefinedConceptFull):
+                    raise SyntaxError(
+                        f"Cannot merge non-existent source concept {source_c.address} on line: {meta.line}"
+                    )
+            new = MergeStatementV2(
+                sources=sources,
+                targets=targets,
+                modifiers=modifiers,
+                source_wildcard=source_wildcard,
+                target_wildcard=target_wildcard,
+            )
+            for source_c in new.sources:
+                self.environment.merge_concept(
+                    source_c, targets[source_c.address], modifiers
+                )
+
+            return new
+        return None
+
+    @v_args(meta=True)
+    def rawsql_statement(self, meta: Meta, args) -> RawSQLStatement:
+        statement = RawSQLStatement(meta=Metadata(line_number=meta.line), text=args[0])
+        return statement
+
+    def COPY_TYPE(self, args) -> IOType:
+        return IOType(args.value)
+
+    @v_args(meta=True)
+    def copy_statement(self, meta: Meta, args) -> CopyStatement:
+        return CopyStatement(
+            target=args[1],
+            target_type=args[0],
+            meta=Metadata(line_number=meta.line),
+            select=args[-1],
+        )
+
+    def resolve_import_address(self, address: str, is_stdlib: bool = False) -> str:
+        if (
+            isinstance(
+                self.environment.config.import_resolver, FileSystemImportResolver
+            )
+            or is_stdlib
+        ):
+            with open(address, "r", encoding="utf-8") as f:
+                text = f.read()
+        elif isinstance(self.environment.config.import_resolver, DictImportResolver):
+            lookup = address
+            if lookup not in self.environment.config.import_resolver.content:
+                raise ImportError(
+                    f"Unable to import file {lookup}, not resolvable from provided source files."
+                )
+            text = self.environment.config.import_resolver.content[lookup]
+        else:
+            raise ImportError(
+                f"Unable to import file {address}, resolver type {type(self.environment.config.import_resolver)} not supported"
+            )
+        return text
+
+    def IMPORT_DOT(self, args) -> str:
+        return "."
+
+    def import_statement(self, args: list[str]) -> ImportStatement:
+        start = datetime.now()
+        is_file_resolver = isinstance(
+            self.environment.config.import_resolver, FileSystemImportResolver
+        )
+        parent_dirs = -1
+        parsed_args = []
+        for x in args:
+            if x == ".":
+                parent_dirs += 1
+            else:
+                parsed_args.append(x)
+        parent_dirs = max(parent_dirs, 0)
+        args = parsed_args
+        if len(args) == 2:
+            alias = args[-1]
+            cache_key = args[-1]
+        else:
+            alias = self.environment.namespace
+            cache_key = args[0]
+        input_path = args[0]
+        # lstrip off '.' from parent if they exist;
+        # each one is an extra directory up after the first
+
+        path = input_path.split(".")
+        is_stdlib = False
+        if path[0] == "std":
+            is_stdlib = True
+            target = join(STDLIB_ROOT, *path) + ".preql"
+            token_lookup: Path | str = Path(target)
+        elif is_file_resolver:
+            troot = Path(self.environment.working_path)
+            if parent_dirs > 0:
+                for _ in range(parent_dirs):
+                    troot = troot.parent
+            target = join(troot, *path) + ".preql"
+            # tokens + text are cached by path
+            token_lookup = Path(target)
+        elif isinstance(self.environment.config.import_resolver, DictImportResolver):
+            target = ".".join(path)
+            token_lookup = target
+        else:
+            raise NotImplementedError
+
+        # parser + env has to be cached by prior import path + current key
+        key_path = self.import_keys + [cache_key]
+        cache_lookup = "-".join(key_path)
+
+        # we don't iterate past the max parse depth
+        if len(key_path) > MAX_PARSE_DEPTH:
+            return ImportStatement(
+                alias=alias, input_path=input_path, path=Path(target)
+            )
+
+        if token_lookup in self.tokens:
+            perf_logger.debug(f"\tTokens cached for {token_lookup}")
+            raw_tokens = self.tokens[token_lookup]
+            text = self.text_lookup[token_lookup]
+        else:
+            perf_logger.debug(f"\tTokens not cached for {token_lookup}, resolving")
+            text = self.resolve_import_address(target, is_stdlib)
+            self.text_lookup[token_lookup] = text
+
+            try:
+                raw_tokens = PARSER.parse(text)
+            except Exception as e:
+                raise ImportError(
+                    f"Unable to import '{target}', parsing error: {e}"
+                ) from e
+            self.tokens[token_lookup] = raw_tokens
+
+        if cache_lookup in self.parsed:
+            perf_logger.debug(f"\tEnvironment cached for {token_lookup}")
+            nparser = self.parsed[cache_lookup]
+            new_env = nparser.environment
+            if nparser.parse_pass != ParsePass.VALIDATION:
+                # nparser.transform(raw_tokens)
+                second_pass_start = datetime.now()
+                nparser.run_second_parse_pass()
+                second_pass_end = datetime.now()
+                perf_logger.debug(
+                    f"{second_pass_end - second_pass_start} seconds | Import {alias} key ({cache_key}) second pass took {second_pass_end - second_pass_start} to parse, {len(new_env.concepts)} concepts"
+                )
+        else:
+            perf_logger.debug(f"\tParsing new for {token_lookup}")
+            root = None
+            if "." in str(token_lookup):
+                root = str(token_lookup).rsplit(".", 1)[0]
+            try:
+                new_env = Environment(
+                    working_path=dirname(target),
+                    env_file_path=token_lookup,
+                    config=self.environment.config.copy_for_root(root=root),
+                    parameters=self.environment.parameters,
+                )
+                new_env.concepts.fail_on_missing = False
+                self.parsed[self.parse_address] = self
+                nparser = ParseToObjects(
+                    environment=new_env,
+                    parse_address=cache_lookup,
+                    token_address=token_lookup,
+                    parsed=self.parsed,
+                    tokens=self.tokens,
+                    text_lookup=self.text_lookup,
+                    import_keys=self.import_keys + [cache_key],
+                    parse_config=self.parse_config,
+                )
+                nparser.transform(raw_tokens)
+                self.parsed[cache_lookup] = nparser
+            except Exception as e:
+                raise ImportError(
+                    f"Unable to import file {target}, parsing error: {e}"
+                ) from e
+
+        parsed_path = Path(args[0])
+        imps = ImportStatement(alias=alias, input_path=input_path, path=parsed_path)
+
+        self.environment.add_import(
+            alias,
+            new_env,
+            Import(
+                alias=alias,
+                path=parsed_path,
+                input_path=Path(target) if is_file_resolver else None,
+            ),
+        )
+        end = datetime.now()
+        perf_logger.debug(
+            f"{end - start} seconds | Import {alias} key ({cache_key}) took  to parse, {len(new_env.concepts)} concepts"
+        )
+        return imps
+
+    @v_args(meta=True)
+    def show_category(self, meta: Meta, args) -> ShowCategory:
+        return ShowCategory(args[0])
+
+    @v_args(meta=True)
+    def show_statement(self, meta: Meta, args) -> ShowStatement:
+        return ShowStatement(content=args[0])
+
+    @v_args(meta=True)
+    def persist_partition_clause(self, meta: Meta, args) -> DatasourcePartitionClause:
+        return DatasourcePartitionClause([ConceptRef(address=a) for a in args[0]])
+
+    @v_args(meta=True)
+    def PERSIST_MODE(self, args) -> PersistMode:
+        base = args.value.lower()
+        if base == "persist":
+            return PersistMode.OVERWRITE
+        return PersistMode(base)
+
+    @v_args(meta=True)
+    def auto_persist(self, meta: Meta, args) -> PersistStatement | None:
+        if self.parse_pass != ParsePass.VALIDATION:
+            return None
+        persist_mode = args[0]
+        target_name = args[1]
+        where = args[2] if len(args) > 2 else None
+
+        if target_name not in self.environment.datasources:
+            raise SyntaxError(
+                f"Auto persist target datasource {target_name} does not exist in environment on line {meta.line}. Have {list(self.environment.datasources.keys())}"
+            )
+        target = self.environment.datasources[target_name]
+        select: SelectStatement = SelectStatement.from_inputs(
+            environment=self.environment,
+            selection=[
+                SelectItem(
+                    content=ConceptRef(address=col.concept.address),
+                    modifiers=[],
+                )
+                for col in target.columns
+            ],
+            where_clause=where,
+            meta=Metadata(line_number=meta.line),
+        )
+        return PersistStatement(
+            select=select,
+            datasource=target,
+            persist_mode=persist_mode,
+            partition_by=target.incremental_by,
+            meta=Metadata(line_number=meta.line),
+        )
+
+    @v_args(meta=True)
+    def full_persist(self, meta: Meta, args) -> PersistStatement | None:
+        if self.parse_pass != ParsePass.VALIDATION:
+            return None
+        partition_clause = DatasourcePartitionClause([])
+        labels = [x for x in args if isinstance(x, str)]
+        for x in args:
+            if isinstance(x, DatasourcePartitionClause):
+                partition_clause = x
+        if len(labels) == 2:
+            identifier = labels[0]
+            address = labels[1]
+        else:
+            identifier = labels[0]
+            address = None
+        target: Datasource | None = self.environment.datasources.get(identifier)
+
+        if not address and not target:
+            raise SyntaxError(
+                f'Append statement without concrete table address on line {meta.line} attempts to insert into datasource "{identifier}" that cannot be found in the environment. Add a physical address to create a new datasource, or check the name.'
+            )
+        elif target:
+            address = target.safe_address
+
+        assert address is not None
+
+        modes = [x for x in args if isinstance(x, PersistMode)]
+        mode = modes[0] if modes else PersistMode.OVERWRITE
+        select: SelectStatement = [x for x in args if isinstance(x, SelectStatement)][0]
+
+        if mode == PersistMode.APPEND:
+            if target is None:
+                raise SyntaxError(
+                    f"Cannot append to non-existent datasource {identifier} on line {meta.line}."
+                )
+            new_datasource: Datasource = target
+            if not new_datasource.partition_by == partition_clause.columns:
+                raise SyntaxError(
+                    f"Cannot append to datasource {identifier} with different partitioning scheme then insert on line {meta.line}. Datasource partitioning: {new_datasource.partition_by}, insert partitioning: {partition_clause.columns if partition_clause else '[]'}"
+                )
+            if len(partition_clause.columns) > 1:
+                raise NotImplementedError(
+                    "Incremental partition overwrites by more than 1 column are not yet supported."
+                )
+            for x in partition_clause.columns:
+                concept = self.environment.concepts[x.address]
+                if concept.output_datatype not in SUPPORTED_INCREMENTAL_TYPES:
+                    raise SyntaxError(
+                        f"Cannot incremental persist on concept {concept.address} of type {concept.output_datatype} on line {meta.line}."
+                    )
+        elif target:
+            new_datasource = target
+        else:
+            new_datasource = select.to_datasource(
+                namespace=(
+                    self.environment.namespace
+                    if self.environment.namespace
+                    else DEFAULT_NAMESPACE
+                ),
+                name=identifier,
+                address=Address(location=address),
+                grain=select.grain,
+                environment=self.environment,
+            )
+        return PersistStatement(
+            select=select,
+            datasource=new_datasource,
+            persist_mode=mode,
+            partition_by=partition_clause.columns if partition_clause else [],
+            meta=Metadata(line_number=meta.line),
+        )
+
+    @v_args(meta=True)
+    def persist_statement(self, meta: Meta, args) -> PersistStatement:
+        return args[0]
+
+    @v_args(meta=True)
+    def align_item(self, meta: Meta, args) -> AlignItem:
+        return AlignItem(
+            alias=args[0],
+            namespace=self.environment.namespace,
+            concepts=[self.environment.concepts[arg].reference for arg in args[1:]],
+        )
+
+    @v_args(meta=True)
+    def align_clause(self, meta: Meta, args) -> AlignClause:
+        return AlignClause(items=args)
+
+    @v_args(meta=True)
+    def derive_item(self, meta: Meta, args) -> DeriveItem:
+        return DeriveItem(
+            expr=args[0], name=args[1], namespace=self.environment.namespace
+        )
+
+    @v_args(meta=True)
+    def derive_clause(self, meta: Meta, args) -> DeriveClause:
+
+        return DeriveClause(items=args)
+
+    @v_args(meta=True)
+    def multi_select_statement(self, meta: Meta, args) -> MultiSelectStatement:
+
+        selects: list[SelectStatement] = []
+        align: AlignClause | None = None
+        limit: int | None = None
+        order_by: OrderBy | None = None
+        where: WhereClause | None = None
+        having: HavingClause | None = None
+        derive: DeriveClause | None = None
+        for arg in args:
+            if isinstance(arg, SelectStatement):
+                selects.append(arg)
+            elif isinstance(arg, Limit):
+                limit = arg.count
+            elif isinstance(arg, OrderBy):
+                order_by = arg
+            elif isinstance(arg, WhereClause):
+                where = arg
+            elif isinstance(arg, HavingClause):
+                having = arg
+            elif isinstance(arg, AlignClause):
+                align = arg
+            elif isinstance(arg, DeriveClause):
+                derive = arg
+
+        assert align
+        assert align is not None
+
+        derived_concepts = []
+        new_selects = [x.as_lineage(self.environment) for x in selects]
+        lineage = MultiSelectLineage(
+            selects=new_selects,
+            align=align,
+            derive=derive,
+            namespace=self.environment.namespace,
+            where_clause=where,
+            having_clause=having,
+            limit=limit,
+            hidden_components=set(y for x in new_selects for y in x.hidden_components),
+        )
+        for x in align.items:
+            concept = align_item_to_concept(
+                x,
+                align,
+                selects,
+                where=where,
+                having=having,
+                limit=limit,
+                environment=self.environment,
+            )
+            derived_concepts.append(concept)
+            self.environment.add_concept(concept, meta=meta)
+        if derive:
+            for derived in derive.items:
+                derivation = derived.expr
+                name = derived.name
+                if not isinstance(derivation, (Function, Comparison, WindowItem)):
+                    raise SyntaxError(
+                        f"Invalid derive expression {derivation} in {meta.line}, must be a function or conditional"
+                    )
+                concept = derive_item_to_concept(
+                    derivation, name, lineage, self.environment.namespace
+                )
+                derived_concepts.append(concept)
+                self.environment.add_concept(concept, meta=meta)
+        multi = MultiSelectStatement(
+            selects=selects,
+            align=align,
+            namespace=self.environment.namespace,
+            where_clause=where,
+            order_by=order_by,
+            limit=limit,
+            meta=Metadata(line_number=meta.line),
+            derived_concepts=derived_concepts,
+            derive=derive,
+        )
+        return multi
+
+    @v_args(meta=True)
+    def select_statement(self, meta: Meta, args) -> SelectStatement:
+        select_items: List[SelectItem] | None = None
+        limit = None
+        order_by = None
+        where = None
+        having = None
+        for arg in args:
+            if isinstance(arg, List):
+                select_items = arg
+            elif isinstance(arg, Limit):
+                limit = arg.count
+            elif isinstance(arg, OrderBy):
+                order_by = arg
+            elif isinstance(arg, WhereClause) and not isinstance(arg, HavingClause):
+                if where is not None:
+                    raise ParseError(
+                        "Multiple where clauses defined are not supported!"
+                    )
+                where = arg
+            elif isinstance(arg, HavingClause):
+                having = arg
+        if not select_items:
+            raise ParseError("Malformed select, missing select items")
+        pre_keys = set(self.environment.concepts.keys())
+        base = SelectStatement.from_inputs(
+            environment=self.environment,
+            selection=select_items,
+            order_by=order_by,
+            where_clause=where,
+            having_clause=having,
+            limit=limit,
+            meta=Metadata(line_number=meta.line),
+        )
+        if (
+            self.parse_pass == ParsePass.INITIAL
+            and self.parse_config.strict_name_shadow_enforcement
+        ):
+            intersection = base.locally_derived.intersection(pre_keys)
+            if intersection:
+                for x in intersection:
+                    if str(base.local_concepts[x].lineage) == str(
+                        self.environment.concepts[x].lineage
+                    ):
+                        local = base.local_concepts[x]
+                        friendly_name = (
+                            local.name
+                            if local.namespace == DEFAULT_NAMESPACE
+                            else local.namespace
+                        )
+                        raise NameShadowError(
+                            f"Select statement {base} creates a new concept '{friendly_name}' with identical definition as the existing concept '{friendly_name}'. Replace {base.local_concepts[x].lineage} with a direct reference to {friendly_name}."
+                        )
+                else:
+                    raise NameShadowError(
+                        f"Select statement {base} creates new named concepts from calculations {list(intersection)} with identical name(s) to existing concept(s). Use new unique names for these."
+                    )
+        return base
+
+    @v_args(meta=True)
+    def address(self, meta: Meta, args):
+        return args[0]
+
+    @v_args(meta=True)
+    def query(self, meta: Meta, args):
+        return Query(text=args[0])
+
+    def where(self, args):
+        root = args[0]
+        root = expr_to_boolean(root, self.function_factory)
+        return WhereClause(conditional=root)
+
+    def having(self, args):
+        root = args[0]
+        if not isinstance(root, (Comparison, Conditional, Parenthetical)):
+            if arg_to_datatype(root) == DataType.BOOL:
+                root = Comparison(left=root, right=True, operator=ComparisonOperator.EQ)
+            else:
+                root = Comparison(
+                    left=root,
+                    right=MagicConstants.NULL,
+                    operator=ComparisonOperator.IS_NOT,
+                )
+        return HavingClause(conditional=root)
+
+    @v_args(meta=True)
+    def function_binding_list(self, meta: Meta, args) -> list[ArgBinding]:
+        return args
+
+    @v_args(meta=True)
+    def function_binding_type(self, meta: Meta, args) -> FunctionBindingType:
+        return FunctionBindingType(type=args[0])
+
+    @v_args(meta=True)
+    def function_binding_default(self, meta: Meta, args):
+        return args[1]
+
+    @v_args(meta=True)
+    def function_binding_item(self, meta: Meta, args) -> ArgBinding:
+        default = None
+        type = None
+        for arg in args[1:]:
+            if isinstance(arg, FunctionBindingType):
+                type = arg.type
+            else:
+                default = arg
+        return ArgBinding.model_construct(name=args[0], datatype=type, default=default)
+
+    @v_args(meta=True)
+    def raw_function(self, meta: Meta, args) -> FunctionDeclaration:
+        identity = args[0]
+        function_arguments: list[ArgBinding] = args[1]
+        output = args[2]
+
+        self.environment.functions[identity] = CustomFunctionFactory(
+            function=output,
+            namespace=self.environment.namespace,
+            function_arguments=function_arguments,
+            name=identity,
+        )
+        return FunctionDeclaration(name=identity, args=function_arguments, expr=output)
+
+    def custom_function(self, args) -> FunctionCallWrapper:
+        name = args[0]
+        args = args[1:]
+        remapped = FunctionCallWrapper(
+            content=self.environment.functions[name](*args), name=name, args=args
+        )
+
+        return remapped
+
+    @v_args(meta=True)
+    def function(self, meta: Meta, args) -> Function:
+        return args[0]
+
+    @v_args(meta=True)
+    def type_drop_clause(self, meta: Meta, args) -> DropOn:
+        return DropOn([FunctionType(x) for x in args])
+
+    @v_args(meta=True)
+    def type_add_clause(self, meta: Meta, args) -> AddOn:
+        return AddOn([FunctionType(x) for x in args])
+
+    @v_args(meta=True)
+    def type_declaration(self, meta: Meta, args) -> TypeDeclaration:
+        key = args[0]
+        datatype: list[DataType] = [x for x in args[1:] if isinstance(x, DataType)]
+        if len(datatype) == 1:
+            final_datatype: list[DataType] | DataType = datatype[0]
+        else:
+            final_datatype = datatype
+        add_on = None
+        drop_on = None
+        for x in args[1:]:
+            if isinstance(x, AddOn):
+                add_on = x
+            elif isinstance(x, DropOn):
+                drop_on = x
+        new = CustomType(
+            name=key,
+            type=final_datatype,
+            drop_on=drop_on.functions if drop_on else [],
+            add_on=add_on.functions if add_on else [],
+        )
+        self.environment.data_types[key] = new
+        return TypeDeclaration(type=new)
+
+    def int_lit(self, args):
+        return int("".join(args))
+
+    def bool_lit(self, args):
+        return args[0].capitalize() == "True"
+
+    def null_lit(self, args):
+        return NULL_VALUE
+
+    def float_lit(self, args):
+        return float(args[0])
+
+    def array_lit(self, args):
+        return list_to_wrapper(args)
+
+    def tuple_lit(self, args):
+        return tuple_to_wrapper(args)
+
+    def string_lit(self, args) -> str:
+        if not args:
+            return ""
+
+        return args[0]
+
+    @v_args(meta=True)
+    def struct_lit(self, meta, args):
+        return self.function_factory.create_function(
+            args, operator=FunctionType.STRUCT, meta=meta
+        )
+
+    def map_lit(self, args):
+        parsed = dict(zip(args[::2], args[1::2]))
+        wrapped = dict_to_map_wrapper(parsed)
+        return wrapped
+
+    def literal(self, args):
+        return args[0]
+
+    def product_operator(self, args) -> Function | Any:
+        if len(args) == 1:
+            return args[0]
+        result = args[0]
+        for i in range(1, len(args), 2):
+            new_result = None
+            op = args[i]
+            right = args[i + 1]
+            if op == "*":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.MULTIPLY
+                )
+            elif op == "**":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.POWER
+                )
+            elif op == "/":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.DIVIDE
+                )
+            elif op == "%":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.MOD
+                )
+            else:
+                raise ValueError(f"Unknown operator: {op}")
+            result = new_result
+        return new_result
+
+    def PLUS_OR_MINUS(self, args) -> str:
+        return args.value
+
+    def MULTIPLY_DIVIDE_PERCENT(self, args) -> str:
+        return args.value
+
+    @v_args(meta=True)
+    def sum_operator(self, meta: Meta, args) -> Function | Any:
+        if len(args) == 1:
+            return args[0]
+        result = args[0]
+        for i in range(1, len(args), 2):
+            new_result = None
+            op = args[i].lower()
+            right = args[i + 1]
+            if op == "+":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.ADD, meta=meta
+                )
+            elif op == "-":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.SUBTRACT, meta=meta
+                )
+            elif op == "||":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.CONCAT, meta=meta
+                )
+            elif op == "like":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.LIKE, meta=meta
+                )
+            else:
+                raise ValueError(f"Unknown operator: {op}")
+            result = new_result
+        return result
+
+    def comparison(self, args) -> Comparison:
+        if len(args) == 1:
+            return args[0]
+        left = args[0]
+        right = args[2]
+        if args[1] in (ComparisonOperator.IN, ComparisonOperator.NOT_IN):
+            return SubselectComparison(
+                left=left,
+                right=right,
+                operator=args[1],
+            )
+        return Comparison(left=left, right=right, operator=args[1])
+
+    def between_comparison(self, args) -> Conditional:
+        left_bound = args[1]
+        right_bound = args[2]
+        return Conditional(
+            left=Comparison(
+                left=args[0], right=left_bound, operator=ComparisonOperator.GTE
+            ),
+            right=Comparison(
+                left=args[0], right=right_bound, operator=ComparisonOperator.LTE
+            ),
+            operator=BooleanOperator.AND,
+        )
+
+    @v_args(meta=True)
+    def subselect_comparison(self, meta: Meta, args) -> SubselectComparison:
+        right = args[2]
+
+        while isinstance(right, Parenthetical) and isinstance(
+            right.content,
+            (
+                Concept,
+                Function,
+                FilterItem,
+                WindowItem,
+                AggregateWrapper,
+                ListWrapper,
+                TupleWrapper,
+            ),
+        ):
+            right = right.content
+        if isinstance(right, (Function, FilterItem, WindowItem, AggregateWrapper)):
+            right_concept = arbitrary_to_concept(right, environment=self.environment)
+            self.environment.add_concept(right_concept, meta=meta)
+            right = right_concept.reference
+        return SubselectComparison(
+            left=args[0],
+            right=right,
+            operator=args[1],
+        )
+
+    def expr_tuple(self, args):
+        datatypes = set([arg_to_datatype(x) for x in args])
+        if len(datatypes) != 1:
+            raise ParseError("Tuple must have same type for all elements")
+        return TupleWrapper(val=tuple(args), type=datatypes.pop())
+
+    def parenthetical(self, args):
+        return Parenthetical(content=args[0])
+
+    @v_args(meta=True)
+    def condition_parenthetical(self, meta, args):
+        if len(args) == 2:
+            return Comparison(
+                left=Parenthetical(content=args[1]),
+                right=False,
+                operator=ComparisonOperator.EQ,
+            )
+        return Parenthetical(content=args[0])
+
+    def conditional(self, args):
+        def munch_args(args):
+            while args:
+                if len(args) == 1:
+                    return args[0]
+                else:
+                    return Conditional(
+                        left=args[0], operator=args[1], right=munch_args(args[2:])
+                    )
+
+        return munch_args(args)
+
+    def window_order(self, args):
+        return WindowOrder(args[0])
+
+    def window_order_by(self, args):
+        # flatten tree
+        return args[0]
+
+    def window(self, args):
+
+        return Window(count=args[1].value, window_order=args[0])
+
+    def WINDOW_TYPE(self, args):
+        return WindowType(args.strip())
+
+    def window_item_over(self, args):
+
+        return WindowItemOver(contents=args[0])
+
+    def window_item_order(self, args):
+        return WindowItemOrder(contents=args[0])
+
+    def logical_operator(self, args):
+        return BooleanOperator(args[0].value.lower())
+
+    def DATE_PART(self, args):
+        return DatePart(args.value)
+
+    @v_args(meta=True)
+    def window_item(self, meta: Meta, args) -> WindowItem:
+        type: WindowType = args[0]
+        order_by = []
+        over = []
+        index = None
+        concept: Concept | None = None
+        for item in args:
+            if isinstance(item, int):
+                index = item
+            elif isinstance(item, WindowItemOrder):
+                order_by = item.contents
+            elif isinstance(item, WindowItemOver):
+                over = item.contents
+            elif isinstance(item, str):
+                concept = self.environment.concepts[item]
+            elif isinstance(item, ConceptRef):
+                concept = self.environment.concepts[item.address]
+            elif isinstance(item, WindowType):
+                type = item
+            else:
+                concept = arbitrary_to_concept(item, environment=self.environment)
+                self.environment.add_concept(concept, meta=meta)
+        if not concept:
+            raise ParseError(
+                f"Window statements must be on fields, not constants - error in: `{self.text_lookup[self.parse_address][meta.start_pos:meta.end_pos]}`"
+            )
+        return WindowItem(
+            type=type,
+            content=concept.reference,
+            over=over,
+            order_by=order_by,
+            index=index,
+        )
+
+    def filter_item(self, args) -> FilterItem:
+        where: WhereClause
+        expr, raw = args
+        if isinstance(raw, WhereClause):
+            where = raw
+        else:
+            where = WhereClause(conditional=expr_to_boolean(raw, self.function_factory))
+        if isinstance(expr, str):
+            expr = self.environment.concepts[expr].reference
+        return FilterItem(content=expr, where=where)
+
+    # BEGIN FUNCTIONS
+    @v_args(meta=True)
+    def expr_reference(self, meta, args) -> Concept:
+        return self.environment.concepts.__getitem__(args[0], meta.line)
+
+    def expr(self, args):
+        if len(args) > 1:
+            raise ParseError("Expression should have one child only.")
+        return args[0]
+
+    def aggregate_over(self, args):
+        return args[0]
+
+    def aggregate_all(self, args):
+        return [
+            ConceptRef(
+                address=f"{INTERNAL_NAMESPACE}.{ALL_ROWS_CONCEPT}",
+                datatype=DataType.INTEGER,
+            )
+        ]
+
+    def aggregate_functions(self, args):
+        if len(args) == 2:
+            return AggregateWrapper(function=args[0], by=args[1])
+        return AggregateWrapper(function=args[0])
+
+    @v_args(meta=True)
+    def index_access(self, meta, args):
+        args = process_function_args(args, meta=meta, environment=self.environment)
+        base = args[0]
+        if base.datatype == DataType.MAP or isinstance(base.datatype, MapType):
+            return self.function_factory.create_function(
+                args, FunctionType.MAP_ACCESS, meta
+            )
+        return self.function_factory.create_function(
+            args, FunctionType.INDEX_ACCESS, meta
+        )
+
+    @v_args(meta=True)
+    def map_key_access(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.MAP_ACCESS, meta
+        )
+
+    @v_args(meta=True)
+    def attr_access(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.ATTR_ACCESS, meta
+        )
+
+    @v_args(meta=True)
+    def fcoalesce(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.COALESCE, meta)
+
+    @v_args(meta=True)
+    def fnullif(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.NULLIF, meta)
+
+    @v_args(meta=True)
+    def frecurse_edge(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.RECURSE_EDGE, meta
+        )
+
+    @v_args(meta=True)
+    def unnest(self, meta, args):
+
+        return self.function_factory.create_function(args, FunctionType.UNNEST, meta)
+
+    @v_args(meta=True)
+    def count(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.COUNT, meta)
+
+    @v_args(meta=True)
+    def fgroup(self, meta, args):
+        if len(args) == 2:
+            fargs = [args[0]] + list(args[1])
+        else:
+            fargs = [args[0]]
+        return self.function_factory.create_function(fargs, FunctionType.GROUP, meta)
+
+    @v_args(meta=True)
+    def fabs(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.ABS, meta)
+
+    @v_args(meta=True)
+    def count_distinct(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.COUNT_DISTINCT, meta
+        )
+
+    @v_args(meta=True)
+    def sum(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.SUM, meta)
+
+    @v_args(meta=True)
+    def array_agg(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.ARRAY_AGG, meta)
+
+    @v_args(meta=True)
+    def any(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.ANY, meta)
+
+    @v_args(meta=True)
+    def bool_and(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.BOOL_AND, meta)
+
+    @v_args(meta=True)
+    def bool_or(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.BOOL_OR, meta)
+
+    @v_args(meta=True)
+    def avg(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.AVG, meta)
+
+    @v_args(meta=True)
+    def max(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.MAX, meta)
+
+    @v_args(meta=True)
+    def min(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.MIN, meta)
+
+    @v_args(meta=True)
+    def len(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.LENGTH, meta)
+
+    @v_args(meta=True)
+    def fsplit(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.SPLIT, meta)
+
+    @v_args(meta=True)
+    def concat(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.CONCAT, meta)
+
+    @v_args(meta=True)
+    def union(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.UNION, meta)
+
+    @v_args(meta=True)
+    def like(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.LIKE, meta)
+
+    @v_args(meta=True)
+    def alt_like(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.LIKE, meta)
+
+    @v_args(meta=True)
+    def ilike(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.LIKE, meta)
+
+    @v_args(meta=True)
+    def upper(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.UPPER, meta)
+
+    @v_args(meta=True)
+    def fstrpos(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.STRPOS, meta)
+
+    @v_args(meta=True)
+    def freplace(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.REPLACE, meta)
+
+    @v_args(meta=True)
+    def fcontains(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.CONTAINS, meta)
+
+    @v_args(meta=True)
+    def ftrim(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.TRIM, meta)
+
+    @v_args(meta=True)
+    def fhash(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.HASH, meta)
+
+    @v_args(meta=True)
+    def fsubstring(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.SUBSTRING, meta)
+
+    @v_args(meta=True)
+    def flower(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.LOWER, meta)
+
+    @v_args(meta=True)
+    def fregexp_contains(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.REGEXP_CONTAINS, meta
+        )
+
+    @v_args(meta=True)
+    def fregexp_extract(self, meta, args):
+        if len(args) == 2:
+            # this is a magic value to represent the default behavior
+            args.append(-1)
+        return self.function_factory.create_function(
+            args, FunctionType.REGEXP_EXTRACT, meta
+        )
+
+    @v_args(meta=True)
+    def fregexp_replace(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.REGEXP_REPLACE, meta
+        )
+
+    # date functions
+    @v_args(meta=True)
+    def fdate(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.DATE, meta)
+
+    @v_args(meta=True)
+    def fdate_trunc(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.DATE_TRUNCATE, meta
+        )
+
+    @v_args(meta=True)
+    def fdate_part(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.DATE_PART, meta)
+
+    @v_args(meta=True)
+    def fdate_add(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.DATE_ADD, meta)
+
+    @v_args(meta=True)
+    def fdate_sub(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.DATE_SUB, meta)
+
+    @v_args(meta=True)
+    def fdate_diff(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.DATE_DIFF, meta)
+
+    @v_args(meta=True)
+    def fdatetime(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.DATETIME, meta)
+
+    @v_args(meta=True)
+    def ftimestamp(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.TIMESTAMP, meta)
+
+    @v_args(meta=True)
+    def fsecond(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.SECOND, meta)
+
+    @v_args(meta=True)
+    def fminute(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.MINUTE, meta)
+
+    @v_args(meta=True)
+    def fhour(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.HOUR, meta)
+
+    @v_args(meta=True)
+    def fday(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.DAY, meta)
+
+    @v_args(meta=True)
+    def fday_name(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.DAY_NAME, meta)
+
+    @v_args(meta=True)
+    def fday_of_week(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.DAY_OF_WEEK, meta
+        )
+
+    @v_args(meta=True)
+    def fweek(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.WEEK, meta)
+
+    @v_args(meta=True)
+    def fmonth(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.MONTH, meta)
+
+    @v_args(meta=True)
+    def fmonth_name(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.MONTH_NAME, meta
+        )
+
+    @v_args(meta=True)
+    def fquarter(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.QUARTER, meta)
+
+    @v_args(meta=True)
+    def fyear(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.YEAR, meta)
+
+    def internal_fcast(self, meta, args) -> Function:
+        args = process_function_args(args, meta=meta, environment=self.environment)
+
+        if isinstance(args[0], str):
+
+            processed: date | datetime | int | float | bool | str
+            if args[1] == DataType.DATE:
+                processed = date.fromisoformat(args[0])
+            elif args[1] == DataType.DATETIME:
+                processed = datetime.fromisoformat(args[0])
+            elif args[1] == DataType.TIMESTAMP:
+                processed = datetime.fromisoformat(args[0])
+            elif args[1] == DataType.INTEGER:
+                processed = int(args[0])
+            elif args[1] == DataType.FLOAT:
+                processed = float(args[0])
+            elif args[1] == DataType.BOOL:
+                processed = args[0].capitalize() == "True"
+            elif args[1] == DataType.STRING:
+                processed = args[0]
+            else:
+                raise SyntaxError(f"Invalid cast type {args[1]}")
+            if isinstance(args[1], TraitDataType):
+                return self.function_factory.create_function(
+                    [processed, args[1]], FunctionType.TYPED_CONSTANT, meta
+                )
+            return self.function_factory.create_function(
+                [processed], FunctionType.CONSTANT, meta
+            )
+        return self.function_factory.create_function(args, FunctionType.CAST, meta)
+
+    @v_args(meta=True)
+    def fdate_spine(self, meta, args) -> Function:
+        return self.function_factory.create_function(
+            args, FunctionType.DATE_SPINE, meta
+        )
+
+    # utility functions
+    @v_args(meta=True)
+    def fcast(self, meta, args) -> Function:
+        return self.internal_fcast(meta, args)
+
+    # math functions
+    @v_args(meta=True)
+    def fadd(self, meta, args) -> Function:
+
+        return self.function_factory.create_function(args, FunctionType.ADD, meta)
+
+    @v_args(meta=True)
+    def fsub(self, meta, args) -> Function:
+        return self.function_factory.create_function(args, FunctionType.SUBTRACT, meta)
+
+    @v_args(meta=True)
+    def fmul(self, meta, args) -> Function:
+        return self.function_factory.create_function(args, FunctionType.MULTIPLY, meta)
+
+    @v_args(meta=True)
+    def fdiv(self, meta: Meta, args) -> Function:
+        return self.function_factory.create_function(args, FunctionType.DIVIDE, meta)
+
+    @v_args(meta=True)
+    def fmod(self, meta: Meta, args) -> Function:
+        return self.function_factory.create_function(args, FunctionType.MOD, meta)
+
+    @v_args(meta=True)
+    def fsqrt(self, meta: Meta, args) -> Function:
+        return self.function_factory.create_function(args, FunctionType.SQRT, meta)
+
+    @v_args(meta=True)
+    def frandom(self, meta: Meta, args) -> Function:
+        return self.function_factory.create_function(args, FunctionType.RANDOM, meta)
+
+    @v_args(meta=True)
+    def fround(self, meta, args) -> Function:
+        if len(args) == 1:
+            args.append(0)
+        return self.function_factory.create_function(args, FunctionType.ROUND, meta)
+
+    @v_args(meta=True)
+    def flog(self, meta, args) -> Function:
+        if len(args) == 1:
+            args.append(10)
+        return self.function_factory.create_function(args, FunctionType.LOG, meta)
+
+    @v_args(meta=True)
+    def ffloor(self, meta, args) -> Function:
+        return self.function_factory.create_function(args, FunctionType.FLOOR, meta)
+
+    @v_args(meta=True)
+    def fceil(self, meta, args) -> Function:
+        return self.function_factory.create_function(args, FunctionType.CEIL, meta)
+
+    @v_args(meta=True)
+    def fcase(self, meta, args: List[Union[CaseWhen, CaseElse]]) -> Function:
+        return self.function_factory.create_function(args, FunctionType.CASE, meta)
+
+    @v_args(meta=True)
+    def fcase_when(self, meta, args) -> CaseWhen:
+        args = process_function_args(args, meta=meta, environment=self.environment)
+        root = expr_to_boolean(args[0], self.function_factory)
+        return CaseWhen(comparison=root, expr=args[1])
+
+    @v_args(meta=True)
+    def fcase_else(self, meta, args) -> CaseElse:
+        args = process_function_args(args, meta=meta, environment=self.environment)
+        return CaseElse(expr=args[0])
+
+    @v_args(meta=True)
+    def fcurrent_date(self, meta, args):
+        return CurrentDate([])
+
+    @v_args(meta=True)
+    def fcurrent_datetime(self, meta, args):
+        return self.function_factory.create_function(
+            args=[], operator=FunctionType.CURRENT_DATETIME, meta=meta
+        )
+
+    @v_args(meta=True)
+    def fcurrent_timestamp(self, meta, args):
+        return self.function_factory.create_function(
+            args=[], operator=FunctionType.CURRENT_TIMESTAMP, meta=meta
+        )
+
+    @v_args(meta=True)
+    def fnot(self, meta, args):
+        if arg_to_datatype(args[0]) == DataType.BOOL:
+            return Comparison(
+                left=self.function_factory.create_function(
+                    [args[0], False], FunctionType.COALESCE, meta
+                ),
+                operator=ComparisonOperator.EQ,
+                right=False,
+                meta=meta,
+            )
+        return self.function_factory.create_function(args, FunctionType.IS_NULL, meta)
+
+    @v_args(meta=True)
+    def fbool(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.BOOL, meta)
+
+    @v_args(meta=True)
+    def fmap_keys(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.MAP_KEYS, meta)
+
+    @v_args(meta=True)
+    def fmap_values(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.MAP_VALUES, meta
+        )
+
+    @v_args(meta=True)
+    def farray_sum(self, meta, args):
+        return self.function_factory.create_function(args, FunctionType.ARRAY_SUM, meta)
+
+    @v_args(meta=True)
+    def fgenerate_array(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.GENERATE_ARRAY, meta
+        )
+
+    @v_args(meta=True)
+    def farray_distinct(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.ARRAY_DISTINCT, meta
+        )
+
+    @v_args(meta=True)
+    def farray_to_string(self, meta, args):
+        return self.function_factory.create_function(
+            args, FunctionType.ARRAY_TO_STRING, meta
+        )
+
+    @v_args(meta=True)
+    def farray_sort(self, meta, args):
+        if len(args) == 1:
+            # this is a magic value to represent the default behavior
+            args = [args[0], Ordering.ASCENDING]
+        return self.function_factory.create_function(
+            args, FunctionType.ARRAY_SORT, meta
+        )
+
+    @v_args(meta=True)
+    def transform_lambda(self, meta, args):
+        return self.environment.functions[args[0]]
+
+    @v_args(meta=True)
+    def farray_transform(self, meta, args) -> Function:
+        factory: CustomFunctionFactory = args[1]
+        if not len(factory.function_arguments) == 1:
+            raise InvalidSyntaxException(
+                "Array transform function must have exactly one argument;"
+            )
+        array_type = arg_to_datatype(args[0])
+        if not isinstance(array_type, ArrayType):
+            raise InvalidSyntaxException(
+                f"Array transform function must be applied to an array, not {array_type}"
+            )
+        return self.function_factory.create_function(
+            [
+                args[0],
+                factory.function_arguments[0],
+                factory(
+                    ArgBinding(
+                        name=factory.function_arguments[0].name,
+                        datatype=array_type.value_data_type,
+                    )
+                ),
+            ],
+            FunctionType.ARRAY_TRANSFORM,
+            meta,
+        )
+
+    @v_args(meta=True)
+    def farray_filter(self, meta, args) -> Function:
+        factory: CustomFunctionFactory = args[1]
+        if not len(factory.function_arguments) == 1:
+            raise InvalidSyntaxException(
+                "Array filter function must have exactly one argument;"
+            )
+        array_type = arg_to_datatype(args[0])
+        if not isinstance(array_type, ArrayType):
+            raise InvalidSyntaxException(
+                f"Array filter function must be applied to an array, not {array_type}"
+            )
+        return self.function_factory.create_function(
+            [
+                args[0],
+                factory.function_arguments[0],
+                factory(
+                    ArgBinding(
+                        name=factory.function_arguments[0].name,
+                        datatype=array_type.value_data_type,
+                    )
+                ),
+            ],
+            FunctionType.ARRAY_FILTER,
+            meta,
+        )
+
+
+def unpack_visit_error(e: VisitError, text: str | None = None):
+    """This is required to get exceptions from imports, which would
+    raise nested VisitErrors"""
+    if isinstance(e.orig_exc, VisitError):
+        unpack_visit_error(e.orig_exc, text)
+    elif isinstance(e.orig_exc, (UndefinedConceptException, ImportError)):
+        raise e.orig_exc
+    elif isinstance(e.orig_exc, InvalidSyntaxException):
+        raise e.orig_exc
+    elif isinstance(e.orig_exc, (SyntaxError, TypeError)):
+        if isinstance(e.obj, Tree):
+            if text:
+                extract = text[e.obj.meta.start_pos - 5 : e.obj.meta.end_pos + 5]
+                raise InvalidSyntaxException(
+                    str(e.orig_exc)
+                    + " Raised when parsing rule: "
+                    + str(e.rule)
+                    + f' Line: {e.obj.meta.line} "...{extract}..."'
+                )
+            InvalidSyntaxException(
+                str(e.orig_exc) + " in " + str(e.rule) + f" Line: {e.obj.meta.line}"
+            )
+        raise InvalidSyntaxException(str(e.orig_exc)).with_traceback(
+            e.orig_exc.__traceback__
+        )
+    raise e.orig_exc
+
+
+def parse_text_raw(text: str, environment: Optional[Environment] = None):
+    PARSER.parse(text)
+
+
+ERROR_CODES: dict[int, str] = {
+    # 100 code are SQL compatability errors
+    101: "Using FROM keyword? Trilogy does not have a FROM clause (Datasource resolution is automatic).",
+    # 200 codes relate to required explicit syntax (we could loosen these?)
+    201: 'Missing alias? Alias must be specified with "AS" - e.g. `SELECT x+1 AS y`',
+    202: "Missing closing semicolon? Statements must be terminated with a semicolon `;`.",
+    210: "Missing order direction? Order by must be explicit about direction - specify `asc` or `desc`.",
+}
+
+DEFAULT_ERROR_SPAN: int = 30
+
+
+def inject_context_maker(pos: int, text: str, span: int = 40) -> str:
+    """Returns a pretty string pinpointing the error in the text,
+    with span amount of context characters around it.
+
+    Note:
+        The parser doesn't hold a copy of the text it has to parse,
+        so you have to provide it again
+    """
+
+    start = max(pos - span, 0)
+    end = pos + span
+    if not isinstance(text, bytes):
+
+        before = text[start:pos].rsplit("\n", 1)[-1]
+        after = text[pos:end].split("\n", 1)[0]
+        rcap = ""
+        # if it goes beyond the end of text, no ...
+        # if it terminates on a space, no need for ...
+        if after and not after[-1].isspace() and not (end > len(text)):
+            rcap = "..."
+        lcap = ""
+        if start > 0 and not before[0].isspace():
+            lcap = "..."
+        lpad = " "
+        rpad = " "
+        if before.endswith(" "):
+            lpad = ""
+        if after.startswith(" "):
+            rpad = ""
+        return f"{lcap}{before}{lpad}???{rpad}{after}{rcap}"
+    else:
+        before = text[start:pos].rsplit(b"\n", 1)[-1]
+        after = text[pos:end].split(b"\n", 1)[0]
+        return (before + b" ??? " + after).decode("ascii", "backslashreplace")
+
+
+def parse_text(
+    text: str,
+    environment: Optional[Environment] = None,
+    root: Path | None = None,
+    parse_config: Parsing | None = None,
+) -> Tuple[
+    Environment,
+    List[
+        Datasource
+        | ImportStatement
+        | SelectStatement
+        | PersistStatement
+        | ShowStatement
+        | RawSQLStatement
+        | ValidateStatement
+        | None
+    ],
+]:
+    def _create_syntax_error(code: int, pos: int, text: str) -> InvalidSyntaxException:
+        """Helper to create standardized syntax error with context."""
+        return InvalidSyntaxException(
+            f"Syntax [{code}]: "
+            + ERROR_CODES[code]
+            + "\nLocation:\n"
+            + inject_context_maker(pos, text.replace("\n", " "), DEFAULT_ERROR_SPAN)
+        )
+
+    def _create_generic_syntax_error(
+        message: str, pos: int, text: str
+    ) -> InvalidSyntaxException:
+        """Helper to create generic syntax error with context."""
+        return InvalidSyntaxException(
+            message
+            + "\nLocation:\n"
+            + inject_context_maker(pos, text.replace("\n", " "), DEFAULT_ERROR_SPAN)
+        )
+
+    def _handle_unexpected_token(e: UnexpectedToken, text: str) -> None:
+        """Handle UnexpectedToken errors to make friendlier error messages."""
+        # Handle ordering direction error
+        pos = e.pos_in_stream or 0
+        if e.interactive_parser.lexer_thread.state:
+            last_token = e.interactive_parser.lexer_thread.state.last_token
+        else:
+            last_token = None
+        if e.expected == {"ORDERING_DIRECTION"}:
+            raise _create_syntax_error(210, pos, text)
+
+        # Handle FROM token error
+        parsed_tokens = (
+            [x.value for x in e.token_history if x] if e.token_history else []
+        )
+
+        if parsed_tokens == ["FROM"]:
+            raise _create_syntax_error(101, pos, text)
+        # check if they are missing a semicolon
+        if last_token and e.token.type == "$END":
+            try:
+
+                e.interactive_parser.feed_token(Token("_TERMINATOR", ";"))
+                state = e.interactive_parser.lexer_thread.state
+                if state and state.last_token:
+                    new_pos = state.last_token.end_pos or pos
+                else:
+                    new_pos = pos
+                raise _create_syntax_error(202, new_pos, text)
+            except UnexpectedToken:
+                pass
+        # check if they forgot an as
+        try:
+            e.interactive_parser.feed_token(Token("AS", "AS"))
+            state = e.interactive_parser.lexer_thread.state
+            if state and state.last_token:
+                new_pos = state.last_token.end_pos or pos
+            else:
+                new_pos = pos
+            e.interactive_parser.feed_token(Token("IDENTIFIER", e.token.value))
+            raise _create_syntax_error(201, new_pos, text)
+        except UnexpectedToken:
+            pass
+
+        # Default UnexpectedToken handling
+        raise _create_generic_syntax_error(str(e), pos, text)
+
+    environment = environment or (
+        Environment(working_path=root) if root else Environment()
+    )
+    parser = ParseToObjects(
+        environment=environment, import_keys=["root"], parse_config=parse_config
+    )
+    start = datetime.now()
+
+    try:
+        parser.set_text(text)
+        # disable fail on missing to allow for circular dependencies
+        parser.prepare_parse()
+        parser.transform(PARSER.parse(text))
+        # this will reset fail on missing
+        pass_two = parser.run_second_parse_pass()
+        output = [v for v in pass_two if v]
+        environment.concepts.fail_on_missing = True
+        end = datetime.now()
+        perf_logger.debug(
+            f"Parse time: {end - start} for {len(text)} characters, {len(output)} objects"
+        )
+    except VisitError as e:
+        unpack_visit_error(e, text)
+        # this will never be reached
+        raise e
+    except UnexpectedToken as e:
+        _handle_unexpected_token(e, text)
+    except (UnexpectedCharacters, UnexpectedEOF, UnexpectedInput) as e:
+        raise _create_generic_syntax_error(str(e), e.pos_in_stream or 0, text)
+    except (ValidationError, TypeError) as e:
+        raise InvalidSyntaxException(str(e))
+
+    return environment, output