relationalai 0.11.2__py3-none-any.whl → 0.11.4__py3-none-any.whl

relationalai/semantics/sql/compiler.py

@@ -52,39 +52,118 @@ class Compiler(c.Compiler):
         """
         return str(self.model_to_sql.to_sql(model, options)), model
 
+@dataclass(frozen=True)
+class OutputVar:
+    value: ir.Value
+    alias: Optional[str] = None
+    value_type: Optional[str] = None
+    task: Optional[ir.Task] = None
+
+@dataclass
+class RelationInfo:
+    used: bool = False
+    view_selects: list[sql.Select] = field(default_factory=list)
+    table_selects: list[sql.Select] = field(default_factory=list)
+    dynamic_table_selects: list[sql.Select] = field(default_factory=list)
+
+@dataclass
+class ImportSpec:
+    value: str
+    module: Optional[str] = None  # e.g., "scipy.special"
+
+    def render(self) -> str:
+        return f"from {self.module} import {self.value}" if self.module else f"import {self.value}"
+
+
+@dataclass
+class UDFConfig:
+    handler: str
+    code: str
+    imports: list[ImportSpec] = field(default_factory=list)
+    packages: list[str] = field(default_factory=list)
 
 @dataclass
 class ModelToSQL:
     """ Generates SQL from an IR Model, assuming the compiler rewrites were done. """
 
     _is_duck_db: bool = False
+    _warehouse: str = 'MAIN_WH'
     _query_compilation: bool = False
+    _default_dynamic_table_target_lag: str = '5 minutes'
     relation_name_cache: NameCache = field(default_factory=NameCache)
     relation_arg_name_cache: NameCache = field(default_factory=NameCache)
-    _error_relation_names: set[str] = field(
-        default_factory=lambda: {'Error', 'pyrel_error_attrs'}
-    )
+    relation_infos: dict[ir.Relation, RelationInfo] = field(default_factory=dict)
+    _error_relation_names: set[str] = field(default_factory=lambda: {'Error', 'pyrel_error_attrs'})
 
     def to_sql(self, model: ir.Model, options:dict) -> sql.Program:
+        self.relation_infos.clear()
         self._is_duck_db = options.get("is_duck_db", False)
+        self._warehouse = options.get("warehouse") or self._warehouse
         self._query_compilation = options.get("query_compilation", False)
+        self._default_dynamic_table_target_lag = (options.get("default_dynamic_table_target_lag") or
+                                                  self._default_dynamic_table_target_lag)
         return sql.Program(self._sort_dependencies(self._union_output_selects(self._generate_statements(model))))
 
     def _generate_statements(self, model: ir.Model) -> list[sql.Node]:
-        relations = self._get_relations(model)
+        table_relations, used_builtins = self._get_relations(model)
 
-        self._register_relation_args(relations)
+        self._register_relation_args(table_relations)
         self._register_external_relations(model)
 
         statements: list[sql.Node] = []
-        for relation in relations:
-            statements.append(self._create_table(cast(ir.Relation, relation)))
+        # 1. Process root logical body
         root = cast(ir.Logical, model.root)
         for child in root.body:
             if isinstance(child, ir.Logical):
                 statements.extend(self._create_statement(cast(ir.Logical, child)))
             elif isinstance(child, ir.Union):
                 statements.append(self._create_recursive_view(cast(ir.Union, child)))
+
+        relation_selects = {
+            relation: info.dynamic_table_selects + info.view_selects + info.table_selects
+            for relation, info in self.relation_infos.items()
+            if info.dynamic_table_selects or info.view_selects or info.table_selects
+        }
+
+        # 3. Handle each relation with proper priority
+        for relation, selects in relation_selects.items():
+            table_name = self._relation_name(relation)
+
+            info = self._get_relation_info(relation)
+            if info.table_selects:
+                # Relation is a table → insert into it
+                columns = [self._var_name(relation.id, f) for f in relation.fields]
+                if len(selects) == 1:
+                    statements.append(sql.Insert(table_name, columns, [], selects[0]))
+                else:
+                    statements.append(sql.Insert(table_name, columns, [],
+                                                 sql.CTE(False, f"{table_name}_cte", columns, selects, True)))
+            elif info.view_selects:
+                statements.append(sql.CreateView(table_name, selects))
+            else:
+                # Snowflake currently has issues when using DISTINCT together with UNION in a Dynamic Table.
+                # As a workaround, we generate a CTE without DISTINCT, using UNION ALL.
+                # Then, we create a dynamic table with `SELECT DISTINCT * FROM CTE` to remove duplicates.
+                columns = [self._var_name(relation.id, f) for f in relation.fields]
+                statements.append(
+                    sql.CreateDynamicTable(
+                        table_name,
+                        sql.CTE(False, f"{table_name}_cte", columns, selects, True),
+                        self._default_dynamic_table_target_lag,
+                        self._warehouse
+                    )
+                )
+
+        # 4. Create physical tables for explicitly declared table relations
+        for relation in table_relations:
+            info = self.relation_infos.get(relation)
+            if info is None or info.table_selects:
+                statements.append(self._create_table(relation))
+
+        #5. Create Snowflake user-defined functions
+        if not self._is_duck_db:
+            statements.extend(self._create_user_defined_functions(used_builtins))
+
         return statements
 
     #--------------------------------------------------
@@ -121,7 +200,9 @@ class ModelToSQL:
 
             aliases = []
             for i, arg in enumerate(update.args):
-                aliases.append((self._var_name(relation.id, relation.fields[i]), arg))
+                relation_field = old_relation.fields[i]
+                field_type = self._convert_type(relation_field.type)
+                aliases.append(OutputVar(arg, self._var_name(old_relation.id, relation_field), value_type=field_type))
 
             return self._make_select(lookups, aliases, nots, unions, constructs)
 
@@ -129,6 +210,7 @@ class ModelToSQL:
         update = v.collect_by_type(ir.Update, union).some()
 
         relation = update.relation
+        self.mark_used(relation)
         return sql.CreateView(
             self._relation_name(relation),
             sql.CTE(
@@ -142,6 +224,62 @@ class ModelToSQL:
             )
         )
 
+    def _create_user_defined_functions(self, relations: list[ir.Relation]) -> list[sql.CreateFunction]:
+        # Central UDF metadata configuration
+        udf_relations: dict[str, UDFConfig] = {
+            builtins.acot.name: UDFConfig(
+                handler="compute",
+                imports=[ImportSpec("math")],
+                code="""def compute(x): return math.atan(1 / x) if x != 0 else math.copysign(math.pi / 2, x)"""
+            ),
+            builtins.erf.name: UDFConfig(
+                handler="compute",
+                imports=[ImportSpec("math")],
+                code="""def compute(x): return math.erf(x)"""
+            ),
+            builtins.erfinv.name: UDFConfig(
+                handler="compute",
+                imports=[ImportSpec("erfinv", module="scipy.special")],
+                packages=["'scipy'"],
+                code="""def compute(x): return erfinv(x)"""
+            )
+        }
+
+        statements: list[sql.CreateFunction] = []
+
+        for r in relations:
+            meta = udf_relations.get(r.name)
+            if not meta:
+                continue
+
+            # Split relation fields into inputs and return type
+            # We expect a single return argument per builtin relation
+            return_type = None
+            input_columns: list[sql.Column] = []
+            for f in r.fields:
+                if f.input:
+                    input_columns.append(sql.Column(self._var_name(r.id, f), self._convert_type(f.type)))
+                else:
+                    return_type = self._convert_type(f.type)
+
+            # Build a full code block (imports + code)
+            imports_code = "\n".join(imp.render() for imp in meta.imports)
+            python_block = "\n".join(part for part in (imports_code, meta.code) if part)
+
+            assert return_type, f"No return type found for relation '{r.name}'"
+            statements.append(
+                sql.CreateFunction(
+                    name=r.name,
+                    inputs=input_columns,
+                    return_type=return_type,
+                    handler=meta.handler,
+                    body=python_block,
+                    packages=meta.packages
+                )
+            )
+
+        return statements
+
     def _create_statement(self, task: ir.Logical):
 
         # TODO - improve the typing info to avoid these casts
@@ -158,7 +296,6 @@ class ModelToSQL:
         var_to_construct = {c.id_var: c for c in constructs} if constructs else {}
 
         statements = []
-        # TODO - this is simplifying soooo much :crying_blood:
         if updates and not lookups and not nots and not aggs and not logicals and not unions:
             for u in updates:
                 r = u.relation
@@ -171,13 +308,15 @@ class ModelToSQL:
                         raise Exception(f"Expected SQL source to be a string, got: {type(src).__name__}")
                     statements.append(sql.RawSource(src))
                 else:
-                    # TODO: this is assuming that the updates are all static values
-                    # Insert static values: INSERT INTO ... SELECT hash(V1, ...), V2, V3 UNION ALL SELECT hash(V4, ...), V5, V6
+                    # Generate select with static values: SELECT hash(V1, ...), V2, V3
                     #   We need to use `SELECT` instead of `VALUES` because Snowflake parses and restricts certain expressions in VALUES(...).
                     #       Built-in functions like HASH() or MD5() are often rejected unless used in SELECT.
-                    statements.append(
-                        sql.Insert(self._relation_name(r), [self._var_name(r.id, f) for f in r.fields], self._get_tuples(task, u), None)
-                    )
+                    for values in self._get_tuples(task, u):
+                        output_vars = [
+                            sql.VarRef(str(value), alias=self._var_name(r.id, f))
+                            for f, value in zip(r.fields, values)
+                        ]
+                        self.add_table_select(r, sql.Select(False, output_vars))
         elif lookups or outputs or nots or aggs or updates:
             # Some of the lookup relations we wrap into logical and we need to get them out for the SQL compilation.
             #    For example QB `decimal(0)` in IR will look like this:
@@ -207,28 +346,39 @@ class ModelToSQL:
                         continue
                     # We shouldn’t create or populate tables for value types that can be directly sourced from existing Snowflake tables.
                     if not self._is_value_type_population_relation(r):
-                        select = None
-                        if aggs:
-                            # After flatten it can be only one aggregation per rule.
-                            select = self._make_agg_select(u, all_lookups, aggs[0], nots, unions, constructs)
-                        elif ranks:
-                            # After flatten it can be only one rank per rule.
-                            select = self._make_rank_select(u, all_lookups, ranks[0], nots, unions, constructs)
+                        if all_lookups and all(builtins.is_builtin(lookup.relation) for lookup in all_lookups):
+                            # Assuming static values insert when you have only builtin lookups (like `cast`, etc.) and you do not have table lookups.
+                            aliases = self._get_update_aliases(u, var_to_construct, var_to_union, True)
+                            select = self._make_select(all_lookups, aliases, nots, unions, constructs)
+                            self.add_table_select(r, select)
                         else:
-                            aliases = []
-                            for i, arg in enumerate(u.args):
-                                field_name = self._var_name(r.id, r.fields[i])
-                                aliases.append(self._get_alias(field_name, arg, var_to_construct, var_to_union))
-
-                            if not unions:
-                                select = self._make_select(all_lookups, aliases, nots, unions, constructs, True)
-                            elif lookups:
-                                select = self._make_match_select(all_lookups, aliases, unions, nots, constructs, True)
+                            select = None
+                            drv = DerivedRelationsVisitor()
+                            task.accept(drv)
+                            if aggs:
+                                # After flatten it can be only one aggregation per rule.
+                                select = self._make_agg_select(u, all_lookups, aggs[0], nots, unions, constructs)
+                            elif ranks:
+                                # After flatten it can be only one rank per rule.
+                                select = self._make_rank_select(u, all_lookups, ranks[0], nots, unions, constructs)
                             else:
-                                select = self._make_full_outer_join_select(aliases, unions, constructs, True)
-
-                        statements.append(sql.Insert(self._relation_name(r), [self._var_name(r.id, f) for f in r.fields],
-                                                     [], select))
+                                # Snowflake currently has issues when using DISTINCT together with UNION in a Dynamic Table.
+                                # That is why we generate statements without DISTINCT, and we remove duplicates later
+                                #   by using CTE + DISTINCT to declare the Dynamic Tables
+                                distinct = True if self._is_duck_db or not drv.is_derived() else False
+                                aliases = self._get_update_aliases(u, var_to_construct, var_to_union)
+
+                                if not unions:
+                                    select = self._make_select(all_lookups, aliases, nots, unions, constructs, distinct)
+                                elif lookups:
+                                    select = self._make_match_select(all_lookups, aliases, unions, nots, constructs, distinct)
+                                else:
+                                    select = self._make_full_outer_join_select(aliases, unions, constructs, distinct)
+
+                            if drv.is_derived() and not self._is_duck_db:
+                                self.add_dynamic_table_select(r, select)
+                            else:
+                                self.add_view_select(r, select)
             elif outputs:
                 # output a query: SELECT ... FROM ... WHERE ...
                 aliases = []
@@ -236,7 +386,7 @@ class ModelToSQL:
                 for output in outputs:
                     distinct = distinct or output.keys is None
                     for key, arg in output.aliases:
-                        aliases.append(self._get_alias(key, arg, var_to_construct, var_to_union))
+                        aliases.append(self._get_alias(key, arg, None, var_to_construct, var_to_union))
 
                 if not unions:
                     if all(builtins.is_builtin(lookup.relation) for lookup in all_lookups):
@@ -331,7 +481,7 @@ class ModelToSQL:
 
         seen_args = set()
         outputs: list[Union[sql.VarRef, sql.RowNumberVar, int]] = []
-        sub_query_outputs: list[Tuple[Optional[str], ir.Value]|Tuple[Optional[str], ir.Value, ir.Task]] = []
+        sub_query_outputs: list[OutputVar] = []
 
         relation = update.relation
         agg_var = agg.args[0] if agg.aggregation == builtins.count else agg.args[1]
@@ -340,22 +490,23 @@ class ModelToSQL:
 
         for i, arg in enumerate(update.args):
             if arg not in seen_args:
-                field_name = self._var_name(relation.id, relation.fields[i])
+                relation_field = relation.fields[i]
+                field_type = self._convert_type(relation_field.type)
+                field_name = self._var_name(relation.id, relation_field)
                 if isinstance(arg, ir.Var) and arg == agg_var:
-                    outputs.append(sql.VarRef(f"{agg.aggregation.name}({field_name})", alias=field_name))
-                    sub_query_outputs.append((field_name, arg, agg))
+                    outputs.append(sql.VarRef(f"{agg.aggregation.name}({field_name})", alias=field_name, type=field_type))
+                    sub_query_outputs.append(OutputVar(arg, field_name, task=agg))
                 else:
-                    var_ref = sql.VarRef(field_name)
-                    group_by.append(var_ref)
-                    outputs.append(var_ref)
-                    sub_query_outputs.append((field_name, arg))
+                    group_by.append(sql.VarRef(field_name))
+                    outputs.append(sql.VarRef(field_name, alias=field_name, type=field_type))
+                    sub_query_outputs.append(OutputVar(arg, field_name))
                 seen_args.add(arg)
 
         for arg in agg.projection:
             if arg not in seen_args:
                 if agg.aggregation == builtins.count and arg == agg.projection[-1]:
                     continue
-                sub_query_outputs.append((None, arg))
+                sub_query_outputs.append(OutputVar(value=arg))
                 seen_args.add(arg)
 
         sub_select = self._make_select(lookups, sub_query_outputs, nots, unions, constructs, True)
@@ -379,7 +530,7 @@ class ModelToSQL:
                     Cat AS v0, cat_name AS v1
                 WHERE
                     v0.cat = v1.cat
-            ) LIMIT 10;
+            ) ORDER BY v LIMIT 10;
 
         Rationale:
         In the IR, it’s not always explicit whether rank should be applied over distinct rows.
@@ -411,7 +562,7 @@ class ModelToSQL:
 
         seen_args = set()
         outputs: list[Union[sql.VarRef, sql.RowNumberVar, int]] = []
-        sub_query_outputs: list[Tuple[Optional[str], ir.Value] | Tuple[Optional[str], ir.Value, ir.Task]] = []
+        sub_query_outputs: list[OutputVar] = []
 
         order_by_vars = []
         for arg, is_ascending in zip(rank.args, rank.arg_is_ascending):
@@ -420,30 +571,33 @@ class ModelToSQL:
 
         relation = update.relation
 
+        rank_result_field_name = None
         for i, arg in enumerate(update.args):
             if arg not in seen_args:
-                field_name = self._var_name(relation.id, relation.fields[i])
+                relation_field = relation.fields[i]
+                field_type = self._convert_type(relation_field.type)
+                field_name = self._var_name(relation.id, relation_field)
                 if isinstance(arg, ir.Var) and arg == rank.result:
-                    outputs.append(sql.RowNumberVar(order_by_vars, partition_by_vars, field_name))
+                    rank_result_field_name = field_name
+                    outputs.append(sql.RowNumberVar(order_by_vars, partition_by_vars, field_name, field_type))
                 else:
-                    outputs.append(sql.VarRef(field_name))
-                sub_query_outputs.append((field_name, arg))
+                    outputs.append(sql.VarRef(field_name, alias=field_name, type=field_type))
+                sub_query_outputs.append(OutputVar(arg, field_name))
                 seen_args.add(arg)
 
         for arg in rank.projection:
             if arg not in seen_args:
-                sub_query_outputs.append((None, arg))
+                sub_query_outputs.append(OutputVar(value=arg))
                 seen_args.add(arg)
 
         sub_select = self._make_select(lookups, sub_query_outputs, nots, unions, constructs, True)
 
-        return sql.Select(False, outputs, sub_select, limit=rank.limit)
+        assert rank_result_field_name is not None, "Rank result variable not found in update.args."
+        return sql.Select(False, outputs, sub_select, order_by=[sql.VarRef(rank_result_field_name)], limit=rank.limit)
 
-    def _make_match_select(self, lookups: list[ir.Lookup],
-                           outputs: list[Tuple[Optional[str], ir.Value] | Tuple[Optional[str], ir.Value, ir.Task]],
-                           unions: list[ir.Union], nots: Optional[list[ir.Not]] = None,
-                           constructs: Optional[list[ir.Construct]] = None, distinct: bool = False,
-                           is_output: bool = False):
+    def _make_match_select(self, lookups: list[ir.Lookup], outputs: list[OutputVar], unions: list[ir.Union],
+                           nots: Optional[list[ir.Not]] = None, constructs: Optional[list[ir.Construct]] = None,
+                           distinct: bool = False, is_output: bool = False):
 
         """
         Generate a SQL SELECT statement representing a match operation.
@@ -480,8 +634,10 @@ class ModelToSQL:
         froms, joins, wheres, sql_vars, var_column, var_lookups = self._extract_match_lookups_metadata(table_lookups, union_lookups)
 
         builtin_lookups = OrderedSet.from_iterable(t for t in lookups if builtins.is_builtin(t.relation))
-        builtin_vars, builtin_wheres = self._resolve_builtins(builtin_lookups, var_lookups, var_column, sql_vars,
-                                                              var_to_construct, outputs)
+        builtin_vars, builtin_wheres, builtin_table_expressions = (
+            self._resolve_builtins(builtin_lookups, var_lookups, var_column, sql_vars, var_to_construct, outputs))
+
+        froms.extend(self._process_builtin_table_expressions(builtin_table_expressions))
 
         wheres.extend(builtin_wheres)
 
@@ -502,10 +658,9 @@ class ModelToSQL:
 
         return sql.Select(distinct, vars, froms, where, joins, is_output=is_output)
 
-    def _make_full_outer_join_select(self,
-                                     outputs: list[Tuple[Optional[str], ir.Value] | Tuple[Optional[str], ir.Value, ir.Task]],
-                                     unions: list[ir.Union], constructs: Optional[list[ir.Construct]] = None,
-                                     distinct: bool = False, is_output: bool = False):
+    def _make_full_outer_join_select(self, outputs: list[OutputVar], unions: list[ir.Union],
+                                     constructs: Optional[list[ir.Construct]] = None, distinct: bool = False,
+                                     is_output: bool = False):
 
         """
         Generate a SQL SELECT statement representing a match operation that combines multiple sets of data
@@ -563,8 +718,7 @@ class ModelToSQL:
 
         return sql.Select(distinct, vars, froms, where, joins, is_output=is_output)
 
-    def _make_left_outer_join_select(self, task: ir.Logical, lookups: list[ir.Lookup],
-                                     outputs: list[Tuple[Optional[str], ir.Value] | Tuple[Optional[str], ir.Value, ir.Task]],
+    def _make_left_outer_join_select(self, task: ir.Logical, lookups: list[ir.Lookup], outputs: list[OutputVar],
                                      nots: Optional[list[ir.Not]] = None, constructs: Optional[list[ir.Construct]] = None,
                                      distinct: bool = False) -> sql.Select:
 
@@ -661,8 +815,14 @@ class ModelToSQL:
             self._extract_left_outer_joins_lookups_metadata(task, table_lookups, nots))
 
         builtin_lookups = OrderedSet.from_iterable(t for t in lookups if builtins.is_builtin(t.relation))
-        builtin_vars, builtin_wheres = self._resolve_builtins(builtin_lookups, var_lookups, var_column, sql_vars,
-                                                              var_to_construct, outputs)
+        builtin_vars, builtin_wheres, builtin_table_expressions = (
+            self._resolve_builtins(builtin_lookups, var_lookups, var_column, sql_vars, var_to_construct, outputs))
+
+        # SF in case of `LEFT OUTER JOIN` and `ARRAY_GENERATE_RANGE` doesn't allow usage of `ON TRUE` but
+        #   for DuckDB this is mandatory that is why we have 2 different join classes.
+        make_join = (lambda e, a: sql.Join(e, a)) if self._is_duck_db else (lambda e, a: sql.JoinWithoutCondition(e, a))
+        joins.extend(make_join(expr, alias) for alias, expr in builtin_table_expressions.items())
+
         wheres.extend(builtin_wheres)
 
         construct_wheres = self._process_constructs(table_lookups, var_lookups, var_column, sql_vars, builtin_vars,
@@ -678,10 +838,9 @@ class ModelToSQL:
 
         return sql.Select(distinct, vars, froms, where, joins, is_output=True)
 
-    def _make_select(self, lookups: list[ir.Lookup],
-                     outputs: list[Tuple[Optional[str], ir.Value]|Tuple[Optional[str], ir.Value, ir.Task]],
-                     nots: Optional[list[ir.Not]] = None, unions: Optional[list[ir.Union]] = None,
-                     constructs: Optional[list[ir.Construct]] = None, distinct: bool = False, is_output: bool = False) -> sql.Select:
+    def _make_select(self, lookups: list[ir.Lookup], outputs: list[OutputVar], nots: Optional[list[ir.Not]] = None,
+                     unions: Optional[list[ir.Union]] = None, constructs: Optional[list[ir.Construct]] = None,
+                     distinct: bool = False, is_output: bool = False) -> sql.Select:
 
         var_to_construct = {c.id_var: c for c in constructs} if constructs else {}
 
@@ -692,8 +851,11 @@ class ModelToSQL:
         froms, wheres, sql_vars, var_column, var_lookups = self._extract_lookups_metadata(table_lookups)
 
         builtin_lookups = OrderedSet.from_iterable(t for t in all_lookups if builtins.is_builtin(t.relation))
-        builtin_vars, builtin_wheres = self._resolve_builtins(builtin_lookups, var_lookups, var_column, sql_vars,
-                                                              var_to_construct, outputs)
+        builtin_vars, builtin_wheres, builtin_table_expressions = (
+            self._resolve_builtins(builtin_lookups, var_lookups, var_column, sql_vars, var_to_construct, outputs))
+
+        froms.extend(self._process_builtin_table_expressions(builtin_table_expressions))
+
         wheres.extend(builtin_wheres)
 
         construct_wheres = self._process_constructs(table_lookups, var_lookups, var_column, sql_vars, builtin_vars,
@@ -991,14 +1153,19 @@ class ModelToSQL:
         """Generate hash expression like hash(`x`, `y`, TABLE_ALIAS.COLUMN_NAME)."""
         elements = []
         for val in values:
-            val = resolve_builtin_var(val)
-            if isinstance(val, ir.Var):
-                if val in var_to_construct:
-                    elements.append(self._resolve_construct_var(reference, resolve_builtin_var, var_to_construct, var_to_construct[val]))
+            resolved_val = resolve_builtin_var(val)
+            if val != resolved_val and isinstance(resolved_val, str):
+                # In case we parsed builtin into some expression, we may add it as an element.
+                # For example, `TO_DATE('1990-1-1', 'Y-m-d')` or `(v1.value + 5)`.
+                elements.append(f"{resolved_val}")
+                continue
+            if isinstance(resolved_val, ir.Var):
+                if resolved_val in var_to_construct:
+                    elements.append(self._resolve_construct_var(reference, resolve_builtin_var, var_to_construct, var_to_construct[resolved_val]))
                 else:
-                    elements.append(reference(val))
+                    elements.append(reference(resolved_val))
             else:
-                elements.append(self._convert_value(val, True))
+                elements.append(str(self._convert_value(resolved_val)))
         return f"hash({', '.join(elements)})"
 
     def _resolve_construct_var(self, reference, resolve_builtin_var, var_to_construct, construct: ir.Construct):
@@ -1012,17 +1179,20 @@ class ModelToSQL:
     def _resolve_builtins(self, builtin_lookups: OrderedSet[ir.Lookup], var_lookups: dict[ir.Var, OrderedSet[ir.Lookup]],
                           var_column: dict[Tuple[ir.Var, ir.Lookup], ir.Field], sql_vars: dict[ir.Lookup, str],
                           var_to_construct: dict[ir.Var, ir.Construct],
-                          outputs: Optional[list[Tuple[Optional[str], ir.Value]|Tuple[Optional[str], ir.Value, ir.Task]]] = None):
+                          outputs: Optional[list[OutputVar]] = None):
 
         wheres: list[sql.Expr] = []
+        # We need to maintain a mapping of these builtin expressions because they generate a new table, which must be
+        #   referenced in the FROM clause as part of a JOIN. Structure is `SQL table variable` -> `generated expression`
+        table_expressions: dict[str, str] = {}
         builtin_vars: dict[ir.Var, ir.Value|str|int] = {}
         # TODO: remove this when we introduce date periods in builtins
         date_period_var_type: dict[ir.Var, str] = {}
 
         output_vars = {
-            output[1]
+            output.value
             for output in outputs or []
-            if isinstance(output[1], ir.Var)
+            if isinstance(output.value, ir.Var)
         }
 
         intermediate_builtin_vars: set[ir.Var] = {
@@ -1073,6 +1243,46 @@ class ModelToSQL:
                 idx = self._var_to_expr(idx_raw, reference, resolve_builtin_var, var_to_construct)
                 assert isinstance(output, ir.Var)
                 builtin_vars[output] = f"split_part({s}, {separator}, {idx})"
+            elif relation == builtins.split:
+                assert len(args) == 4, f"Expected 4 args for `split`, got {len(args)}: {args}"
+                separator_raw, value_raw, index, part = args
+                value = self._var_to_expr(value_raw, reference, resolve_builtin_var, var_to_construct)
+                separator = self._var_to_expr(separator_raw, reference, resolve_builtin_var, var_to_construct)
+                table_sql_var = f"v{len(sql_vars)}"
+                sql_vars[lookup] = table_sql_var
+                if self._is_duck_db:
+                    table_alias = f"{table_sql_var}(data)"
+                    table_expressions[table_alias] = f"VALUES(string_split({value}, {separator}))"
+
+                    part_expr = f"unnest({table_sql_var}.data)"
+                    index_expr = f"generate_subscripts({table_sql_var}.data, 1)"
+                else:
+                    table_expressions[table_sql_var] = f"LATERAL FLATTEN(input => SPLIT({value}, {separator}))"
+
+                    # SF returns values in `""` and to avoid this, we need to cast it to `TEXT` type
+                    part_expr = f"cast({table_sql_var}.value as TEXT)"
+                    index_expr = f"({table_sql_var}.index + 1)" # SF is 0-based internally, adjust to it back
+                assert isinstance(index, ir.Var) and isinstance(part, ir.Var), "Third and fourth arguments (index, part) must be variables"
+                builtin_vars[part] = part_expr
+                builtin_vars[index] = index_expr
+            elif relation == builtins.range:
+                assert len(args) == 4, f"Expected 4 args for `range`, got {len(args)}: {args}"
+                start_raw, stop_raw, step_raw, result = args
+                start = self._var_to_expr(start_raw, reference, resolve_builtin_var, var_to_construct)
+                stop = self._var_to_expr(stop_raw, reference, resolve_builtin_var, var_to_construct)
+                step = self._var_to_expr(step_raw, reference, resolve_builtin_var, var_to_construct)
+                table_sql_var = f"v{len(sql_vars)}"
+                sql_vars[lookup] = table_sql_var
+                # In SQL range is 1...stop exclusive, and because we did `-1` in PyRel v1 we need to return it here
+                if self._is_duck_db:
+                    table_expr = f"LATERAL range(cast({start} as bigint), cast(({stop} + 1) as bigint), cast({step} as bigint))"
+                    expr = f"{table_sql_var}.range"
+                else:
+                    table_expr = f"LATERAL FLATTEN(input => ARRAY_GENERATE_RANGE({start}, ({stop} + 1), {step}))"
+                    expr = f"{table_sql_var}.value"
+                table_expressions[table_sql_var] = table_expr
+                assert isinstance(result, ir.Var), "Fourth argument (result) must be a variable"
+                builtin_vars[result] = f"{expr}"
             elif relation == builtins.cast:
                 assert len(args) == 3, f"Expected 3 args for `cast`, got {len(args)}: {args}"
 
@@ -1080,38 +1290,50 @@ class ModelToSQL:
                 assert isinstance(result, ir.Var), "Third argument (result) must be a variable"
 
                 builtin_vars[result] = original_raw
-            elif relation in (builtins.isnan, builtins.isinf):
+            elif relation in {builtins.isnan, builtins.isinf}:
                 arg_expr = self._var_to_expr(args[0], reference, resolve_builtin_var, var_to_construct)
                 expr = "cast('NaN' AS DOUBLE)" if relation == builtins.isnan else "cast('Infinity' AS DOUBLE)"
                 wheres.append(sql.Terminal(f"{arg_expr} = {expr}"))
+            elif relation == builtins.construct_date:
+                assert len(args) == 4, f"Expected 4 args for `construct_date`, got {len(args)}: {args}"
+                year_raw, month_raw, day_raw, result = args
+                year = self._var_to_expr(year_raw, reference, resolve_builtin_var, var_to_construct)
+                month = self._var_to_expr(month_raw, reference, resolve_builtin_var, var_to_construct)
+                day = self._var_to_expr(day_raw, reference, resolve_builtin_var, var_to_construct)
+
+                assert isinstance(result, ir.Var), "Fourth argument (result) must be a variable."
+                if self._is_duck_db:
+                    expr = f"make_date(cast({year} as bigint), cast({month} as bigint), cast({day} as bigint))"
+                else:
+                    expr = f"date_from_parts({year}, {month}, {day})"
+                builtin_vars[result] = expr
             elif relation == builtins.construct_datetime_ms_tz:
                 assert len(args) == 9, f"Expected 9 args for `construct_datetime_ms_tz`, got {len(args)}: {args}"
 
-                year_raw, month_raw, day_raw, hour_raw, minute_raw, second_raw, millisecond_raw, tz, result = args
-                tz = self._convert_value(tz, quote_strings=False)
-                assert isinstance(tz, str), "Timezone argument (tz) must be a string."
+                year_raw, month_raw, day_raw, hour_raw, minute_raw, second_raw, millisecond_raw, tz_raw, result = args
                 assert isinstance(result, ir.Var), "Ninth argument (result) must be a variable."
 
                 year = self._var_to_expr(year_raw, reference, resolve_builtin_var, var_to_construct)
                 month = self._var_to_expr(month_raw, reference, resolve_builtin_var, var_to_construct)
                 day = self._var_to_expr(day_raw, reference, resolve_builtin_var, var_to_construct)
-                hour = self._convert_value(hour_raw)
-                minute = self._convert_value(minute_raw)
-                second = self._convert_value(second_raw)
-                millisecond = self._convert_value(millisecond_raw)
+                hour = self._var_to_expr(hour_raw, reference, resolve_builtin_var, var_to_construct)
+                minute = self._var_to_expr(minute_raw, reference, resolve_builtin_var, var_to_construct)
+                second = self._var_to_expr(second_raw, reference, resolve_builtin_var, var_to_construct)
+                millisecond = self._var_to_expr(millisecond_raw, reference, resolve_builtin_var, var_to_construct)
+                tz = self._var_to_expr(tz_raw, reference, resolve_builtin_var, var_to_construct)
 
                 if self._is_duck_db:
-                    sub_expr = (f"make_timestamp({year}, {month}, {day}, {hour}, {minute}, "
-                                f"{second} + {millisecond}/1000.0)")
-                    if tz.lower() != "utc":
-                        sub_expr = f"(({sub_expr} at time zone '{tz}') at time zone 'UTC')"
+                    sub_expr = (f"make_timestamp(cast({year} as bigint), cast({month} as bigint), cast({day} as bigint), "
+                                f"cast({hour} as bigint), cast({minute} as bigint), cast({second} as bigint) + {millisecond}/1000.0)")
+                    if tz.lower() != "'utc'":
+                        sub_expr = f"(({sub_expr} at time zone {tz}) at time zone 'UTC')"
                 else:
                     sub_expr = (f"to_timestamp_ntz(lpad({year}, 4, '0') || '-' || lpad({month}, 2, '0') || '-' || "
                                 f"lpad({day}, 2, '0') || ' ' || lpad({hour}, 2, '0') || ':' || "
                                 f"lpad({minute}, 2, '0') || ':' || lpad({second}, 2, '0') || '.' || "
                                 f"lpad({millisecond}, 3, '0'), 'YYYY-MM-DD HH24:MI:SS.FF3')")
-                    if tz.lower() != "utc":
-                        sub_expr = f"convert_timezone('{tz}', 'UTC', {sub_expr})"
+                    if tz.lower() != "'utc'":
+                        sub_expr = f"convert_timezone({tz}, 'UTC', {sub_expr})"
                 builtin_vars[result] = f"cast({sub_expr} as DATETIME)"
             else:
                 # Assuming infix binary or ternary operators here
@@ -1150,6 +1372,12 @@ class ModelToSQL:
                     function = "levenshtein" if self._is_duck_db else "editdistance"
                     assert isinstance(args[2], ir.Var)
                     builtin_vars[args[2]] = f"{function}({left}, {right})"
+                elif relation == builtins.concat:
+                    assert len(args) == 3, f"Expected 3 args for `concat`, got {len(args)}: {args}"
+                    left = self._var_to_expr(lhs, reference, resolve_builtin_var, var_to_construct)
+                    right = self._var_to_expr(rhs, reference, resolve_builtin_var, var_to_construct)
+                    assert isinstance(args[2], ir.Var)
+                    builtin_vars[args[2]] = f"concat({left}, {right})"
                 elif relation == builtins.join:
                     assert len(args) == 3, f"Expected 3 args for `join`, got {len(args)}: {args}"
                     assert isinstance(lhs, tuple)
@@ -1175,12 +1403,32 @@ class ModelToSQL:
                 elif relation == builtins.parse_float and isinstance(rhs, ir.Var):
                     left = self._var_to_expr(lhs, reference, resolve_builtin_var, var_to_construct)
                     builtin_vars[rhs] = f"cast({left} AS DOUBLE)"
+                elif relation == builtins.parse_date:
+                    if self._is_duck_db:
+                        raise Exception("DuckDB: unsupported builtin relation 'parse_date'.")
+                    assert len(args) == 3, f"Expected 3 args for `parse_date`, got {len(args)}: {args}"
+                    left = self._var_to_expr(lhs, reference, resolve_builtin_var, var_to_construct)
+                    right = self._var_to_expr(rhs, reference, resolve_builtin_var, var_to_construct)
+                    assert isinstance(args[2], ir.Var)
+                    builtin_vars[args[2]] = f"to_date({left}, {right})"
+                elif relation == builtins.parse_datetime:
+                    assert len(args) == 3, f"Expected 3 args for `parse_datetime`, got {len(args)}: {args}"
+                    left = self._var_to_expr(lhs, reference, resolve_builtin_var, var_to_construct)
+                    right = self._var_to_expr(rhs, reference, resolve_builtin_var, var_to_construct)
+                    sub_expr = left
+                    if 'z' in right:  # this means that out datetime formatter includes timezone, and we need to convert first.
+                        if self._is_duck_db:
+                            sub_expr = f"({left} AT TIME ZONE 'UTC')"
+                        else:
+                            sub_expr = f"convert_timezone('UTC', to_timestamp_tz({left}))"
+                    assert isinstance(args[2], ir.Var)
+                    builtin_vars[args[2]] = f"cast({sub_expr} as DATETIME)"
                 elif relation in builtins.date_periods and isinstance(rhs, ir.Var):
                     builtin_vars[rhs] = lhs
                     date_period_var_type[rhs] = relation.name
                 elif relation in builtins.date_builtins:
-                    if relation in (builtins.date_add, builtins.date_subtract, builtins.datetime_add,
-                                    builtins.datetime_subtract):
+                    if relation in {builtins.date_add, builtins.date_subtract, builtins.datetime_add,
+                                    builtins.datetime_subtract}:
                         assert len(args) == 3, f"Expected 3 args for {relation}, got {len(args)}: {args}"
                         assert isinstance(rhs, ir.Var), f"Period variable must be `ir.Var`, got: {rhs}"
                         period = date_period_var_type[rhs]
@@ -1189,10 +1437,10 @@ class ModelToSQL:
                         left = self._var_to_expr(lhs, reference, resolve_builtin_var, var_to_construct)
 
                         if self._is_duck_db:
-                            op = "+" if relation in (builtins.date_add, builtins.datetime_add) else "-"
-                            expr = f"({left} {op} interval {period_val} {period})"
+                            op = "+" if relation in {builtins.date_add, builtins.datetime_add} else "-"
+                            expr = f"({left} {op} {period_val} * interval 1 {period})"
                         else:
-                            sign = 1 if relation in (builtins.date_add, builtins.datetime_add) else -1
+                            sign = 1 if relation in {builtins.date_add, builtins.datetime_add} else -1
                             expr = f"dateadd({period}, ({sign} * {period_val}), {left})"
 
                         result_var = args[2]
@@ -1206,9 +1454,13 @@ class ModelToSQL:
                         assert isinstance(rhs, ir.Var), f"Resulting variable must be `ir.Var`, got: {rhs}"
                         expr_map = {
                             builtins.date_year: "year",
+                            builtins.date_quarter: "quarter",
                             builtins.date_month: "month",
                             builtins.date_week: "week",
-                            builtins.date_day: "day"
+                            builtins.date_day: "day",
+                            builtins.date_dayofyear: "dayofyear",
+                            builtins.date_weekday: "isodow" if self._is_duck_db else "dayofweekiso",
+                            builtins.datetime_second: "second",
                         }
                         expr = expr_map.get(relation)
                         lhs = self._var_to_expr(lhs, reference, resolve_builtin_var, var_to_construct)
@@ -1217,19 +1469,34 @@ class ModelToSQL:
                         result_var = args[2]
                         assert isinstance(result_var, ir.Var), f"Resulting variable must be `ir.Var`, got: {result_var}"
                         expr_map = {
+                            builtins.datetime_year: "year",
+                            builtins.datetime_quarter: "quarter",
+                            builtins.datetime_month: "month",
                             builtins.datetime_week: "week",
+                            builtins.datetime_day: "day",
+                            builtins.datetime_dayofyear: "dayofyear",
+                            builtins.datetime_hour: "hour",
+                            builtins.datetime_minute: "minute",
+                            builtins.datetime_weekday: "isodow" if self._is_duck_db else "dayofweekiso",
+                            builtins.dates_period_days: "date_diff" if self._is_duck_db else "datediff",
+                            builtins.datetimes_period_milliseconds: "date_diff" if self._is_duck_db else "datediff"
                         }
                         expr = expr_map.get(relation)
                         lhs = self._var_to_expr(lhs, reference, resolve_builtin_var, var_to_construct)
-                        rhs = self._var_to_expr(rhs, reference, resolve_builtin_var, var_to_construct, quote_strings=False)
-                        sub_expr = self._convert_timezone(lhs, rhs)
+                        rhs = self._var_to_expr(rhs, reference, resolve_builtin_var, var_to_construct)
+                        if relation == builtins.dates_period_days:
+                            sub_expr = f"'day', {lhs}, {rhs}" if self._is_duck_db else f"day, {lhs}, {rhs}"
+                        elif relation == builtins.datetimes_period_milliseconds:
+                            sub_expr = f"'millisecond', {lhs}, {rhs}" if self._is_duck_db else f"millisecond, {lhs}, {rhs}"
+                        else:
+                            sub_expr = self._convert_timezone(lhs, rhs)
                         builtin_vars[result_var] = f"{expr}({sub_expr})"
                     else:
                         raise NotImplementedError("Unsupported number of arguments for date builtin (3+).")
                 elif relation == builtins.construct_date_from_datetime:
-                    assert len(args) == 3, f"Expected 3 args for `construct_date`, got {len(args)}: {args}"
+                    assert len(args) == 3, f"Expected 3 args for `construct_date_from_datetime`, got {len(args)}: {args}"
                     dt_raw, tz, result = args
-                    tz = self._convert_value(tz, quote_strings=False)
+                    tz = self._convert_value(tz)
 
                     assert isinstance(tz, str), "Timezone argument (tz) must be a string."
                     assert isinstance(result, ir.Var), "Third argument (result) must be a variable."
@@ -1253,9 +1520,14 @@ class ModelToSQL:
                         if rel_name == builtins.factorial.name and self._is_duck_db:
                             # Factorial requires an integer operand in DuckDB
                             sub_expr = f"{left}::INTEGER"
+                        elif rel_name == builtins.log10.name:
+                            # log10 is not supported, so we use log with base 10
+                            sub_expr = f"10, {left}"
+                            method = "log"
                         expr = f"{method}({sub_expr})"
-                    elif rel_name in (builtins.minimum.name, builtins.maximum.name, builtins.trunc_div.name,
-                                      builtins.power.name, builtins.mod.name, builtins.pow.name):
+                    elif rel_name in {builtins.minimum.name, builtins.maximum.name, builtins.trunc_div.name,
+                                      builtins.power.name, builtins.mod.name, builtins.pow.name,
+                                      builtins.log.name}:
                         assert len(args) == 3, f"Expected 3 args for {relation}, got {len(args)}: {args}"
 
                         result_var = args[2]
@@ -1269,6 +1541,8 @@ class ModelToSQL:
                             expr = f"trunc({left} / {right})"
                         elif rel_name == builtins.power.name or rel_name == builtins.pow.name:
                             expr = f"power({left}, {right})"
+                        elif rel_name == builtins.log.name:
+                            expr = f"log({left}, {right})"
                         else:
                             expr = f"mod({left}, {right})"
                     else:
@@ -1278,15 +1552,11 @@ class ModelToSQL:
                         f"but got `{type(result_var).__name__}`: {result_var}"
                     )
                     builtin_vars[result_var] = expr
-                elif relation in (builtins.parse_int64, builtins.parse_int128) and isinstance(rhs, ir.Var):
+                elif relation in {builtins.parse_int64, builtins.parse_int128} and isinstance(rhs, ir.Var):
                     builtin_vars[rhs] = self._var_to_expr(lhs, reference, resolve_builtin_var, var_to_construct, False)
                 elif helpers.is_from_cast(lookup) and isinstance(rhs, ir.Var):
                     # For the `from cast` relations we keep the raw var, and we will ground it later.
                     builtin_vars[rhs] = lhs
-                elif isinstance(lhs, ir.Var) and lhs in output_vars & intermediate_builtin_vars:
-                    builtin_vars[lhs] = self._var_to_expr(rhs, reference, resolve_builtin_var, var_to_construct)
-                elif isinstance(rhs, ir.Var) and rhs in output_vars & intermediate_builtin_vars:
-                    builtin_vars[rhs] = self._var_to_expr(lhs, reference, resolve_builtin_var, var_to_construct)
                 elif isinstance(lhs, ir.Var) and lhs in intermediate_builtin_vars and lhs not in (builtin_vars | var_to_construct):
                     # Example IR:
                     #   Logical
@@ -1314,26 +1584,13 @@ class ModelToSQL:
                         out_var = args[2]
                         if isinstance(out_var, ir.Var):
                             out_var = resolve_builtin_var(out_var)
-                            if relation == builtins.concat:
-                                expr = f"{relation_name}({left}, {right})"
-                            elif relation == builtins.parse_date:
-                                expr = f"cast({left} as DATE)"
-                            elif relation == builtins.parse_datetime:
-                                sub_expr = left
-                                if 'z' in right: # this means that out datetime formatter includes timezone and we need to convert first.
-                                    if self._is_duck_db:
-                                        sub_expr = f"({left} AT TIME ZONE 'UTC')"
-                                    else:
-                                        sub_expr = f"convert_timezone('UTC', to_timestamp_tz({left}))"
-                                expr = f"cast({sub_expr} as DATETIME)"
-                            else:
-                                expr = f"({left} {relation_name} {right})"
+                            expr = f"({left} {relation_name} {right})"
                             if isinstance(out_var, ir.Var):
                                 # For example, when this is an intermediate result
                                 # example: c = a - b in the IR is (a - b = d) and (d = c)
                                 builtin_vars[out_var] = expr
                             else:
-                                # This means that var was already grounded and we can add WHERE clause.
+                                # This means that var was already grounded, and we can add a WHERE clause.
                                 wheres.append(sql.Terminal(f"{expr} = {out_var}"))
                         else:
                             raise Exception(
@@ -1363,16 +1620,23 @@ class ModelToSQL:
                         rhs = str(rhs_ref)
                     wheres.append(sql.Terminal(f"{ref} = {rhs}"))
 
-        return builtin_vars, wheres
+        return builtin_vars, wheres, table_expressions
 
     def _convert_timezone(self, dt: str, tz: str) -> str:
-        if tz.lower() != "utc":
+        if tz.lower() != "'utc'":
             if self._is_duck_db:
-                return f"({dt} at time zone 'UTC') at time zone '{tz}'"
+                return f"({dt} at time zone 'UTC') at time zone {tz}"
             else:
-                return f"convert_timezone('UTC', '{tz}', {dt})"
+                return f"convert_timezone('UTC', {tz}, {dt})"
         return dt
 
+    def _process_builtin_table_expressions(self, builtin_table_expressions: dict[str, str]):
+        """Convert builtin table expressions into SQL FROM clauses."""
+        return [
+            sql.From(expr, alias)
+            for alias, expr in builtin_table_expressions.items()
+        ]
+
     def _process_constructs(self, lookups: OrderedSet[ir.Lookup], var_lookups: dict[ir.Var, OrderedSet[ir.Lookup]],
                             var_column: dict[Tuple[ir.Var, ir.Lookup], ir.Field], sql_vars: dict[ir.Lookup, str],
                             builtin_vars: dict[ir.Var, ir.Value|str|int], var_to_construct: dict[ir.Var, ir.Construct]) -> list[sql.Expr]:
@@ -1485,21 +1749,20 @@ class ModelToSQL:
             where = sql.Where(sql.And(wheres))
         return where
 
-    def _generate_select_output(self, outputs: list[Tuple[Optional[str], ir.Value]|Tuple[Optional[str], ir.Value, ir.Task]],
-                                builtin_vars: dict[ir.Var, ir.Value|str|int], sql_vars: dict[ir.Lookup, str],
-                                var_column: dict[Tuple[ir.Var, ir.Lookup], ir.Field],
+    def _generate_select_output(self, outputs: list[OutputVar], builtin_vars: dict[ir.Var, ir.Value|str|int],
+                                sql_vars: dict[ir.Lookup, str], var_column: dict[Tuple[ir.Var, ir.Lookup], ir.Field],
                                 var_lookups: dict[ir.Var, OrderedSet[ir.Lookup]],
                                 var_to_construct: dict[ir.Var, ir.Construct]):
 
         reference = partial(self._var_reference, var_lookups, sql_vars, var_column)
         resolve_builtin_var = partial(self._resolve_builtin_var, builtin_vars)
 
-        def handle_lookup_var(var):
+        def handle_lookup_var(var, var_type, alias):
             lookup = var_lookups[var].some()
             relation = lookup.relation
             var_name = sql_vars[lookup]
             column_name = self._var_name(relation.id, var_column[var, lookup])
-            vars.append(sql.VarRef(var_name, column_name, alias))
+            vars.append(sql.VarRef(var_name, column_name, alias, var_type))
             if from_cdc_annotation in relation.annotations:
                 not_null_vars.add(f"{var_name}.{column_name}")
 
@@ -1509,6 +1772,11 @@ class ModelToSQL:
             for val in construct.values:
                 if val in builtin_vars:
                     val = resolve_builtin_var(val)
+                    if isinstance(val, str):
+                        # In case we parsed builtin into some expression, we may add it as an element.
+                        # For example, `TO_DATE('1990-1-1', 'Y-m-d')` or `(v1.value + 5)`.
+                        elements.append(f"{val}")
+                        continue
                 if isinstance(val, ir.Var):
                     if val in var_to_construct:
                         elements.append(handle_construct(var_to_construct[val]))
@@ -1520,41 +1788,40 @@ class ModelToSQL:
                         if from_cdc_annotation in lookup.relation.annotations:
                             not_null_vars.add(lookup_var)
                 else:
-                    elements.append(self._convert_value(val, True))
+                    elements.append(str(self._convert_value(val)))
             return f"hash({', '.join(elements)})"
 
         # finally, compute what the select will return
         vars = []
         not_null_vars = ordered_set()
         for output in outputs:
-            alias, var = output[0], output[1]
-            task = output[2] if len(output) > 2 else None
+            alias, var, var_type, task = output.alias, output.value, output.value_type, output.task
             if isinstance(var, ir.Var):
                 if var in var_lookups and not task:
-                    handle_lookup_var(var)
+                    handle_lookup_var(var, var_type, alias)
                 elif var in builtin_vars:
                     var_ref = resolve_builtin_var(var)
                     if var_ref in var_lookups:
                         # Case: result of `cast` variable
-                        handle_lookup_var(var_ref)
+                        handle_lookup_var(var_ref, var_type, alias)
                     elif isinstance(var_ref, ir.Literal):
                         # Case: literal value from `cast` relation, e.g. `decimal(0)`
-                        vars.append(sql.VarRef(str(self._convert_value(var_ref.value)), alias=alias))
+                        vars.append(sql.VarRef(str(self._convert_value(var_ref.value)), alias=alias, type=var_type))
                     else:
                         # Example: We may have `decimal(0)` in QB which turns in IR into:
                         #   (cast(Decimal128, 0, vDecimal128) and decimal128(vDecimal128, res_3))
                         #   and we need to make it `0` in SQL.
                         var_ref = var_ref.name if isinstance(var_ref, ir.Var) else str(var_ref)
-                        vars.append(sql.VarRef(var_ref, alias=alias))
+                        vars.append(sql.VarRef(var_ref, alias=alias, type=var_type))
                 elif task:
                     if isinstance(task, ir.Construct):
                         # Generate constructions like hash(`x`, `y`, TABLE_ALIAS.COLUMN_NAME) as `alias`
-                        vars.append(sql.VarRef(handle_construct(task), alias=alias))
+                        vars.append(sql.VarRef(handle_construct(task), alias=alias, type=var_type))
                     elif isinstance(task, ir.Aggregate):
                         result_arg = task.projection[-1] if task.aggregation == builtins.count else task.args[0]
                         result_arg = resolve_builtin_var(result_arg)
                         ref = reference(result_arg) if isinstance(result_arg, ir.Var) else str(result_arg)
-                        vars.append(sql.VarRef(str(ref), alias=alias))
+                        vars.append(sql.VarRef(str(ref), alias=alias, type=var_type))
                     elif isinstance(task, ir.Union):
                         # Handle `COALESCE` of all lookups of this var from the union
                         lookups = self._extract_all_lookups_from_union(task)
@@ -1566,10 +1833,10 @@ class ModelToSQL:
                                 elements.append(f"{sql_vars[lu]}.{column_name}")
 
                         expr = "COALESCE(" + ", ".join(elements) + ")"
-                        vars.append(sql.VarRef(expr, alias=alias))
+                        vars.append(sql.VarRef(expr, alias=alias, type=var_type))
             else:
                 # TODO - abusing even more here, because var is a value!
-                vars.append(sql.VarRef(str(self._convert_value(var)), alias=alias))
+                vars.append(sql.VarRef(str(self._convert_value(var)), alias=alias, type=var_type))
         return not_null_vars, vars
 
     def _generate_select_nots(self, nots: Optional[list[ir.Not]], var_lookups: dict[ir.Var, OrderedSet[ir.Lookup]],
@@ -1615,8 +1882,11 @@ class ModelToSQL:
 
                 var_to_construct = {c.id_var: c for c in constructs} if constructs else {}
                 builtin_lookups = OrderedSet.from_iterable(t for t in all_lookups if builtins.is_builtin(t.relation))
-                builtin_vars, builtin_wheres = self._resolve_builtins(builtin_lookups, all_var_lookups, all_var_column,
-                                                                      all_sql_vars, var_to_construct)
+                builtin_vars, builtin_wheres, builtin_table_expressions = (
+                    self._resolve_builtins(builtin_lookups, all_var_lookups, all_var_column, all_sql_vars, var_to_construct))
+
+                froms.extend(self._process_builtin_table_expressions(builtin_table_expressions))
+
                 wheres.extend(builtin_wheres)
 
                 construct_wheres = self._process_constructs(lookups, var_lookups, var_column, sql_vars, builtin_vars,
@@ -1728,12 +1998,24 @@ class ModelToSQL:
         else:
             return str(resolved) if isinstance(var, ir.Var) or not quote_strings else f"'{resolved}'"
 
-    def _get_alias(self, key, arg, var_to_construct, var_to_union):
+    def _get_update_aliases(self, update: ir.Update, var_to_construct, var_to_union, skip_type:bool=False):
+        relation = update.relation
+        return [
+            self._get_alias(
+                self._var_name(relation.id, f),
+                arg,
+                self._convert_type(f.type) if not skip_type else None,
+                var_to_construct,
+                var_to_union,
+            )
+            for f, arg in zip(relation.fields, update.args)
+        ]
+
+    def _get_alias(self, key, arg, arg_type, var_to_construct, var_to_union):
         if not isinstance(arg, ir.Var):
-            return key, arg
+            return OutputVar(arg, key, arg_type)
 
-        var_task = var_to_construct.get(arg) or var_to_union.get(arg)
-        return (key, arg, var_task) if var_task else (key, arg)
+        return OutputVar(arg, key, arg_type, var_to_construct.get(arg) or var_to_union.get(arg))
 
     def _get_tuples(self, logical: ir.Logical, u: ir.Update):
         """
@@ -1763,12 +2045,12 @@ class ModelToSQL:
                     if isinstance(val, ir.Var):
                         inner_construct = find_construct(val)
                         if inner_construct:
-                            nested = [str(self._convert_value(x, True)) for x in inner_construct.values]
+                            nested = [str(self._convert_value(x)) for x in inner_construct.values]
                             resolved.append(f"hash({', '.join(nested)})")
                         else:
-                            resolved.append(self._convert_value(val, True))
+                            resolved.append(str(self._convert_value(val)))
                     else:
-                        resolved.append(self._convert_value(val, True))
+                        resolved.append(str(self._convert_value(val)))
 
                 return f"hash({', '.join(resolved)})"
             elif isinstance(arg, FrozenOrderedSet):
@@ -1791,7 +2073,7 @@ class ModelToSQL:
                 tuples = [prev + [value] for prev in tuples]
         return [tuple(t) for t in tuples]
 
-    def _convert_value(self, v, quote_numbers:bool=False, quote_strings:bool=True) -> str|int:
+    def _convert_value(self, v, quote_strings:bool=True) -> str|int:
         """ Convert the literal value in v to a SQL value."""
         if isinstance(v, str):
             return f"'{v}'" if quote_strings else v
@@ -1804,7 +2086,7 @@ class ModelToSQL:
                 return f"cast('{v.value}' as date)"
             if v.type == types.DateTime:
                 return f"cast('{v.value}' as datetime)"
-            return self._convert_value(v.value, quote_numbers, quote_strings)
+            return self._convert_value(v.value, quote_strings)
         if isinstance(v, float):
             if math.isnan(v):
                 return "cast('NaN' as DOUBLE)"
@@ -1812,7 +2094,7 @@ class ModelToSQL:
                 return "cast('Infinity' as DOUBLE)"
             elif v == float("-inf"):
                 return "cast('-Infinity' as DOUBLE)"
-            return str(v) if not quote_numbers else f"'{v}'"
+            return str(v)
         if isinstance(v, datetime.datetime):
             return f"cast('{v}' as datetime)"
         if isinstance(v, datetime.date):
@@ -1820,7 +2102,7 @@ class ModelToSQL:
         if isinstance(v, bool):
             return str(v).lower()
         if isinstance(v, int):
-            return v if not quote_numbers else f"'{v}'"
+            return v
         return str(v)
 
     COMMON_CONVERSION = {
@@ -1861,21 +2143,27 @@ class ModelToSQL:
             return f"DECIMAL({base_type.precision},{base_type.scale})"
         raise Exception(f"Unknown built-in type: {t}")
 
-    def _get_relations(self, model: ir.Model) -> list[ir.Relation]:
+    def _get_relations(self, model: ir.Model) -> Tuple[list[ir.Relation], list[ir.Relation]]:
+        rw = ReadWriteVisitor()
+        model.accept(rw)
+
+        root = cast(ir.Logical, model.root)
+
+        # For query compilation exclude read-only tables because we do not need to declare `CREATE TABLE` statements
+        used_relations = rw.writes(root) if self._query_compilation else rw.writes(root) | rw.reads(root)
+
         # Filter only relations that require table creation
-        relations = [
-            r for r in model.relations
+        table_relations = [
+            r for r in used_relations
             if self._is_table_creation_required(r)
         ]
 
-        # Optionally exclude read-only tables
-        if self._query_compilation:
-            rw = ReadWriteVisitor()
-            model.accept(rw)
-            writable = rw.writes(cast(ir.Logical, model.root))
-            relations = [r for r in relations if r in writable]
+        used_builtins = [
+            r for r in rw.reads(root)
+            if builtins.is_builtin(r)
+        ]
 
-        return relations
+        return table_relations, used_builtins
 
     def _is_table_creation_required(self, r: ir.Relation) -> bool:
         """
@@ -1932,9 +2220,26 @@ class ModelToSQL:
             if helpers.is_external(r):
                 self.relation_name_cache.get_name(r.id, r.name)
 
+    def _get_relation_info(self, relation: ir.Relation) -> RelationInfo:
+        if relation not in self.relation_infos:
+            self.relation_infos[relation] = RelationInfo()
+        return self.relation_infos[relation]
+
+    def mark_used(self, relation: ir.Relation):
+        self._get_relation_info(relation).used = True
+
+    def add_table_select(self, relation: ir.Relation, select: sql.Select):
+        self._get_relation_info(relation).table_selects.append(select)
+
+    def add_view_select(self, relation: ir.Relation, select: sql.Select):
+        self._get_relation_info(relation).view_selects.append(select)
+
+    def add_dynamic_table_select(self, relation: ir.Relation, select: sql.Select):
+        self._get_relation_info(relation).dynamic_table_selects.append(select)
+
     def _var_name(self, relation_id: int, arg: Union[ir.Var, ir.Field]):
         name = helpers.sanitize(self.relation_arg_name_cache.get_name((relation_id, arg.id), arg.name))
-        return f'"{name}"' if name.lower() in ("any", "order") else name
+        return f'"{name}"' if name.lower() in {"any", "order"} else name
 
     def _register_relation_args(self, relations: list[ir.Relation]):
         """
@@ -1960,11 +2265,13 @@ class ModelToSQL:
     def _build_builtin_lookups_dependencies(lookups: list[ir.Lookup]) -> list[Tuple[ir.Lookup, ir.Lookup]]:
         """
         Builds dependency edges for topological_sort:
-        1. A lookup whose last argument is used non-terminally in another must come first.
-        2. Terminal comparisons (neq, gt, lt, gte, lte) come last.
-        3. Conditionals (starts_with, contains, etc.) come after basic lookups but before terminals.
-        4. eq with only constants comes first.
-        5. eq with two vars must wait until one of them is grounded.
+        1. Terminal comparisons (neq, gt, lt, gte, lte) come last.
+        2. Conditionals (starts_with, contains, etc.) come after basic lookups but before terminals.
+        3. eq with only constants comes first.
+        4. eq with two vars must wait until one of them is grounded.
+        5. A lookup whose last argument is used non-terminally in another must come first.
+        6. For builtins that take multiple input arguments (like range, concat, substring, etc.),
+            ensure that all non-terminal arguments are processed before the builtin that consumes them.
         """
 
         edges = []
@@ -1988,7 +2295,7 @@ class ModelToSQL:
             relation_name = lookup.relation.name
             args = lookup.args
 
-            # Rule 2: Terminal relations depend on everything else
+            # Rule 1: Terminal relations depend on everything else
             if relation_name in terminal_relations:
                 for other in lookups:
                     other_name = other.relation.name
@@ -1996,7 +2303,7 @@ class ModelToSQL:
                         edges.append((other, lookup))
                 continue  # skip rest of rules for terminal lookups
 
-            # Rule 3: Conditional relations go before terminals, but after others
+            # Rule 2: Conditional relations go before terminals, but after others
             if relation_name in conditional_relations:
                 for other in lookups:
                     if other is not lookup:
@@ -2008,31 +2315,46 @@ class ModelToSQL:
             if relation_name == builtins.eq.name:
                 var_args = [arg for arg in args if isinstance(arg, ir.Var)]
 
-                # Rule 4: eq with only constants comes first
+                # Rule 3: eq with only constants comes first
                 if len(var_args) == 1:
                     # This lookup defines a var — should come before any that use this var non-terminally
                     grounded_var = var_args[0]
-                    for other_lookup, pos in arg_usages[grounded_var]:
-                        if other_lookup is not lookup:
-                            if pos != len(other_lookup.args) - 1:
-                                edges.append((lookup, other_lookup))
+                    for other, pos in arg_usages[grounded_var]:
+                        if other is not lookup:
+                            if pos != len(other.args) - 1:
+                                edges.append((lookup, other))
                     continue  # skip adding other edges among terminal assignments like a=2, b=2
 
-                # Rule 5: eq with two vars must wait until one of them is grounded
+                # Rule 4: eq with two vars must wait until one of them is grounded
                 elif len(var_args) == 2:
                     # eq(x, y): both are vars — lookup must come after those grounding either var
                     for var in var_args:
-                        for other_lookup, pos in arg_usages[var]:
-                            if other_lookup is not lookup:
-                                if other_lookup.args[-1] == var:
-                                    edges.append((other_lookup, lookup))
+                        for other, pos in arg_usages[var]:
+                            if other is not lookup:
+                                if other.args[-1] == var:
+                                    edges.append((other, lookup))
                     continue
 
-            # Rule 1: last argument used elsewhere non-terminally — must go first
-            last_arg = args[-1]
-            for other_lookup, pos in arg_usages.get(last_arg, []):
-                if other_lookup is not lookup and pos != len(other_lookup.args) - 1:
-                    edges.append((lookup, other_lookup))
+            # In generate builtins has a single output var but `split` returns `index` and `part`
+            num_outputs = 2 if lookup.relation == builtins.split else 1
+
+            # Rule 5: last output args must go first if used elsewhere non-terminally
+            for out_arg in args[-num_outputs:]:
+                for other, pos in arg_usages.get(out_arg, []):
+                    if other is not lookup and pos != len(other.args) - 1:
+                        edges.append((lookup, other))
+
+            # Rule 6: builtins with multiple input args must wait until all input args are grounded,
+            #   for example, range(start, end, step, result)
+            if len(args) > num_outputs:
+                for input_arg in args[:-num_outputs]:
+                    for other, pos in arg_usages.get(input_arg, []):
+                        if other is not lookup:
+                            other_name = other.relation.name
+                            if other_name not in terminal_relations and other_name not in conditional_relations:
+                                # Ensure any lookup that defines this arg (as last) comes before
+                                if other.args[-1] == input_arg:
+                                    edges.append((other, lookup))
 
         return edges
 
@@ -2064,8 +2386,9 @@ class ModelToSQL:
             3. Other statements except SELECT queries
             4. SELECT queries
         """
+        udfs = []
         create_tables = []
-        need_sort: dict[str, list[Union[sql.Insert, sql.CreateView]]] = defaultdict(list)
+        need_sort: dict[str, list[Union[sql.Insert, sql.CreateView, sql.CreateDynamicTable]]] = defaultdict(list)
         updates = []
         miscellaneous_statements = []
         selects = []
@@ -2077,19 +2400,23 @@ class ModelToSQL:
                 need_sort[statement.table].append(statement)
             elif isinstance(statement, sql.CreateView):
                 need_sort[statement.name].append(statement)
+            elif isinstance(statement, sql.CreateDynamicTable):
+                need_sort[statement.name].append(statement)
             elif isinstance(statement, sql.Update):
                 updates.append(statement)
             elif isinstance(statement, sql.Select):
                 selects.append(statement)
+            elif isinstance(statement, sql.CreateFunction):
+                udfs.append(statement)
             else:
                 miscellaneous_statements.append(statement)
 
         sorted_statements = self._sort_statements_dependency_graph(need_sort)
 
-        return create_tables + sorted_statements + updates + miscellaneous_statements + selects
+        return udfs + create_tables + sorted_statements + updates + miscellaneous_statements + selects
 
     @staticmethod
-    def _sort_statements_dependency_graph(statements: dict[str, list[Union[sql.Insert, sql.CreateView]]]) -> list[sql.Insert]:
+    def _sort_statements_dependency_graph(statements: dict[str, list[Union[sql.Insert, sql.CreateView, sql.CreateDynamicTable]]]) -> list[sql.Insert]:
         """ Topologic sort INSERT and CREATE VIEW statements based on dependencies in their SELECT FROM clauses. """
         edges = ordered_set()
         nodes = OrderedSet.from_iterable(statements.keys())
@@ -2124,16 +2451,17 @@ class ModelToSQL:
                     for sub_expr in expr.expr:
                         _extract_from_expr(sub_expr)
 
-            if select.where:
+            if select.where and select.where.expression:
                 _extract_from_expr(select.where.expression)
 
         for target_table, table_statements in statements.items():
             for statement in table_statements:
-                if isinstance(statement, sql.Insert):
-                    extract_dependencies(statement.select, target_table)
-                elif isinstance(statement, sql.CreateView):
+                if statement.query:
                     query = statement.query
-                    if isinstance(query, sql.Select):
+                    if isinstance(query, list):
+                        for sub_query in query:
+                            extract_dependencies(sub_query, target_table)
+                    elif isinstance(query, sql.Select):
                         extract_dependencies(query, target_table)
                     elif isinstance(query, sql.CTE):
                         for select in query.selects:
@@ -2158,3 +2486,13 @@ class RecursiveLookupsRewriter(v.Rewriter):
         if node.relation == self._recursive_relation:
             return node.reconstruct(node.engine, self._new_recursive_relation, node.args, node.annotations)
         return node
+
+class DerivedRelationsVisitor(v.Visitor):
+    _is_derived: bool = True
+
+    def is_derived(self) -> bool:
+        return self._is_derived
+
+    def visit_relation(self, node: ir.Relation, parent: Optional[ir.Node]):
+        if self._is_derived and from_cdc_annotation in node.annotations:
+            self._is_derived = False