squirrels 0.5.0rc0__py3-none-any.whl → 0.5.1__py3-none-any.whl

squirrels/_project.py

@@ -1,92 +1,113 @@
-from dotenv import dotenv_values
-from uuid import uuid4
+from dotenv import dotenv_values, load_dotenv
+from pathlib import Path
 import asyncio, typing as t, functools as ft, shutil, json, os
-import logging as l, matplotlib.pyplot as plt, networkx as nx, polars as pl
-import sqlglot, sqlglot.expressions
+import sqlglot, sqlglot.expressions, duckdb, polars as pl
 
-from ._auth import Authenticator, BaseUser
+from ._auth import Authenticator, AuthProviderArgs, ProviderFunctionType
+from ._schemas.auth_models import CustomUserFields, AbstractUser, GuestUser, RegisteredUser
+from ._schemas import response_models as rm
 from ._model_builder import ModelBuilder
 from ._exceptions import InvalidInputError, ConfigurationError
-from . import _utils as u, _constants as c, _manifest as mf, _connection_set as cs, _api_response_models as arm
+from ._py_module import PyModule
+from . import _dashboards as d, _utils as u, _constants as c, _manifest as mf, _connection_set as cs
 from . import _seeds as s, _models as m, _model_configs as mc, _model_queries as mq, _sources as so
-from . import _parameter_sets as ps, _dashboards_io as d, dashboards as dash, dataset_result as dr
+from . import _parameter_sets as ps, _dataset_types as dr, _logging as l
 
-T = t.TypeVar("T", bound=dash.Dashboard)
+T = t.TypeVar("T", bound=d.Dashboard)
 M = t.TypeVar("M", bound=m.DataModel)
 
 
-class _CustomJsonFormatter(l.Formatter):
-    def format(self, record: l.LogRecord) -> str:
-        super().format(record)
-        info = {
-            "timestamp": self.formatTime(record),
-            "project_id": record.name,
-            "level": record.levelname,
-            "message": record.getMessage(),
-            "thread": record.thread,
-            "thread_name": record.threadName,
-            "process": record.process,
-            **record.__dict__.get("info", {})
-        }
-        output = {
-            "data": record.__dict__.get("data", {}),
-            "info": info
-        }
-        return json.dumps(output)
-
-
 class SquirrelsProject:
     """
     Initiate an instance of this class to interact with a Squirrels project through Python code. For example this can be handy to experiment with the datasets produced by Squirrels in a Jupyter notebook.
     """
     
-    def __init__(self, *, filepath: str = ".", log_file: str | None = c.LOGS_FILE, log_level: str = "INFO", log_format: str = "text") -> None:
+    def __init__(
+        self, *, filepath: str = ".", load_dotenv_globally: bool = False,
+        log_to_file: bool = False, log_level: str | None = None, log_format: str | None = None,
+    ) -> None:
         """
         Constructor for SquirrelsProject class. Loads the file contents of the Squirrels project into memory as member fields.
 
         Arguments:
             filepath: The path to the Squirrels project file. Defaults to the current working directory.
-            log_level: The logging level to use. Options are "DEBUG", "INFO", and "WARNING". Default is "INFO".
-            log_file: The name of the log file to write to from the "logs/" subfolder. If None or empty string, then file logging is disabled. Default is "squirrels.log".
-            log_format: The format of the log records. Options are "text" and "json". Default is "text".
+            log_level: The logging level to use. Options are "DEBUG", "INFO", and "WARNING". Default is from SQRL_LOGGING__LOG_LEVEL environment variable or "INFO".
+            log_to_file: Whether to enable logging to file(s) in the "logs/" folder with rotation and retention policies. Default is False.
+            log_format: The format of the log records. Options are "text" and "json". Default is from SQRL_LOGGING__LOG_FORMAT environment variable or "text".
         """
         self._filepath = filepath
-        self._logger = self._get_logger(self._filepath, log_file, log_level, log_format)
-
-    def _get_logger(self, base_path: str, log_file: str | None, log_level: str, log_format: str) -> u.Logger:
-        logger = u.Logger(name=uuid4().hex)
-        logger.setLevel(log_level.upper())
-
-        handler = l.StreamHandler()
-        handler.setLevel("WARNING")
-        handler.setFormatter(l.Formatter("%(levelname)s:   %(asctime)s - %(message)s"))
-        logger.addHandler(handler)
-        
-        if log_format.lower() == "json":
-            formatter = _CustomJsonFormatter()
-        elif log_format.lower() == "text":
-            formatter = l.Formatter("[%(name)s] %(asctime)s - %(levelname)s - %(message)s")
-        else:
-            raise ValueError("log_format must be either 'text' or 'json'")
+        self._load_dotenv_globally = load_dotenv_globally
+        self._logger = self._get_logger(filepath, log_to_file, log_level, log_format)
+        self._ensure_virtual_datalake_exists(filepath)
+    
+    def _get_logger(self, filepath: str, log_to_file: bool, log_level: str | None, log_format: str | None) -> u.Logger:
+        env_vars = self._env_vars
+        # CLI arguments take precedence over environment variables
+        log_level = log_level if log_level is not None else env_vars.get(c.SQRL_LOGGING_LOG_LEVEL, "INFO")
+        log_format = log_format if log_format is not None else env_vars.get(c.SQRL_LOGGING_LOG_FORMAT, "text")
+        log_to_file = log_to_file or u.to_bool(env_vars.get(c.SQRL_LOGGING_LOG_TO_FILE, "false"))
+        log_file_size_mb = int(env_vars.get(c.SQRL_LOGGING_LOG_FILE_SIZE_MB, 50))
+        log_file_backup_count = int(env_vars.get(c.SQRL_LOGGING_LOG_FILE_BACKUP_COUNT, 1))
+        return l.get_logger(filepath, log_to_file, log_level, log_format, log_file_size_mb, log_file_backup_count)
+
+    def _ensure_virtual_datalake_exists(self, project_path: str) -> None:
+        target_path = u.Path(project_path, c.TARGET_FOLDER)
+        target_path.mkdir(parents=True, exist_ok=True)
+
+        # Attempt to set up the virtual data lake with DATA_PATH if possible
+        try:
+            is_ducklake = self._datalake_db_path.startswith("ducklake:")
             
-        if log_file:
-            path = u.Path(base_path, c.LOGS_FOLDER, log_file)
-            path.parent.mkdir(parents=True, exist_ok=True)
-
-            handler = l.FileHandler(path)
-            handler.setFormatter(formatter)
-            logger.addHandler(handler)
+            data_path = self._env_vars.get(c.SQRL_VDL_DATA_PATH, c.DEFAULT_VDL_DATA_PATH)
+            data_path = data_path.format(project_path=project_path)
+            
+            options = f"(DATA_PATH '{data_path}')" if is_ducklake else ""
+            attach_stmt = f"ATTACH '{self._datalake_db_path}' AS vdl {options}"
+            with duckdb.connect() as conn:
+                conn.execute(attach_stmt)
+                # TODO: support incremental loads for build models and avoid cleaning up old files all the time
+                conn.execute("CALL ducklake_expire_snapshots('vdl', older_than => now())")
+                conn.execute("CALL ducklake_cleanup_old_files('vdl', cleanup_all => true)")
         
-        return logger
+        except Exception as e:
+            if "DATA_PATH parameter" in str(e):
+                first_line = str(e).split("\n")[0]
+                note = "NOTE: Squirrels does not allow changing the data path for an existing Virtual Data Lake (VDL)"
+                raise u.ConfigurationError(f"{first_line}\n\n{note}")
+            
+            if is_ducklake and not any(x in self._datalake_db_path for x in [":sqlite:", ":postgres:", ":mysql:"]):
+                extended_error = "\n  Note: if you're using DuckDB for the metadata database, only one process can connect to the VDL at a time."
+            else:
+                extended_error = ""
+            
+            raise u.ConfigurationError(f"Failed to attach Virtual Data Lake (VDL).{extended_error}") from e
     
     @ft.cached_property
     def _env_vars(self) -> dict[str, str]:
         dotenv_files = [c.DOTENV_FILE, c.DOTENV_LOCAL_FILE]
         dotenv_vars = {}
         for file in dotenv_files:
-            dotenv_vars.update({k: v for k, v in dotenv_values(f"{self._filepath}/{file}").items() if v is not None})
+            full_path = u.Path(self._filepath, file)
+            if self._load_dotenv_globally:
+                load_dotenv(full_path)
+            dotenv_vars.update({k: v for k, v in dotenv_values(full_path).items() if v is not None})
         return {**os.environ, **dotenv_vars}
 
+    @ft.cached_property
+    def _elevated_access_level(self) -> u.ACCESS_LEVEL:
+        elevated_access_level = self._env_vars.get(c.SQRL_PERMISSIONS_ELEVATED_ACCESS_LEVEL, "admin").lower()
+
+        if elevated_access_level not in ["admin", "member", "guest"]:
+            raise u.ConfigurationError(f"{c.SQRL_PERMISSIONS_ELEVATED_ACCESS_LEVEL} has been set to an invalid access level: {elevated_access_level}")
+        
+        return elevated_access_level
+    
+    @ft.cached_property
+    def _datalake_db_path(self) -> str:
+        datalake_db_path = self._env_vars.get(c.SQRL_VDL_CATALOG_DB_PATH, c.DEFAULT_VDL_CATALOG_DB_PATH)
+        datalake_db_path = datalake_db_path.format(project_path=self._filepath)
+        return datalake_db_path
+
     @ft.cached_property
     def _manifest_cfg(self) -> mf.ManifestConfig:
         return mf.ManifestIO.load_from_file(self._logger, self._filepath, self._env_vars)
@@ -127,29 +148,77 @@ class SquirrelsProject:
     def _conn_set(self) -> cs.ConnectionSet:
         return cs.ConnectionSetIO.load_from_file(self._logger, self._filepath, self._manifest_cfg, self._conn_args)
     
+    @ft.cached_property
+    def _custom_user_fields_cls_and_provider_functions(self) -> tuple[type[CustomUserFields], list[ProviderFunctionType]]:
+        user_module_path = u.Path(self._filepath, c.PYCONFIGS_FOLDER, c.USER_FILE)
+        user_module = PyModule(user_module_path)
+        
+        # Load CustomUserFields class (adds to Authenticator.providers as side effect)
+        CustomUserFieldsCls = user_module.get_func_or_class("CustomUserFields", default_attr=CustomUserFields)
+        provider_functions = Authenticator.providers
+        Authenticator.providers = []
+        
+        if not issubclass(CustomUserFieldsCls, CustomUserFields):
+            raise ConfigurationError(f"CustomUserFields class in '{c.USER_FILE}' must inherit from CustomUserFields")
+        
+        return CustomUserFieldsCls, provider_functions
+    
+    @ft.cached_property
+    def _auth_args(self) -> AuthProviderArgs:
+        conn_args = self._conn_args
+        return AuthProviderArgs(conn_args.project_path, conn_args.proj_vars, conn_args.env_vars)
+    
     @ft.cached_property
     def _auth(self) -> Authenticator:
-        return Authenticator(self._logger, self._filepath, self._env_vars)
+        CustomUserFieldsCls, provider_functions = self._custom_user_fields_cls_and_provider_functions
+        external_only = (self._manifest_cfg.authentication.type == mf.AuthenticationType.EXTERNAL)
+        return Authenticator(self._logger, self._filepath, self._auth_args, provider_functions, custom_user_fields_cls=CustomUserFieldsCls, external_only=external_only)
+    
+    @ft.cached_property
+    def _guest_user(self) -> AbstractUser:
+        custom_fields = self._auth.CustomUserFields()
+        return GuestUser(username="", custom_fields=custom_fields)
+
+    @ft.cached_property
+    def _admin_user(self) -> AbstractUser:
+        custom_fields = self._auth.CustomUserFields()
+        return RegisteredUser(username="", access_level="admin", custom_fields=custom_fields)
     
     @ft.cached_property
     def _param_args(self) -> ps.ParametersArgs:
-        return ps.ParameterConfigsSetIO.get_param_args(self._conn_args)
+        conn_args = self._conn_args
+        return ps.ParametersArgs(conn_args.project_path, conn_args.proj_vars, conn_args.env_vars)
     
     @ft.cached_property
     def _param_cfg_set(self) -> ps.ParameterConfigsSet:
         return ps.ParameterConfigsSetIO.load_from_file(
-            self._logger, self._filepath, self._manifest_cfg, self._seeds, self._conn_set, self._param_args
+            self._logger, self._filepath, self._manifest_cfg, self._seeds, self._conn_set, self._param_args, self._datalake_db_path
         )
     
     @ft.cached_property
     def _j2_env(self) -> u.EnvironmentWithMacros:
-        return u.EnvironmentWithMacros(self._logger, loader=u.j2.FileSystemLoader(self._filepath))
+        env = u.EnvironmentWithMacros(self._logger, loader=u.j2.FileSystemLoader(self._filepath))
+
+        def value_to_str(value: t.Any, attribute: str | None = None) -> str:
+            if attribute is None:
+                return str(value)
+            else:
+                return str(getattr(value, attribute))
+
+        def join(value: list[t.Any], d: str = ", ", attribute: str | None = None) -> str:
+            return d.join(map(lambda x: value_to_str(x, attribute), value))
+
+        def quote(value: t.Any, q: str = "'", attribute: str | None = None) -> str:
+            return q + value_to_str(value, attribute) + q
+        
+        def quote_and_join(value: list[t.Any], q: str = "'", d: str = ", ", attribute: str | None = None) -> str:
+            return d.join(map(lambda x: quote(x, q, attribute), value))
+        
+        env.filters["join"] = join
+        env.filters["quote"] = quote
+        env.filters["quote_and_join"] = quote_and_join
+        return env
 
-    @ft.cached_property
-    def _duckdb_venv_path(self) -> str:
-        duckdb_filepath_setting_val = self._env_vars.get(c.SQRL_DUCKDB_VENV_DB_FILE_PATH, f"{c.TARGET_FOLDER}/{c.DUCKDB_VENV_FILE}")
-        return str(u.Path(self._filepath, duckdb_filepath_setting_val))
-    
     def close(self) -> None:
         """
         Deliberately close any open resources within the Squirrels project, such as database connections (instead of relying on the garbage collector).
@@ -184,20 +253,20 @@ class SquirrelsProject:
         return models_dict
 
 
-    async def build(self, *, full_refresh: bool = False, select: str | None = None, stage_file: bool = False) -> None:
+    async def build(self, *, full_refresh: bool = False, select: str | None = None) -> None:
         """
-        Build the virtual data environment for the Squirrels project
+        Build the Virtual Data Lake (VDL) for the Squirrels project
 
         Arguments:
-            full_refresh: Whether to drop all tables and rebuild the virtual data environment from scratch. Default is False.
-            stage_file: Whether to stage the DuckDB file to overwrite the existing one later if the virtual data environment is in use. Default is False.
+            full_refresh: Whether to drop all tables and rebuild the VDL from scratch. Default is False.
+            select: The name of a specific model to build. If None, all models are built. Default is None.
         """
         models_dict: dict[str, m.StaticModel] = self._get_static_models()
-        builder = ModelBuilder(self._duckdb_venv_path, self._conn_set, models_dict, self._conn_args, self._logger)
-        await builder.build(full_refresh, select, stage_file)
+        builder = ModelBuilder(self._datalake_db_path, self._conn_set, models_dict, self._conn_args, self._logger)
+        await builder.build(full_refresh, select)
 
     def _get_models_dict(self, always_python_df: bool) -> dict[str, m.DataModel]:
-        models_dict: dict[str, m.DataModel] = dict(self._get_static_models())
+        models_dict: dict[str, m.DataModel] = self._get_static_models()
         
         for name, val in self._dbview_model_files.items():
             self._add_model(models_dict, m.DbviewModel(
@@ -213,19 +282,18 @@ class SquirrelsProject:
         
         return models_dict
     
-    def _generate_dag(self, dataset: str, *, target_model_name: str | None = None, always_python_df: bool = False) -> m.DAG:
-        models_dict = self._get_models_dict(always_python_df)
+    def _generate_dag(self, dataset: str) -> m.DAG:
+        models_dict = self._get_models_dict(always_python_df=False)
         
         dataset_config = self._manifest_cfg.datasets[dataset]
-        target_model_name = dataset_config.model if target_model_name is None else target_model_name
-        target_model = models_dict[target_model_name]
+        target_model = models_dict[dataset_config.model]
         target_model.is_target = True
-        dag = m.DAG(dataset_config, target_model, models_dict, self._duckdb_venv_path, self._logger)
+        dag = m.DAG(dataset_config, target_model, models_dict, self._datalake_db_path, self._logger)
         
         return dag
     
-    def _generate_dag_with_fake_target(self, sql_query: str | None) -> m.DAG:
-        models_dict = self._get_models_dict(always_python_df=False)
+    def _generate_dag_with_fake_target(self, sql_query: str | None, *, always_python_df: bool = False) -> m.DAG:
+        models_dict = self._get_models_dict(always_python_df=always_python_df)
 
         if sql_query is None:
             dependencies = set(models_dict.keys())
@@ -235,227 +303,260 @@ class SquirrelsProject:
             substitutions = {}
             for model_name in dependencies:
                 model = models_dict[model_name]
-                if isinstance(model, m.SourceModel) and not model.model_config.load_to_duckdb:
-                    raise InvalidInputError(203, f"Source model '{model_name}' cannot be queried with DuckDB")
-                if isinstance(model, (m.SourceModel, m.BuildModel)):
-                    substitutions[model_name] = f"venv.{model_name}"
+                if isinstance(model, m.SourceModel) and not model.is_queryable:
+                    raise InvalidInputError(400, "cannot_query_source_model", f"Source model '{model_name}' cannot be queried with DuckDB")
+                if isinstance(model, m.BuildModel):
+                    substitutions[model_name] = f"vdl.{model_name}"
+                elif isinstance(model, m.SourceModel):
+                    if model.model_config.load_to_vdl:
+                        substitutions[model_name] = f"vdl.{model_name}"
+                    else:
+                        # DuckDB connection without load_to_vdl - reference via attached database
+                        conn_name = model.model_config.get_connection()
+                        table_name = model.model_config.get_table()
+                        substitutions[model_name] = f"db_{conn_name}.{table_name}"
             
             sql_query = parsed.transform(
-                lambda node: sqlglot.expressions.Table(this=substitutions[node.name])
+                lambda node: sqlglot.expressions.Table(this=substitutions[node.name], alias=node.alias)
                 if isinstance(node, sqlglot.expressions.Table) and node.name in substitutions
                 else node
             ).sql()
         
         model_config = mc.FederateModelConfig(depends_on=dependencies)
-        query_file = mq.SqlQueryFile("", sql_query or "")
+        query_file = mq.SqlQueryFile("", sql_query or "SELECT 1")
         fake_target_model = m.FederateModel(
             "__fake_target", model_config, query_file, logger=self._logger, env_vars=self._env_vars, conn_set=self._conn_set, j2_env=self._j2_env
         )
         fake_target_model.is_target = True
-        dag = m.DAG(None, fake_target_model, models_dict, self._duckdb_venv_path, self._logger)
+        dag = m.DAG(None, fake_target_model, models_dict, self._datalake_db_path, self._logger)
         return dag
     
-    def _draw_dag(self, dag: m.DAG, output_folder: u.Path) -> None:
-        color_map = {
-            m.ModelType.SEED: "green", m.ModelType.DBVIEW: "red", m.ModelType.FEDERATE: "skyblue",
-            m.ModelType.BUILD: "purple", m.ModelType.SOURCE: "orange"
-        }
-
-        G = dag.to_networkx_graph()
-        
-        fig, _ = plt.subplots()
-        pos = nx.multipartite_layout(G, subset_key="layer")
-        colors = [color_map[node[1]] for node in G.nodes(data="model_type")] # type: ignore
-        nx.draw(G, pos=pos, node_shape='^', node_size=1000, node_color=colors, arrowsize=20)
-        
-        y_values = [val[1] for val in pos.values()]
-        scale = max(y_values) - min(y_values) if len(y_values) > 0 else 0
-        label_pos = {key: (val[0], val[1]-0.002-0.1*scale) for key, val in pos.items()}
-        nx.draw_networkx_labels(G, pos=label_pos, font_size=8)
-        
-        fig.tight_layout()
-        plt.margins(x=0.1, y=0.1)
-        fig.savefig(u.Path(output_folder, "dag.png"))
-        plt.close(fig)
-    
-    async def _get_compiled_dag(self, *, sql_query: str | None = None, selections: dict[str, t.Any] = {}, user: BaseUser | None = None) -> m.DAG:
-        dag = self._generate_dag_with_fake_target(sql_query)
+    async def _get_compiled_dag(
+        self, user: AbstractUser, *, sql_query: str | None = None, selections: dict[str, t.Any] = {}, configurables: dict[str, str] = {}, 
+        always_python_df: bool = False
+    ) -> m.DAG:
+        dag = self._generate_dag_with_fake_target(sql_query, always_python_df=always_python_df)
         
-        default_traits = self._manifest_cfg.get_default_traits()
-        await dag.execute(self._param_args, self._param_cfg_set, self._context_func, user, selections, runquery=False, default_traits=default_traits)
+        configurables = {**self._manifest_cfg.get_default_configurables(), **configurables}
+        await dag.execute(
+            self._param_args, self._param_cfg_set, self._context_func, user, selections,
+            runquery=False, configurables=configurables
+        )
         return dag
     
-    def _get_all_connections(self) -> list[arm.ConnectionItemModel]:
+    def _get_all_connections(self) -> list[rm.ConnectionItemModel]:
         connections = []
         for conn_name, conn_props in self._conn_set.get_connections_as_dict().items():
             if isinstance(conn_props, mf.ConnectionProperties):
                 label = conn_props.label if conn_props.label is not None else conn_name
-                connections.append(arm.ConnectionItemModel(name=conn_name, label=label))
+                connections.append(rm.ConnectionItemModel(name=conn_name, label=label))
         return connections
     
-    def _get_all_data_models(self, compiled_dag: m.DAG) -> list[arm.DataModelItem]:
+    def _get_all_data_models(self, compiled_dag: m.DAG) -> list[rm.DataModelItem]:
         return compiled_dag.get_all_data_models()
     
-    async def get_all_data_models(self) -> list[arm.DataModelItem]:
+    async def get_all_data_models(self) -> list[rm.DataModelItem]:
         """
         Get all data models in the project
 
         Returns:
             A list of DataModelItem objects
         """
-        compiled_dag = await self._get_compiled_dag()
+        compiled_dag = await self._get_compiled_dag(self._admin_user)
         return self._get_all_data_models(compiled_dag)
 
-    def _get_all_data_lineage(self, compiled_dag: m.DAG) -> list[arm.LineageRelation]:
+    def _get_all_data_lineage(self, compiled_dag: m.DAG) -> list[rm.LineageRelation]:
         all_lineage = compiled_dag.get_all_model_lineage()
 
         # Add dataset nodes to the lineage
         for dataset in self._manifest_cfg.datasets.values():
-            target_dataset = arm.LineageNode(name=dataset.name, type="dataset")
-            source_model = arm.LineageNode(name=dataset.model, type="model")
-            all_lineage.append(arm.LineageRelation(type="runtime", source=source_model, target=target_dataset))
+            target_dataset = rm.LineageNode(name=dataset.name, type="dataset")
+            source_model = rm.LineageNode(name=dataset.model, type="model")
+            all_lineage.append(rm.LineageRelation(type="runtime", source=source_model, target=target_dataset))
 
         # Add dashboard nodes to the lineage
         for dashboard in self._dashboards.values():
-            target_dashboard = arm.LineageNode(name=dashboard.dashboard_name, type="dashboard")
+            target_dashboard = rm.LineageNode(name=dashboard.dashboard_name, type="dashboard")
             datasets = set(x.dataset for x in dashboard.config.depends_on)
             for dataset in datasets:
-                source_dataset = arm.LineageNode(name=dataset, type="dataset")
-                all_lineage.append(arm.LineageRelation(type="runtime", source=source_dataset, target=target_dashboard))
+                source_dataset = rm.LineageNode(name=dataset, type="dataset")
+                all_lineage.append(rm.LineageRelation(type="runtime", source=source_dataset, target=target_dashboard))
 
         return all_lineage
 
-    async def get_all_data_lineage(self) -> list[arm.LineageRelation]:
+    async def get_all_data_lineage(self) -> list[rm.LineageRelation]:
         """
         Get all data lineage in the project
 
         Returns:
             A list of LineageRelation objects
         """
-        compiled_dag = await self._get_compiled_dag()
+        compiled_dag = await self._get_compiled_dag(self._admin_user)
         return self._get_all_data_lineage(compiled_dag)
 
-    async def _write_dataset_outputs_given_test_set(
-        self, dataset: str, select: str, test_set: str | None, runquery: bool, recurse: bool
-    ) -> t.Any | None:
-        dataset_conf = self._manifest_cfg.datasets[dataset]
-        default_test_set_conf = self._manifest_cfg.get_default_test_set(dataset)
-        if test_set in self._manifest_cfg.selection_test_sets:
-            test_set_conf = self._manifest_cfg.selection_test_sets[test_set]
-        elif test_set is None or test_set == default_test_set_conf.name:
-            test_set, test_set_conf = default_test_set_conf.name, default_test_set_conf
-        else:
-            raise ConfigurationError(f"No test set named '{test_set}' was found when compiling dataset '{dataset}'. The test set must be defined if not default for dataset.")
-        
-        error_msg_intro = f"Cannot compile dataset '{dataset}' with test set '{test_set}'."
-        if test_set_conf.datasets is not None and dataset not in test_set_conf.datasets:
-            raise ConfigurationError(f"{error_msg_intro}\n Applicable datasets for test set '{test_set}' does not include dataset '{dataset}'.")
-        
-        user_attributes = test_set_conf.user_attributes.copy() if test_set_conf.user_attributes is not None else {}
-        selections = test_set_conf.parameters.copy()
-        username, is_admin = user_attributes.pop("username", ""), user_attributes.pop("is_admin", False)
-        if test_set_conf.is_authenticated:
-            user = self._auth.User(username=username, is_admin=is_admin, **user_attributes)
-        elif dataset_conf.scope == mf.PermissionScope.PUBLIC:
-            user = None
-        else:
-            raise ConfigurationError(f"{error_msg_intro}\n Non-public datasets require a test set with 'user_attributes' section defined")
-        
-        if dataset_conf.scope == mf.PermissionScope.PRIVATE and not is_admin:
-            raise ConfigurationError(f"{error_msg_intro}\n Private datasets require a test set with user_attribute 'is_admin' set to true")
-
-        # always_python_df is set to True for creating CSV files from results (when runquery is True)
-        dag = self._generate_dag(dataset, target_model_name=select, always_python_df=runquery)
-        await dag.execute(
-            self._param_args, self._param_cfg_set, self._context_func, user, selections, 
-            runquery=runquery, recurse=recurse, default_traits=self._manifest_cfg.get_default_traits()
-        )
-        
-        output_folder = u.Path(self._filepath, c.TARGET_FOLDER, c.COMPILE_FOLDER, dataset, test_set)
-        if output_folder.exists():
-            shutil.rmtree(output_folder)
-        output_folder.mkdir(parents=True, exist_ok=True)
-        
-        def write_placeholders() -> None:
-            output_filepath = u.Path(output_folder, "placeholders.json")
-            with open(output_filepath, 'w') as f:
-                json.dump(dag.placeholders, f, indent=4)
-        
-        def write_model_outputs(model: m.DataModel) -> None:
-            assert isinstance(model, m.QueryModel)
-            subfolder = c.DBVIEWS_FOLDER if model.model_type == m.ModelType.DBVIEW else c.FEDERATES_FOLDER
-            subpath = u.Path(output_folder, subfolder)
-            subpath.mkdir(parents=True, exist_ok=True)
-            if isinstance(model.compiled_query, mq.SqlModelQuery):
-                output_filepath = u.Path(subpath, model.name+'.sql')
-                query = model.compiled_query.query
-                with open(output_filepath, 'w') as f:
-                    f.write(query)
-            if runquery and isinstance(model.result, pl.LazyFrame):
-                output_filepath = u.Path(subpath, model.name+'.csv')
-                model.result.collect().write_csv(output_filepath)
-
-        write_placeholders()
-        all_model_names = dag.get_all_query_models()
-        coroutines = [asyncio.to_thread(write_model_outputs, dag.models_dict[name]) for name in all_model_names]
-        await u.asyncio_gather(coroutines)
-
-        if recurse:
-            self._draw_dag(dag, output_folder)
-        
-        if isinstance(dag.target_model, m.QueryModel) and dag.target_model.compiled_query is not None:
-            return dag.target_model.compiled_query.query
-    
     async def compile(
-        self, *, dataset: str | None = None, do_all_datasets: bool = False, selected_model: str | None = None, test_set: str | None = None, 
-        do_all_test_sets: bool = False, runquery: bool = False
+        self, *, selected_model: str | None = None, test_set: str | None = None, do_all_test_sets: bool = False,
+        runquery: bool = False, clear: bool = False, buildtime_only: bool = False, runtime_only: bool = False
     ) -> None:
         """
-        Async method to compile the SQL templates into files in the "target/" folder. Same functionality as the "sqrl compile" CLI.
+        Compile models into the "target/compile" folder.
 
-        Although all arguments are "optional", the "dataset" argument is required if "do_all_datasets" argument is False.
+        Behavior:
+        - Buildtime outputs: target/compile/buildtime/*.sql (for SQL build models) and dag.png
+        - Runtime outputs: target/compile/runtime/[test_set]/dbviews/*.sql, federates/*.sql, dag.png
+          If runquery=True, also write CSVs for runtime models.
+        - Options: clear entire compile folder first; compile only buildtime or only runtime.
 
         Arguments:
-            dataset: The name of the dataset to compile. Ignored if "do_all_datasets" argument is True, but required (i.e., cannot be None) if "do_all_datasets" is False. Default is None.
-            do_all_datasets: If True, compile all datasets and ignore the "dataset" argument. Default is False.
             selected_model: The name of the model to compile. If specified, the compiled SQL query is also printed in the terminal. If None, all models for the selected dataset are compiled. Default is None.
             test_set: The name of the test set to compile with. If None, the default test set is used (which can vary by dataset). Ignored if `do_all_test_sets` argument is True. Default is None.
             do_all_test_sets: Whether to compile all applicable test sets for the selected dataset(s). If True, the `test_set` argument is ignored. Default is False.
-            runquery**: Whether to run all compiled queries and save each result as a CSV file. If True and `selected_model` is specified, all upstream models of the selected model is compiled as well. Default is False.
+            runquery: Whether to run all compiled queries and save each result as a CSV file. If True and `selected_model` is specified, all upstream models of the selected model is compiled as well. Default is False.
+            clear: Whether to clear the "target/compile/" folder before compiling. Default is False.
+            buildtime_only: Whether to compile only buildtime models. Default is False.
+            runtime_only: Whether to compile only runtime models. Default is False.
         """
-        recurse = True
-        if do_all_datasets:
-            selected_models = [(dataset.name, dataset.model) for dataset in self._manifest_cfg.datasets.values()]
-        else:
-            assert isinstance(dataset, str), "argument 'dataset' must be provided a string value if argument 'do_all_datasets' is False"
-            assert dataset in self._manifest_cfg.datasets, f"dataset '{dataset}' not found in {c.MANIFEST_FILE}"
-            if selected_model is None:
-                selected_model = self._manifest_cfg.datasets[dataset].model
-            else:
-                recurse = False
-            selected_models = [(dataset, selected_model)]
+        border = "=" * 80
+        underlines = "-" * len(border)
+
+        compile_root = Path(self._filepath, c.TARGET_FOLDER, c.COMPILE_FOLDER)
+        if clear and compile_root.exists():
+            shutil.rmtree(compile_root)
+
+        models_dict = self._get_models_dict(always_python_df=False)
+
+        if selected_model is not None:
+            selected_model = u.normalize_name(selected_model)
+            if selected_model not in models_dict:
+                print(f"No such model found: {selected_model}")
+                return
+            if not isinstance(models_dict[selected_model], m.QueryModel):
+                print(f"Model '{selected_model}' is not a query model. Nothing to do.")
+                return
         
-        coroutines: list[t.Coroutine] = []
-        for dataset, selected_model in selected_models:
-            if do_all_test_sets:
-                for test_set_name in self._manifest_cfg.get_applicable_test_sets(dataset):
-                    coroutine = self._write_dataset_outputs_given_test_set(dataset, selected_model, test_set_name, runquery, recurse)
-                    coroutines.append(coroutine)
+        model_to_compile = None
+
+        # Buildtime compilation
+        if not runtime_only:
+            print(underlines)
+            print(f"Compiling buildtime models")
+            print(underlines)
+
+            buildtime_folder = Path(compile_root, c.COMPILE_BUILDTIME_FOLDER)
+            buildtime_folder.mkdir(parents=True, exist_ok=True)
+
+            def write_buildtime_model(model: m.DataModel, static_models: dict[str, m.StaticModel]) -> None:
+                if not isinstance(model, m.BuildModel):
+                    return
+                
+                model.compile_for_build(self._conn_args, static_models)
+
+                if isinstance(model.compiled_query, mq.SqlModelQuery):
+                    out_path = Path(buildtime_folder, f"{model.name}.sql")
+                    with open(out_path, 'w') as f:
+                        f.write(model.compiled_query.query)
+                    print(f"Successfully compiled build model: {model.name}")
+                elif isinstance(model.compiled_query, mq.PyModelQuery):
+                    print(f"The build model '{model.name}' is in Python. Compilation for Python is not supported yet.")
+
+            static_models = self._get_static_models()
+            if selected_model is not None:
+                model_to_compile = models_dict[selected_model]
+                write_buildtime_model(model_to_compile, static_models)
+            else:
+                coros = [asyncio.to_thread(write_buildtime_model, m, static_models) for m in static_models.values()]
+                await u.asyncio_gather(coros)
             
-            coroutine = self._write_dataset_outputs_given_test_set(dataset, selected_model, test_set, runquery, recurse)
-            coroutines.append(coroutine)
-        
-        queries = await u.asyncio_gather(coroutines)
+            print(underlines)
+            print()
         
-        print(f"Compiled successfully! See the '{c.TARGET_FOLDER}/' folder for results.")
-        print()
-        if not recurse and len(queries) == 1 and isinstance(queries[0], str):
-            print(queries[0])
+        # Runtime compilation
+        if not buildtime_only:
+            if do_all_test_sets:
+                test_set_names_set = set(self._manifest_cfg.selection_test_sets.keys())
+                test_set_names_set.add(c.DEFAULT_TEST_SET_NAME)
+                test_set_names = list(test_set_names_set)
+            else:
+                test_set_names = [test_set or c.DEFAULT_TEST_SET_NAME]
+
+            for ts_name in test_set_names:
+                print(underlines)
+                print(f"Compiling runtime models (test set '{ts_name}')")
+                print(underlines)
+
+                # Build user and selections from test set config if present
+                ts_conf = self._manifest_cfg.selection_test_sets.get(ts_name, self._manifest_cfg.get_default_test_set())
+                # Separate base fields from custom fields
+                access_level = ts_conf.user.access_level
+                custom_fields = self._auth.CustomUserFields(**ts_conf.user.custom_fields)
+                if access_level == "guest":
+                    user = GuestUser(username="", custom_fields=custom_fields)
+                else:
+                    user = RegisteredUser(username="", access_level=access_level, custom_fields=custom_fields)
+
+                # Generate DAG across all models. When runquery=True, force models to produce Python dataframes so CSVs can be written.
+                dag = await self._get_compiled_dag(
+                    user=user, selections=ts_conf.parameters, configurables=ts_conf.configurables, always_python_df=runquery,
+                )
+                if runquery:
+                    await dag._run_models()
+
+                # Prepare output folders
+                runtime_folder = Path(compile_root, c.COMPILE_RUNTIME_FOLDER, ts_name)
+                dbviews_folder = Path(runtime_folder, c.DBVIEWS_FOLDER)
+                federates_folder = Path(runtime_folder, c.FEDERATES_FOLDER)
+                dbviews_folder.mkdir(parents=True, exist_ok=True)
+                federates_folder.mkdir(parents=True, exist_ok=True)
+                with open(Path(runtime_folder, "placeholders.json"), "w") as f:
+                    json.dump(dag.placeholders, f)
+
+                # Function to write runtime models
+                def write_runtime_model(model: m.DataModel) -> None:
+                    if not isinstance(model, m.QueryModel):
+                        return
+                    
+                    if model.model_type not in (m.ModelType.DBVIEW, m.ModelType.FEDERATE):
+                        return
+                    
+                    subfolder = dbviews_folder if model.model_type == m.ModelType.DBVIEW else federates_folder
+                    model_type = "dbview" if model.model_type == m.ModelType.DBVIEW else "federate"
+
+                    if isinstance(model.compiled_query, mq.SqlModelQuery):
+                        out_sql = Path(subfolder, f"{model.name}.sql")
+                        with open(out_sql, 'w') as f:
+                            f.write(model.compiled_query.query)
+                        print(f"Successfully compiled {model_type} model: {model.name}")
+                    elif isinstance(model.compiled_query, mq.PyModelQuery):
+                        print(f"The {model_type} model '{model.name}' is in Python. Compilation for Python is not supported yet.")
+                    
+                    if runquery and isinstance(model.result, pl.LazyFrame):
+                        out_csv = Path(subfolder, f"{model.name}.csv")
+                        model.result.collect().write_csv(out_csv)
+                        print(f"Successfully created CSV for {model_type} model: {model.name}")
+
+                # If selected_model is provided for runtime, only emit that model's outputs
+                if selected_model is not None:
+                    model_to_compile = dag.models_dict[selected_model]
+                    write_runtime_model(model_to_compile)
+                else:
+                    coros = [asyncio.to_thread(write_runtime_model, model) for model in dag.models_dict.values()]
+                    await u.asyncio_gather(coros)
+
+                print(underlines)
+                print()
+
+        print(f"All compilations complete! See the '{c.TARGET_FOLDER}/{c.COMPILE_FOLDER}/' folder for results.")
+        if model_to_compile and isinstance(model_to_compile, m.QueryModel) and isinstance(model_to_compile.compiled_query, mq.SqlModelQuery):
+            print()
+            print(border)
+            print(f"Compiled SQL query for model '{model_to_compile.name}':")
+            print(underlines)
+            print(model_to_compile.compiled_query.query)
+            print(border)
             print()
 
-    def _permission_error(self, user: BaseUser | None, data_type: str, data_name: str, scope: str) -> InvalidInputError:
-        username = "" if user is None else f" '{user.username}'"
-        return InvalidInputError(25, f"User{username} does not have permission to access {scope} {data_type}: {data_name}")
+    def _permission_error(self, user: AbstractUser, data_type: str, data_name: str, scope: str) -> InvalidInputError:
+        return InvalidInputError(403, f"unauthorized_access_to_{data_type}", f"User '{user}' does not have permission to access {scope} {data_type}: {data_name}")
     
     def seed(self, name: str) -> pl.LazyFrame:
         """
@@ -491,7 +592,8 @@ class SquirrelsProject:
         )
     
     async def dataset(
-        self, name: str, *, selections: dict[str, t.Any] = {}, user: BaseUser | None = None, require_auth: bool = True
+        self, name: str, *, selections: dict[str, t.Any] = {}, user: AbstractUser | None = None, require_auth: bool = True,
+        configurables: dict[str, str] = {}
     ) -> dr.DatasetResult:
         """
         Async method to retrieve a dataset as a DatasetResult object (with metadata) given parameter selections.
@@ -504,14 +606,17 @@ class SquirrelsProject:
         Returns:
             A DatasetResult object containing the dataset result (as a polars DataFrame), its description, and the column details.
         """
+        if user is None:
+            user = self._guest_user
+        
         scope = self._manifest_cfg.datasets[name].scope
         if require_auth and not self._auth.can_user_access_scope(user, scope):
             raise self._permission_error(user, "dataset", name, scope.name)
         
         dag = self._generate_dag(name)
+        configurables = {**self._manifest_cfg.get_default_configurables(name), **configurables}
         await dag.execute(
-            self._param_args, self._param_cfg_set, self._context_func, user, dict(selections), 
-            default_traits=self._manifest_cfg.get_default_traits()
+            self._param_args, self._param_cfg_set, self._context_func, user, dict(selections), configurables=configurables
         )
         assert isinstance(dag.target_model.result, pl.LazyFrame)
         return dr.DatasetResult(
@@ -520,7 +625,8 @@ class SquirrelsProject:
         )
     
     async def dashboard(
-        self, name: str, *, selections: dict[str, t.Any] = {}, user: BaseUser | None = None, dashboard_type: t.Type[T] = dash.Dashboard
+        self, name: str, *, selections: dict[str, t.Any] = {}, user: AbstractUser | None = None, dashboard_type: t.Type[T] = d.PngDashboard,
+        configurables: dict[str, str] = {}
     ) -> T:
         """
         Async method to retrieve a dashboard given parameter selections.
@@ -534,13 +640,18 @@ class SquirrelsProject:
         Returns:
             The dashboard type specified by the "dashboard_type" argument.
         """
+        if user is None:
+            user = self._guest_user
+        
         scope = self._dashboards[name].config.scope
         if not self._auth.can_user_access_scope(user, scope):
             raise self._permission_error(user, "dashboard", name, scope.name)
         
         async def get_dataset_df(dataset_name: str, fixed_params: dict[str, t.Any]) -> pl.DataFrame:
             final_selections = {**selections, **fixed_params}
-            result = await self.dataset(dataset_name, selections=final_selections, user=user, require_auth=False)
+            result = await self.dataset(
+                dataset_name, selections=final_selections, user=user, require_auth=False, configurables=configurables
+            )
             return result.df
         
         args = d.DashboardArgs(self._param_args, get_dataset_df)
@@ -550,12 +661,62 @@ class SquirrelsProject:
             raise KeyError(f"No dashboard file found for: {name}")
     
     async def query_models(
-        self, sql_query: str, *, selections: dict[str, t.Any] = {}, user: BaseUser | None = None
+        self, sql_query: str, *, user: AbstractUser | None = None, selections: dict[str, t.Any] = {}, configurables: dict[str, str] = {}
     ) -> dr.DatasetResult:
-        dag = await self._get_compiled_dag(sql_query=sql_query, selections=selections, user=user)
+        if user is None:
+            user = self._guest_user
+        
+        dag = await self._get_compiled_dag(user=user, sql_query=sql_query, selections=selections, configurables=configurables)
         await dag._run_models()
         assert isinstance(dag.target_model.result, pl.LazyFrame)
         return dr.DatasetResult(
             target_model_config=dag.target_model.model_config, 
             df=dag.target_model.result.collect().with_row_index("_row_num", offset=1)
         )
+
+    async def get_compiled_model_query(
+        self, model_name: str, *, user: AbstractUser | None = None, selections: dict[str, t.Any] = {}, configurables: dict[str, str] = {}
+    ) -> rm.CompiledQueryModel:
+        """
+        Compile the specified data model and return its language and compiled definition.
+        """
+        if user is None:
+            user = self._guest_user
+        
+        name = u.normalize_name(model_name)
+        models_dict = self._get_models_dict(always_python_df=False)
+        if name not in models_dict:
+            raise InvalidInputError(404, "model_not_found", f"No data model found with name: {model_name}")
+
+        model = models_dict[name]
+        # Only build, dbview, and federate models support runtime compiled definition in this context
+        if not isinstance(model, (m.BuildModel, m.DbviewModel, m.FederateModel)):
+            raise InvalidInputError(400, "unsupported_model_type", "Only build, dbview, and federate models currently support compiled definition via this endpoint")
+
+        # Build a DAG with this model as the target, without a dataset context
+        model.is_target = True
+        dag = m.DAG(None, model, models_dict, self._datalake_db_path, self._logger)
+
+        cfg = {**self._manifest_cfg.get_default_configurables(), **configurables}
+        await dag.execute(
+            self._param_args, self._param_cfg_set, self._context_func, user, selections, runquery=False, configurables=cfg
+        )
+
+        language = "sql" if isinstance(model.query_file, mq.SqlQueryFile) else "python"
+        if isinstance(model, m.BuildModel):
+            # Compile SQL build models; Python build models not yet supported
+            if isinstance(model.query_file, mq.SqlQueryFile):
+                static_models = self._get_static_models()
+                compiled = model._compile_sql_model(model.query_file, self._conn_args, static_models)
+                definition = compiled.query
+            else:
+                definition = "# Compiling Python build models is currently not supported. This will be available in a future version of Squirrels..."
+        elif isinstance(model.compiled_query, mq.SqlModelQuery):
+            definition = model.compiled_query.query 
+        elif isinstance(model.compiled_query, mq.PyModelQuery): 
+            definition = "# Compiling Python data models is currently not supported. This will be available in a future version of Squirrels..."
+        else:
+            raise NotImplementedError(f"Query type not supported: {model.compiled_query.__class__.__name__}")
+        
+        return rm.CompiledQueryModel(language=language, definition=definition, placeholders=dag.placeholders)
+