squirrels 0.5.0b4__py3-none-any.whl → 0.5.1__py3-none-any.whl

squirrels/_project.py

@@ -1,95 +1,113 @@
-from dotenv import dotenv_values
-from uuid import uuid4
+from dotenv import dotenv_values, load_dotenv
 from pathlib import Path
 import asyncio, typing as t, functools as ft, shutil, json, os
-import logging as l, matplotlib.pyplot as plt, networkx as nx, polars as pl
-import sqlglot, sqlglot.expressions
+import sqlglot, sqlglot.expressions, duckdb, polars as pl
 
-from ._auth import Authenticator, BaseUser, AuthProviderArgs, ProviderFunctionType
+from ._auth import Authenticator, AuthProviderArgs, ProviderFunctionType
+from ._schemas.auth_models import CustomUserFields, AbstractUser, GuestUser, RegisteredUser
 from ._schemas import response_models as rm
 from ._model_builder import ModelBuilder
 from ._exceptions import InvalidInputError, ConfigurationError
 from ._py_module import PyModule
 from . import _dashboards as d, _utils as u, _constants as c, _manifest as mf, _connection_set as cs
 from . import _seeds as s, _models as m, _model_configs as mc, _model_queries as mq, _sources as so
-from . import _parameter_sets as ps, _dataset_types as dr
+from . import _parameter_sets as ps, _dataset_types as dr, _logging as l
 
 T = t.TypeVar("T", bound=d.Dashboard)
 M = t.TypeVar("M", bound=m.DataModel)
 
 
-class _CustomJsonFormatter(l.Formatter):
-    def format(self, record: l.LogRecord) -> str:
-        super().format(record)
-        info = {
-            "timestamp": self.formatTime(record),
-            "project_id": record.name,
-            "level": record.levelname,
-            "message": record.getMessage(),
-            "thread": record.thread,
-            "thread_name": record.threadName,
-            "process": record.process,
-            **record.__dict__.get("info", {})
-        }
-        output = {
-            "data": record.__dict__.get("data", {}),
-            "info": info
-        }
-        return json.dumps(output)
-
-
 class SquirrelsProject:
     """
     Initiate an instance of this class to interact with a Squirrels project through Python code. For example this can be handy to experiment with the datasets produced by Squirrels in a Jupyter notebook.
     """
     
-    def __init__(self, *, filepath: str = ".", log_file: str | None = c.LOGS_FILE, log_level: str = "INFO", log_format: str = "text") -> None:
+    def __init__(
+        self, *, filepath: str = ".", load_dotenv_globally: bool = False,
+        log_to_file: bool = False, log_level: str | None = None, log_format: str | None = None,
+    ) -> None:
         """
         Constructor for SquirrelsProject class. Loads the file contents of the Squirrels project into memory as member fields.
 
         Arguments:
             filepath: The path to the Squirrels project file. Defaults to the current working directory.
-            log_level: The logging level to use. Options are "DEBUG", "INFO", and "WARNING". Default is "INFO".
-            log_file: The name of the log file to write to from the "logs/" subfolder. If None or empty string, then file logging is disabled. Default is "squirrels.log".
-            log_format: The format of the log records. Options are "text" and "json". Default is "text".
+            log_level: The logging level to use. Options are "DEBUG", "INFO", and "WARNING". Default is from SQRL_LOGGING__LOG_LEVEL environment variable or "INFO".
+            log_to_file: Whether to enable logging to file(s) in the "logs/" folder with rotation and retention policies. Default is False.
+            log_format: The format of the log records. Options are "text" and "json". Default is from SQRL_LOGGING__LOG_FORMAT environment variable or "text".
         """
         self._filepath = filepath
-        self._logger = self._get_logger(self._filepath, log_file, log_level, log_format)
-
-    def _get_logger(self, base_path: str, log_file: str | None, log_level: str, log_format: str) -> u.Logger:
-        logger = u.Logger(name=uuid4().hex)
-        logger.setLevel(log_level.upper())
-
-        handler = l.StreamHandler()
-        handler.setLevel("WARNING")
-        handler.setFormatter(l.Formatter("%(levelname)s:   %(asctime)s - %(message)s"))
-        logger.addHandler(handler)
-        
-        if log_format.lower() == "json":
-            formatter = _CustomJsonFormatter()
-        elif log_format.lower() == "text":
-            formatter = l.Formatter("[%(name)s] %(asctime)s - %(levelname)s - %(message)s")
-        else:
-            raise ValueError("log_format must be either 'text' or 'json'")
+        self._load_dotenv_globally = load_dotenv_globally
+        self._logger = self._get_logger(filepath, log_to_file, log_level, log_format)
+        self._ensure_virtual_datalake_exists(filepath)
+    
+    def _get_logger(self, filepath: str, log_to_file: bool, log_level: str | None, log_format: str | None) -> u.Logger:
+        env_vars = self._env_vars
+        # CLI arguments take precedence over environment variables
+        log_level = log_level if log_level is not None else env_vars.get(c.SQRL_LOGGING_LOG_LEVEL, "INFO")
+        log_format = log_format if log_format is not None else env_vars.get(c.SQRL_LOGGING_LOG_FORMAT, "text")
+        log_to_file = log_to_file or u.to_bool(env_vars.get(c.SQRL_LOGGING_LOG_TO_FILE, "false"))
+        log_file_size_mb = int(env_vars.get(c.SQRL_LOGGING_LOG_FILE_SIZE_MB, 50))
+        log_file_backup_count = int(env_vars.get(c.SQRL_LOGGING_LOG_FILE_BACKUP_COUNT, 1))
+        return l.get_logger(filepath, log_to_file, log_level, log_format, log_file_size_mb, log_file_backup_count)
+
+    def _ensure_virtual_datalake_exists(self, project_path: str) -> None:
+        target_path = u.Path(project_path, c.TARGET_FOLDER)
+        target_path.mkdir(parents=True, exist_ok=True)
+
+        # Attempt to set up the virtual data lake with DATA_PATH if possible
+        try:
+            is_ducklake = self._datalake_db_path.startswith("ducklake:")
             
-        if log_file:
-            path = Path(base_path, c.LOGS_FOLDER, log_file)
-            path.parent.mkdir(parents=True, exist_ok=True)
-
-            handler = l.FileHandler(path)
-            handler.setFormatter(formatter)
-            logger.addHandler(handler)
-        
-        return logger
+            data_path = self._env_vars.get(c.SQRL_VDL_DATA_PATH, c.DEFAULT_VDL_DATA_PATH)
+            data_path = data_path.format(project_path=project_path)
+            
+            options = f"(DATA_PATH '{data_path}')" if is_ducklake else ""
+            attach_stmt = f"ATTACH '{self._datalake_db_path}' AS vdl {options}"
+            with duckdb.connect() as conn:
+                conn.execute(attach_stmt)
+                # TODO: support incremental loads for build models and avoid cleaning up old files all the time
+                conn.execute("CALL ducklake_expire_snapshots('vdl', older_than => now())")
+                conn.execute("CALL ducklake_cleanup_old_files('vdl', cleanup_all => true)")
+        
+        except Exception as e:
+            if "DATA_PATH parameter" in str(e):
+                first_line = str(e).split("\n")[0]
+                note = "NOTE: Squirrels does not allow changing the data path for an existing Virtual Data Lake (VDL)"
+                raise u.ConfigurationError(f"{first_line}\n\n{note}")
+            
+            if is_ducklake and not any(x in self._datalake_db_path for x in [":sqlite:", ":postgres:", ":mysql:"]):
+                extended_error = "\n  Note: if you're using DuckDB for the metadata database, only one process can connect to the VDL at a time."
+            else:
+                extended_error = ""
+            
+            raise u.ConfigurationError(f"Failed to attach Virtual Data Lake (VDL).{extended_error}") from e
     
     @ft.cached_property
     def _env_vars(self) -> dict[str, str]:
         dotenv_files = [c.DOTENV_FILE, c.DOTENV_LOCAL_FILE]
         dotenv_vars = {}
         for file in dotenv_files:
-            dotenv_vars.update({k: v for k, v in dotenv_values(f"{self._filepath}/{file}").items() if v is not None})
+            full_path = u.Path(self._filepath, file)
+            if self._load_dotenv_globally:
+                load_dotenv(full_path)
+            dotenv_vars.update({k: v for k, v in dotenv_values(full_path).items() if v is not None})
         return {**os.environ, **dotenv_vars}
 
+    @ft.cached_property
+    def _elevated_access_level(self) -> u.ACCESS_LEVEL:
+        elevated_access_level = self._env_vars.get(c.SQRL_PERMISSIONS_ELEVATED_ACCESS_LEVEL, "admin").lower()
+
+        if elevated_access_level not in ["admin", "member", "guest"]:
+            raise u.ConfigurationError(f"{c.SQRL_PERMISSIONS_ELEVATED_ACCESS_LEVEL} has been set to an invalid access level: {elevated_access_level}")
+        
+        return elevated_access_level
+    
+    @ft.cached_property
+    def _datalake_db_path(self) -> str:
+        datalake_db_path = self._env_vars.get(c.SQRL_VDL_CATALOG_DB_PATH, c.DEFAULT_VDL_CATALOG_DB_PATH)
+        datalake_db_path = datalake_db_path.format(project_path=self._filepath)
+        return datalake_db_path
+
     @ft.cached_property
     def _manifest_cfg(self) -> mf.ManifestConfig:
         return mf.ManifestIO.load_from_file(self._logger, self._filepath, self._env_vars)
@@ -131,18 +149,19 @@ class SquirrelsProject:
         return cs.ConnectionSetIO.load_from_file(self._logger, self._filepath, self._manifest_cfg, self._conn_args)
     
     @ft.cached_property
-    def _user_cls_and_provider_functions(self) -> tuple[type[BaseUser], list[ProviderFunctionType]]:
+    def _custom_user_fields_cls_and_provider_functions(self) -> tuple[type[CustomUserFields], list[ProviderFunctionType]]:
         user_module_path = u.Path(self._filepath, c.PYCONFIGS_FOLDER, c.USER_FILE)
         user_module = PyModule(user_module_path)
         
-        User = user_module.get_func_or_class("User", default_attr=BaseUser)  # adds to Authenticator.providers as side effect
+        # Load CustomUserFields class (adds to Authenticator.providers as side effect)
+        CustomUserFieldsCls = user_module.get_func_or_class("CustomUserFields", default_attr=CustomUserFields)
         provider_functions = Authenticator.providers
         Authenticator.providers = []
         
-        if not issubclass(User, BaseUser):
-            raise ConfigurationError(f"User class in '{c.USER_FILE}' must inherit from BaseUser")
+        if not issubclass(CustomUserFieldsCls, CustomUserFields):
+            raise ConfigurationError(f"CustomUserFields class in '{c.USER_FILE}' must inherit from CustomUserFields")
         
-        return User, provider_functions
+        return CustomUserFieldsCls, provider_functions
     
     @ft.cached_property
     def _auth_args(self) -> AuthProviderArgs:
@@ -150,9 +169,20 @@ class SquirrelsProject:
         return AuthProviderArgs(conn_args.project_path, conn_args.proj_vars, conn_args.env_vars)
     
     @ft.cached_property
-    def _auth(self) -> Authenticator[BaseUser]:
-        User, provider_functions = self._user_cls_and_provider_functions
-        return Authenticator(self._logger, self._filepath, self._auth_args, provider_functions, user_cls=User)
+    def _auth(self) -> Authenticator:
+        CustomUserFieldsCls, provider_functions = self._custom_user_fields_cls_and_provider_functions
+        external_only = (self._manifest_cfg.authentication.type == mf.AuthenticationType.EXTERNAL)
+        return Authenticator(self._logger, self._filepath, self._auth_args, provider_functions, custom_user_fields_cls=CustomUserFieldsCls, external_only=external_only)
+    
+    @ft.cached_property
+    def _guest_user(self) -> AbstractUser:
+        custom_fields = self._auth.CustomUserFields()
+        return GuestUser(username="", custom_fields=custom_fields)
+
+    @ft.cached_property
+    def _admin_user(self) -> AbstractUser:
+        custom_fields = self._auth.CustomUserFields()
+        return RegisteredUser(username="", access_level="admin", custom_fields=custom_fields)
     
     @ft.cached_property
     def _param_args(self) -> ps.ParametersArgs:
@@ -162,7 +192,7 @@ class SquirrelsProject:
     @ft.cached_property
     def _param_cfg_set(self) -> ps.ParameterConfigsSet:
         return ps.ParameterConfigsSetIO.load_from_file(
-            self._logger, self._filepath, self._manifest_cfg, self._seeds, self._conn_set, self._param_args
+            self._logger, self._filepath, self._manifest_cfg, self._seeds, self._conn_set, self._param_args, self._datalake_db_path
         )
     
     @ft.cached_property
@@ -189,11 +219,6 @@ class SquirrelsProject:
         env.filters["quote_and_join"] = quote_and_join
         return env
 
-    @ft.cached_property
-    def _duckdb_venv_path(self) -> str:
-        duckdb_filepath_setting_val = self._env_vars.get(c.SQRL_DUCKDB_VENV_DB_FILE_PATH, f"{c.TARGET_FOLDER}/{c.DUCKDB_VENV_FILE}")
-        return str(Path(self._filepath, duckdb_filepath_setting_val))
-    
     def close(self) -> None:
         """
         Deliberately close any open resources within the Squirrels project, such as database connections (instead of relying on the garbage collector).
@@ -228,20 +253,20 @@ class SquirrelsProject:
         return models_dict
 
 
-    async def build(self, *, full_refresh: bool = False, select: str | None = None, stage_file: bool = False) -> None:
+    async def build(self, *, full_refresh: bool = False, select: str | None = None) -> None:
         """
-        Build the virtual data environment for the Squirrels project
+        Build the Virtual Data Lake (VDL) for the Squirrels project
 
         Arguments:
-            full_refresh: Whether to drop all tables and rebuild the virtual data environment from scratch. Default is False.
-            stage_file: Whether to stage the DuckDB file to overwrite the existing one later if the virtual data environment is in use. Default is False.
+            full_refresh: Whether to drop all tables and rebuild the VDL from scratch. Default is False.
+            select: The name of a specific model to build. If None, all models are built. Default is None.
         """
         models_dict: dict[str, m.StaticModel] = self._get_static_models()
-        builder = ModelBuilder(self._duckdb_venv_path, self._conn_set, models_dict, self._conn_args, self._logger)
-        await builder.build(full_refresh, select, stage_file)
+        builder = ModelBuilder(self._datalake_db_path, self._conn_set, models_dict, self._conn_args, self._logger)
+        await builder.build(full_refresh, select)
 
     def _get_models_dict(self, always_python_df: bool) -> dict[str, m.DataModel]:
-        models_dict: dict[str, m.DataModel] = dict(self._get_static_models())
+        models_dict: dict[str, m.DataModel] = self._get_static_models()
         
         for name, val in self._dbview_model_files.items():
             self._add_model(models_dict, m.DbviewModel(
@@ -257,19 +282,18 @@ class SquirrelsProject:
         
         return models_dict
     
-    def _generate_dag(self, dataset: str, *, target_model_name: str | None = None, always_python_df: bool = False) -> m.DAG:
-        models_dict = self._get_models_dict(always_python_df)
+    def _generate_dag(self, dataset: str) -> m.DAG:
+        models_dict = self._get_models_dict(always_python_df=False)
         
         dataset_config = self._manifest_cfg.datasets[dataset]
-        target_model_name = dataset_config.model if target_model_name is None else target_model_name
-        target_model = models_dict[target_model_name]
+        target_model = models_dict[dataset_config.model]
         target_model.is_target = True
-        dag = m.DAG(dataset_config, target_model, models_dict, self._duckdb_venv_path, self._logger)
+        dag = m.DAG(dataset_config, target_model, models_dict, self._datalake_db_path, self._logger)
         
         return dag
     
-    def _generate_dag_with_fake_target(self, sql_query: str | None) -> m.DAG:
-        models_dict = self._get_models_dict(always_python_df=False)
+    def _generate_dag_with_fake_target(self, sql_query: str | None, *, always_python_df: bool = False) -> m.DAG:
+        models_dict = self._get_models_dict(always_python_df=always_python_df)
 
         if sql_query is None:
             dependencies = set(models_dict.keys())
@@ -279,54 +303,45 @@ class SquirrelsProject:
             substitutions = {}
             for model_name in dependencies:
                 model = models_dict[model_name]
-                if isinstance(model, m.SourceModel) and not model.model_config.load_to_duckdb:
-                    raise InvalidInputError(400, "Unqueryable source model", f"Source model '{model_name}' cannot be queried with DuckDB")
-                if isinstance(model, (m.SourceModel, m.BuildModel)):
-                    substitutions[model_name] = f"venv.{model_name}"
+                if isinstance(model, m.SourceModel) and not model.is_queryable:
+                    raise InvalidInputError(400, "cannot_query_source_model", f"Source model '{model_name}' cannot be queried with DuckDB")
+                if isinstance(model, m.BuildModel):
+                    substitutions[model_name] = f"vdl.{model_name}"
+                elif isinstance(model, m.SourceModel):
+                    if model.model_config.load_to_vdl:
+                        substitutions[model_name] = f"vdl.{model_name}"
+                    else:
+                        # DuckDB connection without load_to_vdl - reference via attached database
+                        conn_name = model.model_config.get_connection()
+                        table_name = model.model_config.get_table()
+                        substitutions[model_name] = f"db_{conn_name}.{table_name}"
             
             sql_query = parsed.transform(
-                lambda node: sqlglot.expressions.Table(this=substitutions[node.name])
+                lambda node: sqlglot.expressions.Table(this=substitutions[node.name], alias=node.alias)
                 if isinstance(node, sqlglot.expressions.Table) and node.name in substitutions
                 else node
             ).sql()
         
         model_config = mc.FederateModelConfig(depends_on=dependencies)
-        query_file = mq.SqlQueryFile("", sql_query or "")
+        query_file = mq.SqlQueryFile("", sql_query or "SELECT 1")
         fake_target_model = m.FederateModel(
             "__fake_target", model_config, query_file, logger=self._logger, env_vars=self._env_vars, conn_set=self._conn_set, j2_env=self._j2_env
         )
         fake_target_model.is_target = True
-        dag = m.DAG(None, fake_target_model, models_dict, self._duckdb_venv_path, self._logger)
+        dag = m.DAG(None, fake_target_model, models_dict, self._datalake_db_path, self._logger)
         return dag
     
-    def _draw_dag(self, dag: m.DAG, output_folder: Path) -> None:
-        color_map = {
-            m.ModelType.SEED: "green", m.ModelType.DBVIEW: "red", m.ModelType.FEDERATE: "skyblue",
-            m.ModelType.BUILD: "purple", m.ModelType.SOURCE: "orange"
-        }
-
-        G = dag.to_networkx_graph()
-        
-        fig, _ = plt.subplots()
-        pos = nx.multipartite_layout(G, subset_key="layer")
-        colors = [color_map[node[1]] for node in G.nodes(data="model_type")] # type: ignore
-        nx.draw(G, pos=pos, node_shape='^', node_size=1000, node_color=colors, arrowsize=20)
+    async def _get_compiled_dag(
+        self, user: AbstractUser, *, sql_query: str | None = None, selections: dict[str, t.Any] = {}, configurables: dict[str, str] = {}, 
+        always_python_df: bool = False
+    ) -> m.DAG:
+        dag = self._generate_dag_with_fake_target(sql_query, always_python_df=always_python_df)
         
-        y_values = [val[1] for val in pos.values()]
-        scale = max(y_values) - min(y_values) if len(y_values) > 0 else 0
-        label_pos = {key: (val[0], val[1]-0.002-0.1*scale) for key, val in pos.items()}
-        nx.draw_networkx_labels(G, pos=label_pos, font_size=8)
-        
-        fig.tight_layout()
-        plt.margins(x=0.1, y=0.1)
-        fig.savefig(Path(output_folder, "dag.png"))
-        plt.close(fig)
-    
-    async def _get_compiled_dag(self, *, sql_query: str | None = None, selections: dict[str, t.Any] = {}, user: BaseUser | None = None) -> m.DAG:
-        dag = self._generate_dag_with_fake_target(sql_query)
-        
-        default_traits = self._manifest_cfg.get_default_traits()
-        await dag.execute(self._param_args, self._param_cfg_set, self._context_func, user, selections, runquery=False, default_traits=default_traits)
+        configurables = {**self._manifest_cfg.get_default_configurables(), **configurables}
+        await dag.execute(
+            self._param_args, self._param_cfg_set, self._context_func, user, selections,
+            runquery=False, configurables=configurables
+        )
         return dag
     
     def _get_all_connections(self) -> list[rm.ConnectionItemModel]:
@@ -347,7 +362,7 @@ class SquirrelsProject:
         Returns:
             A list of DataModelItem objects
         """
-        compiled_dag = await self._get_compiled_dag()
+        compiled_dag = await self._get_compiled_dag(self._admin_user)
         return self._get_all_data_models(compiled_dag)
 
     def _get_all_data_lineage(self, compiled_dag: m.DAG) -> list[rm.LineageRelation]:
@@ -376,130 +391,172 @@ class SquirrelsProject:
         Returns:
             A list of LineageRelation objects
         """
-        compiled_dag = await self._get_compiled_dag()
+        compiled_dag = await self._get_compiled_dag(self._admin_user)
         return self._get_all_data_lineage(compiled_dag)
 
-    async def _write_dataset_outputs_given_test_set(
-        self, dataset: str, select: str, test_set: str | None, runquery: bool, recurse: bool
-    ) -> t.Any | None:
-        dataset_conf = self._manifest_cfg.datasets[dataset]
-        default_test_set_conf = self._manifest_cfg.get_default_test_set(dataset)
-        if test_set in self._manifest_cfg.selection_test_sets:
-            test_set_conf = self._manifest_cfg.selection_test_sets[test_set]
-        elif test_set is None or test_set == default_test_set_conf.name:
-            test_set, test_set_conf = default_test_set_conf.name, default_test_set_conf
-        else:
-            raise ConfigurationError(f"No test set named '{test_set}' was found when compiling dataset '{dataset}'. The test set must be defined if not default for dataset.")
-        
-        error_msg_intro = f"Cannot compile dataset '{dataset}' with test set '{test_set}'."
-        if test_set_conf.datasets is not None and dataset not in test_set_conf.datasets:
-            raise ConfigurationError(f"{error_msg_intro}\n Applicable datasets for test set '{test_set}' does not include dataset '{dataset}'.")
-        
-        user_attributes = test_set_conf.user_attributes.copy() if test_set_conf.user_attributes is not None else {}
-        selections = test_set_conf.parameters.copy()
-        username, is_admin = user_attributes.pop("username", ""), user_attributes.pop("is_admin", False)
-        if test_set_conf.is_authenticated:
-            user = self._auth.User(username=username, is_admin=is_admin, **user_attributes)
-        elif dataset_conf.scope == mf.PermissionScope.PUBLIC:
-            user = None
-        else:
-            raise ConfigurationError(f"{error_msg_intro}\n Non-public datasets require a test set with 'user_attributes' section defined")
-        
-        if dataset_conf.scope == mf.PermissionScope.PRIVATE and not is_admin:
-            raise ConfigurationError(f"{error_msg_intro}\n Private datasets require a test set with user_attribute 'is_admin' set to true")
-
-        # always_python_df is set to True for creating CSV files from results (when runquery is True)
-        dag = self._generate_dag(dataset, target_model_name=select, always_python_df=runquery)
-        await dag.execute(
-            self._param_args, self._param_cfg_set, self._context_func, user, selections, 
-            runquery=runquery, recurse=recurse, default_traits=self._manifest_cfg.get_default_traits()
-        )
-        
-        output_folder = Path(self._filepath, c.TARGET_FOLDER, c.COMPILE_FOLDER, dataset, test_set)
-        if output_folder.exists():
-            shutil.rmtree(output_folder)
-        output_folder.mkdir(parents=True, exist_ok=True)
-        
-        def write_placeholders() -> None:
-            output_filepath = Path(output_folder, "placeholders.json")
-            with open(output_filepath, 'w') as f:
-                json.dump(dag.placeholders, f, indent=4)
-        
-        def write_model_outputs(model: m.DataModel) -> None:
-            assert isinstance(model, m.QueryModel)
-            subfolder = c.DBVIEWS_FOLDER if model.model_type == m.ModelType.DBVIEW else c.FEDERATES_FOLDER
-            subpath = Path(output_folder, subfolder)
-            subpath.mkdir(parents=True, exist_ok=True)
-            if isinstance(model.compiled_query, mq.SqlModelQuery):
-                output_filepath = Path(subpath, model.name+'.sql')
-                query = model.compiled_query.query
-                with open(output_filepath, 'w') as f:
-                    f.write(query)
-            if runquery and isinstance(model.result, pl.LazyFrame):
-                output_filepath = Path(subpath, model.name+'.csv')
-                model.result.collect().write_csv(output_filepath)
-
-        write_placeholders()
-        all_model_names = dag.get_all_query_models()
-        coroutines = [asyncio.to_thread(write_model_outputs, dag.models_dict[name]) for name in all_model_names]
-        await u.asyncio_gather(coroutines)
-
-        if recurse:
-            self._draw_dag(dag, output_folder)
-        
-        if isinstance(dag.target_model, m.QueryModel) and dag.target_model.compiled_query is not None:
-            return dag.target_model.compiled_query.query
-    
     async def compile(
-        self, *, dataset: str | None = None, do_all_datasets: bool = False, selected_model: str | None = None, test_set: str | None = None, 
-        do_all_test_sets: bool = False, runquery: bool = False
+        self, *, selected_model: str | None = None, test_set: str | None = None, do_all_test_sets: bool = False,
+        runquery: bool = False, clear: bool = False, buildtime_only: bool = False, runtime_only: bool = False
     ) -> None:
         """
-        Async method to compile the SQL templates into files in the "target/" folder. Same functionality as the "sqrl compile" CLI.
+        Compile models into the "target/compile" folder.
 
-        Although all arguments are "optional", the "dataset" argument is required if "do_all_datasets" argument is False.
+        Behavior:
+        - Buildtime outputs: target/compile/buildtime/*.sql (for SQL build models) and dag.png
+        - Runtime outputs: target/compile/runtime/[test_set]/dbviews/*.sql, federates/*.sql, dag.png
+          If runquery=True, also write CSVs for runtime models.
+        - Options: clear entire compile folder first; compile only buildtime or only runtime.
 
         Arguments:
-            dataset: The name of the dataset to compile. Ignored if "do_all_datasets" argument is True, but required (i.e., cannot be None) if "do_all_datasets" is False. Default is None.
-            do_all_datasets: If True, compile all datasets and ignore the "dataset" argument. Default is False.
             selected_model: The name of the model to compile. If specified, the compiled SQL query is also printed in the terminal. If None, all models for the selected dataset are compiled. Default is None.
             test_set: The name of the test set to compile with. If None, the default test set is used (which can vary by dataset). Ignored if `do_all_test_sets` argument is True. Default is None.
             do_all_test_sets: Whether to compile all applicable test sets for the selected dataset(s). If True, the `test_set` argument is ignored. Default is False.
-            runquery**: Whether to run all compiled queries and save each result as a CSV file. If True and `selected_model` is specified, all upstream models of the selected model is compiled as well. Default is False.
+            runquery: Whether to run all compiled queries and save each result as a CSV file. If True and `selected_model` is specified, all upstream models of the selected model is compiled as well. Default is False.
+            clear: Whether to clear the "target/compile/" folder before compiling. Default is False.
+            buildtime_only: Whether to compile only buildtime models. Default is False.
+            runtime_only: Whether to compile only runtime models. Default is False.
         """
-        recurse = True
-        if do_all_datasets:
-            selected_models = [(dataset.name, dataset.model) for dataset in self._manifest_cfg.datasets.values()]
-        else:
-            assert isinstance(dataset, str), "argument 'dataset' must be provided a string value if argument 'do_all_datasets' is False"
-            assert dataset in self._manifest_cfg.datasets, f"dataset '{dataset}' not found in {c.MANIFEST_FILE}"
-            if selected_model is None:
-                selected_model = self._manifest_cfg.datasets[dataset].model
+        border = "=" * 80
+        underlines = "-" * len(border)
+
+        compile_root = Path(self._filepath, c.TARGET_FOLDER, c.COMPILE_FOLDER)
+        if clear and compile_root.exists():
+            shutil.rmtree(compile_root)
+
+        models_dict = self._get_models_dict(always_python_df=False)
+
+        if selected_model is not None:
+            selected_model = u.normalize_name(selected_model)
+            if selected_model not in models_dict:
+                print(f"No such model found: {selected_model}")
+                return
+            if not isinstance(models_dict[selected_model], m.QueryModel):
+                print(f"Model '{selected_model}' is not a query model. Nothing to do.")
+                return
+        
+        model_to_compile = None
+
+        # Buildtime compilation
+        if not runtime_only:
+            print(underlines)
+            print(f"Compiling buildtime models")
+            print(underlines)
+
+            buildtime_folder = Path(compile_root, c.COMPILE_BUILDTIME_FOLDER)
+            buildtime_folder.mkdir(parents=True, exist_ok=True)
+
+            def write_buildtime_model(model: m.DataModel, static_models: dict[str, m.StaticModel]) -> None:
+                if not isinstance(model, m.BuildModel):
+                    return
+                
+                model.compile_for_build(self._conn_args, static_models)
+
+                if isinstance(model.compiled_query, mq.SqlModelQuery):
+                    out_path = Path(buildtime_folder, f"{model.name}.sql")
+                    with open(out_path, 'w') as f:
+                        f.write(model.compiled_query.query)
+                    print(f"Successfully compiled build model: {model.name}")
+                elif isinstance(model.compiled_query, mq.PyModelQuery):
+                    print(f"The build model '{model.name}' is in Python. Compilation for Python is not supported yet.")
+
+            static_models = self._get_static_models()
+            if selected_model is not None:
+                model_to_compile = models_dict[selected_model]
+                write_buildtime_model(model_to_compile, static_models)
             else:
-                recurse = False
-            selected_models = [(dataset, selected_model)]
-        
-        coroutines: list[t.Coroutine] = []
-        for dataset, selected_model in selected_models:
-            if do_all_test_sets:
-                for test_set_name in self._manifest_cfg.get_applicable_test_sets(dataset):
-                    coroutine = self._write_dataset_outputs_given_test_set(dataset, selected_model, test_set_name, runquery, recurse)
-                    coroutines.append(coroutine)
+                coros = [asyncio.to_thread(write_buildtime_model, m, static_models) for m in static_models.values()]
+                await u.asyncio_gather(coros)
             
-            coroutine = self._write_dataset_outputs_given_test_set(dataset, selected_model, test_set, runquery, recurse)
-            coroutines.append(coroutine)
-        
-        queries = await u.asyncio_gather(coroutines)
+            print(underlines)
+            print()
         
-        print(f"Compiled successfully! See the '{c.TARGET_FOLDER}/' folder for results.")
-        print()
-        if not recurse and len(queries) == 1 and isinstance(queries[0], str):
-            print(queries[0])
+        # Runtime compilation
+        if not buildtime_only:
+            if do_all_test_sets:
+                test_set_names_set = set(self._manifest_cfg.selection_test_sets.keys())
+                test_set_names_set.add(c.DEFAULT_TEST_SET_NAME)
+                test_set_names = list(test_set_names_set)
+            else:
+                test_set_names = [test_set or c.DEFAULT_TEST_SET_NAME]
+
+            for ts_name in test_set_names:
+                print(underlines)
+                print(f"Compiling runtime models (test set '{ts_name}')")
+                print(underlines)
+
+                # Build user and selections from test set config if present
+                ts_conf = self._manifest_cfg.selection_test_sets.get(ts_name, self._manifest_cfg.get_default_test_set())
+                # Separate base fields from custom fields
+                access_level = ts_conf.user.access_level
+                custom_fields = self._auth.CustomUserFields(**ts_conf.user.custom_fields)
+                if access_level == "guest":
+                    user = GuestUser(username="", custom_fields=custom_fields)
+                else:
+                    user = RegisteredUser(username="", access_level=access_level, custom_fields=custom_fields)
+
+                # Generate DAG across all models. When runquery=True, force models to produce Python dataframes so CSVs can be written.
+                dag = await self._get_compiled_dag(
+                    user=user, selections=ts_conf.parameters, configurables=ts_conf.configurables, always_python_df=runquery,
+                )
+                if runquery:
+                    await dag._run_models()
+
+                # Prepare output folders
+                runtime_folder = Path(compile_root, c.COMPILE_RUNTIME_FOLDER, ts_name)
+                dbviews_folder = Path(runtime_folder, c.DBVIEWS_FOLDER)
+                federates_folder = Path(runtime_folder, c.FEDERATES_FOLDER)
+                dbviews_folder.mkdir(parents=True, exist_ok=True)
+                federates_folder.mkdir(parents=True, exist_ok=True)
+                with open(Path(runtime_folder, "placeholders.json"), "w") as f:
+                    json.dump(dag.placeholders, f)
+
+                # Function to write runtime models
+                def write_runtime_model(model: m.DataModel) -> None:
+                    if not isinstance(model, m.QueryModel):
+                        return
+                    
+                    if model.model_type not in (m.ModelType.DBVIEW, m.ModelType.FEDERATE):
+                        return
+                    
+                    subfolder = dbviews_folder if model.model_type == m.ModelType.DBVIEW else federates_folder
+                    model_type = "dbview" if model.model_type == m.ModelType.DBVIEW else "federate"
+
+                    if isinstance(model.compiled_query, mq.SqlModelQuery):
+                        out_sql = Path(subfolder, f"{model.name}.sql")
+                        with open(out_sql, 'w') as f:
+                            f.write(model.compiled_query.query)
+                        print(f"Successfully compiled {model_type} model: {model.name}")
+                    elif isinstance(model.compiled_query, mq.PyModelQuery):
+                        print(f"The {model_type} model '{model.name}' is in Python. Compilation for Python is not supported yet.")
+                    
+                    if runquery and isinstance(model.result, pl.LazyFrame):
+                        out_csv = Path(subfolder, f"{model.name}.csv")
+                        model.result.collect().write_csv(out_csv)
+                        print(f"Successfully created CSV for {model_type} model: {model.name}")
+
+                # If selected_model is provided for runtime, only emit that model's outputs
+                if selected_model is not None:
+                    model_to_compile = dag.models_dict[selected_model]
+                    write_runtime_model(model_to_compile)
+                else:
+                    coros = [asyncio.to_thread(write_runtime_model, model) for model in dag.models_dict.values()]
+                    await u.asyncio_gather(coros)
+
+                print(underlines)
+                print()
+
+        print(f"All compilations complete! See the '{c.TARGET_FOLDER}/{c.COMPILE_FOLDER}/' folder for results.")
+        if model_to_compile and isinstance(model_to_compile, m.QueryModel) and isinstance(model_to_compile.compiled_query, mq.SqlModelQuery):
+            print()
+            print(border)
+            print(f"Compiled SQL query for model '{model_to_compile.name}':")
+            print(underlines)
+            print(model_to_compile.compiled_query.query)
+            print(border)
             print()
 
-    def _permission_error(self, user: BaseUser | None, data_type: str, data_name: str, scope: str) -> InvalidInputError:
-        username = "" if user is None else f" '{user.username}'"
-        return InvalidInputError(403, f"Unauthorized access to {data_type}", f"User{username} does not have permission to access {scope} {data_type}: {data_name}")
+    def _permission_error(self, user: AbstractUser, data_type: str, data_name: str, scope: str) -> InvalidInputError:
+        return InvalidInputError(403, f"unauthorized_access_to_{data_type}", f"User '{user}' does not have permission to access {scope} {data_type}: {data_name}")
     
     def seed(self, name: str) -> pl.LazyFrame:
         """
@@ -535,7 +592,8 @@ class SquirrelsProject:
         )
     
     async def dataset(
-        self, name: str, *, selections: dict[str, t.Any] = {}, user: BaseUser | None = None, require_auth: bool = True
+        self, name: str, *, selections: dict[str, t.Any] = {}, user: AbstractUser | None = None, require_auth: bool = True,
+        configurables: dict[str, str] = {}
     ) -> dr.DatasetResult:
         """
         Async method to retrieve a dataset as a DatasetResult object (with metadata) given parameter selections.
@@ -548,14 +606,17 @@ class SquirrelsProject:
         Returns:
             A DatasetResult object containing the dataset result (as a polars DataFrame), its description, and the column details.
         """
+        if user is None:
+            user = self._guest_user
+        
         scope = self._manifest_cfg.datasets[name].scope
         if require_auth and not self._auth.can_user_access_scope(user, scope):
             raise self._permission_error(user, "dataset", name, scope.name)
         
         dag = self._generate_dag(name)
+        configurables = {**self._manifest_cfg.get_default_configurables(name), **configurables}
         await dag.execute(
-            self._param_args, self._param_cfg_set, self._context_func, user, dict(selections), 
-            default_traits=self._manifest_cfg.get_default_traits()
+            self._param_args, self._param_cfg_set, self._context_func, user, dict(selections), configurables=configurables
         )
         assert isinstance(dag.target_model.result, pl.LazyFrame)
         return dr.DatasetResult(
@@ -564,7 +625,8 @@ class SquirrelsProject:
         )
     
     async def dashboard(
-        self, name: str, *, selections: dict[str, t.Any] = {}, user: BaseUser | None = None, dashboard_type: t.Type[T] = d.PngDashboard
+        self, name: str, *, selections: dict[str, t.Any] = {}, user: AbstractUser | None = None, dashboard_type: t.Type[T] = d.PngDashboard,
+        configurables: dict[str, str] = {}
     ) -> T:
         """
         Async method to retrieve a dashboard given parameter selections.
@@ -578,13 +640,18 @@ class SquirrelsProject:
         Returns:
             The dashboard type specified by the "dashboard_type" argument.
         """
+        if user is None:
+            user = self._guest_user
+        
         scope = self._dashboards[name].config.scope
         if not self._auth.can_user_access_scope(user, scope):
             raise self._permission_error(user, "dashboard", name, scope.name)
         
         async def get_dataset_df(dataset_name: str, fixed_params: dict[str, t.Any]) -> pl.DataFrame:
             final_selections = {**selections, **fixed_params}
-            result = await self.dataset(dataset_name, selections=final_selections, user=user, require_auth=False)
+            result = await self.dataset(
+                dataset_name, selections=final_selections, user=user, require_auth=False, configurables=configurables
+            )
             return result.df
         
         args = d.DashboardArgs(self._param_args, get_dataset_df)
@@ -594,12 +661,62 @@ class SquirrelsProject:
             raise KeyError(f"No dashboard file found for: {name}")
     
     async def query_models(
-        self, sql_query: str, *, selections: dict[str, t.Any] = {}, user: BaseUser | None = None
+        self, sql_query: str, *, user: AbstractUser | None = None, selections: dict[str, t.Any] = {}, configurables: dict[str, str] = {}
     ) -> dr.DatasetResult:
-        dag = await self._get_compiled_dag(sql_query=sql_query, selections=selections, user=user)
+        if user is None:
+            user = self._guest_user
+        
+        dag = await self._get_compiled_dag(user=user, sql_query=sql_query, selections=selections, configurables=configurables)
         await dag._run_models()
         assert isinstance(dag.target_model.result, pl.LazyFrame)
         return dr.DatasetResult(
             target_model_config=dag.target_model.model_config, 
             df=dag.target_model.result.collect().with_row_index("_row_num", offset=1)
         )
+
+    async def get_compiled_model_query(
+        self, model_name: str, *, user: AbstractUser | None = None, selections: dict[str, t.Any] = {}, configurables: dict[str, str] = {}
+    ) -> rm.CompiledQueryModel:
+        """
+        Compile the specified data model and return its language and compiled definition.
+        """
+        if user is None:
+            user = self._guest_user
+        
+        name = u.normalize_name(model_name)
+        models_dict = self._get_models_dict(always_python_df=False)
+        if name not in models_dict:
+            raise InvalidInputError(404, "model_not_found", f"No data model found with name: {model_name}")
+
+        model = models_dict[name]
+        # Only build, dbview, and federate models support runtime compiled definition in this context
+        if not isinstance(model, (m.BuildModel, m.DbviewModel, m.FederateModel)):
+            raise InvalidInputError(400, "unsupported_model_type", "Only build, dbview, and federate models currently support compiled definition via this endpoint")
+
+        # Build a DAG with this model as the target, without a dataset context
+        model.is_target = True
+        dag = m.DAG(None, model, models_dict, self._datalake_db_path, self._logger)
+
+        cfg = {**self._manifest_cfg.get_default_configurables(), **configurables}
+        await dag.execute(
+            self._param_args, self._param_cfg_set, self._context_func, user, selections, runquery=False, configurables=cfg
+        )
+
+        language = "sql" if isinstance(model.query_file, mq.SqlQueryFile) else "python"
+        if isinstance(model, m.BuildModel):
+            # Compile SQL build models; Python build models not yet supported
+            if isinstance(model.query_file, mq.SqlQueryFile):
+                static_models = self._get_static_models()
+                compiled = model._compile_sql_model(model.query_file, self._conn_args, static_models)
+                definition = compiled.query
+            else:
+                definition = "# Compiling Python build models is currently not supported. This will be available in a future version of Squirrels..."
+        elif isinstance(model.compiled_query, mq.SqlModelQuery):
+            definition = model.compiled_query.query 
+        elif isinstance(model.compiled_query, mq.PyModelQuery): 
+            definition = "# Compiling Python data models is currently not supported. This will be available in a future version of Squirrels..."
+        else:
+            raise NotImplementedError(f"Query type not supported: {model.compiled_query.__class__.__name__}")
+        
+        return rm.CompiledQueryModel(language=language, definition=definition, placeholders=dag.placeholders)
+